diff --git a/.ci/before_build_wheel.sh b/.ci/before_build_wheel.sh new file mode 100644 index 0000000000..56108dcd60 --- /dev/null +++ b/.ci/before_build_wheel.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -xeu + +# On 32-bit Linux platforms, we need libatomic1 to use rustup +if command -v yum &> /dev/null; then + yum install -y libatomic +fi + +# Install a Rust toolchain +curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.82.0 -y --profile minimal diff --git a/.ci/scripts/calculate_jobs.py b/.ci/scripts/calculate_jobs.py index ea278173db..5249acdc5d 100755 --- a/.ci/scripts/calculate_jobs.py +++ b/.ci/scripts/calculate_jobs.py @@ -60,7 +60,7 @@ trial_postgres_tests = [ { "python-version": "3.9", "database": "postgres", - "postgres-version": "11", + "postgres-version": "13", "extras": "all", } ] diff --git a/.ci/scripts/check_lockfile.py b/.ci/scripts/check_lockfile.py index 19cec7ddd6..46d3952b4c 100755 --- a/.ci/scripts/check_lockfile.py +++ b/.ci/scripts/check_lockfile.py @@ -11,12 +11,12 @@ with open("poetry.lock", "rb") as f: try: lock_version = lockfile["metadata"]["lock-version"] - assert lock_version == "2.0" + assert lock_version == "2.1" except Exception: print( """\ - Lockfile is not version 2.0. You probably need to upgrade poetry on your local box - and re-run `poetry lock --no-update`. See the Poetry cheat sheet at + Lockfile is not version 2.1. You probably need to upgrade poetry on your local box + and re-run `poetry lock`. See the Poetry cheat sheet at https://element-hq.github.io/synapse/develop/development/dependencies.html """ ) diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh index 8cc41d3dca..3816e03240 100755 --- a/.ci/scripts/test_synapse_port_db.sh +++ b/.ci/scripts/test_synapse_port_db.sh @@ -61,7 +61,7 @@ poetry run update_synapse_database --database-config .ci/postgres-config-unporte echo "+++ Comparing ported schema with unported schema" # Ignore the tables that portdb creates. (Should it tidy them up when the porting is completed?) psql synapse -c "DROP TABLE port_from_sqlite3;" -pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse_unported > unported.sql -pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse > ported.sql +pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner --restrict-key=TESTING synapse_unported > unported.sql +pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner --restrict-key=TESTING synapse > ported.sql # By default, `diff` returns zero if there are no changes and nonzero otherwise -diff -u unported.sql ported.sql | tee schema_diff \ No newline at end of file +diff -u unported.sql ported.sql | tee schema_diff diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 07d4f6dfce..f8e60815fa 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,5 +9,4 @@ - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. -* [ ] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct - (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) +* [ ] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ebf866e3d5..dc65625c6f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,7 +5,7 @@ name: Build docker images on: push: tags: ["v*"] - branches: [ master, main, develop ] + branches: [master, main, develop] workflow_dispatch: permissions: @@ -14,26 +14,24 @@ permissions: id-token: write # needed for signing the images with GitHub OIDC Token jobs: build: - runs-on: ubuntu-latest + name: Build and push image for ${{ matrix.platform }} + runs-on: ${{ matrix.runs_on }} + strategy: + matrix: + include: + - platform: linux/amd64 + runs_on: ubuntu-24.04 + suffix: linux-amd64 + - platform: linux/arm64 + runs_on: ubuntu-24.04-arm + suffix: linux-arm64 steps: - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Inspect builder - run: docker buildx inspect - - - name: Install Cosign - uses: sigstore/cosign-installer@v3.7.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see @@ -43,25 +41,91 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Calculate docker image tag - id: set-tag - uses: docker/metadata-action@master + - name: Build and push by digest + id: build + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: - images: | + push: true + labels: | + gitsha1=${{ github.sha }} + org.opencontainers.image.version=${{ env.SYNAPSE_VERSION }} + tags: | docker.io/matrixdotorg/synapse ghcr.io/element-hq/synapse + file: "docker/Dockerfile" + platforms: ${{ matrix.platform }} + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p ${{ runner.temp }}/digests + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.suffix }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: Push merged images to ${{ matrix.repository }} + runs-on: ubuntu-latest + strategy: + matrix: + repository: + - docker.io/matrixdotorg/synapse + - ghcr.io/element-hq/synapse + + needs: + - build + steps: + - name: Download digests + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true + + - name: Log in to DockerHub + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + if: ${{ startsWith(matrix.repository, 'docker.io') }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GHCR + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + if: ${{ startsWith(matrix.repository, 'ghcr.io') }} + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Install Cosign + uses: sigstore/cosign-installer@d7543c93d881b35a8faa02e8e3605f69b7a1ce62 # v3.10.0 + + - name: Calculate docker image tag + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: ${{ matrix.repository }} flavor: | latest=false tags: | @@ -69,31 +133,23 @@ jobs: type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }} type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} type=pep440,pattern={{raw}} + type=sha - - name: Build and push all platforms - id: build-and-push - uses: docker/build-push-action@v6 - with: - push: true - labels: | - gitsha1=${{ github.sha }} - org.opencontainers.image.version=${{ env.SYNAPSE_VERSION }} - tags: "${{ steps.set-tag.outputs.tags }}" - file: "docker/Dockerfile" - platforms: linux/amd64,linux/arm64 - - # arm64 builds OOM without the git fetch setting. c.f. - # https://github.com/rust-lang/cargo/issues/10583 - build-args: | - CARGO_NET_GIT_FETCH_WITH_CLI=true - - - name: Sign the images with GitHub OIDC Token + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests env: - DIGEST: ${{ steps.build-and-push.outputs.digest }} - TAGS: ${{ steps.set-tag.outputs.tags }} + REPOSITORY: ${{ matrix.repository }} run: | - images="" - for tag in ${TAGS}; do - images+="${tag}@${DIGEST} " + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf "$REPOSITORY@sha256:%s " *) + + - name: Sign each manifest + env: + REPOSITORY: ${{ matrix.repository }} + run: | + DIGESTS="" + for TAG in $(echo "$DOCKER_METADATA_OUTPUT_JSON" | jq -r '.tags[]'); do + DIGEST="$(docker buildx imagetools inspect $TAG --format '{{json .Manifest}}' | jq -r '.digest')" + DIGESTS="$DIGESTS $REPOSITORY@$DIGEST" done - cosign sign --yes ${images} + cosign sign --yes $DIGESTS diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml index 6d184a21e0..53a2d6b597 100644 --- a/.github/workflows/docs-pr-netlify.yaml +++ b/.github/workflows/docs-pr-netlify.yaml @@ -14,7 +14,7 @@ jobs: # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: - name: 📥 Download artifact - uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6 + uses: dawidd6/action-download-artifact@ac66b43f0e6a346234dd65d4d0c8fbb31cb316e5 # v11 with: workflow: docs-pr.yaml run_id: ${{ github.event.workflow_run.id }} @@ -22,7 +22,7 @@ jobs: path: book - name: 📤 Deploy to Netlify - uses: matrix-org/netlify-pr-preview@v3 + uses: matrix-org/netlify-pr-preview@9805cd123fc9a7e421e35340a05e1ebc5dee46b5 # v3 with: path: book owner: ${{ github.event.workflow_run.head_repository.owner.login }} diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 07dc301b1a..a0af38a6c5 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -13,7 +13,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -24,7 +24,7 @@ jobs: mdbook-version: '0.4.17' - name: Setup python - uses: actions/setup-python@v5 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" @@ -39,7 +39,7 @@ jobs: cp book/welcome_and_overview.html book/index.html - name: Upload Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: book path: book @@ -50,7 +50,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 434dcbb6c7..f260a4f804 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,7 @@ jobs: needs: - pre steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -64,7 +64,7 @@ jobs: run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - name: Setup python - uses: actions/setup-python@v5 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" @@ -78,6 +78,18 @@ jobs: mdbook build cp book/welcome_and_overview.html book/index.html + - name: Prepare and publish schema files + run: | + sudo apt-get update && sudo apt-get install -y yq + mkdir -p book/schema + # Remove developer notice before publishing. + rm schema/v*/Do\ not\ edit\ files\ in\ this\ folder + # Copy schema files that are independent from current Synapse version. + cp -r -t book/schema schema/v*/ + # Convert config schema from YAML source file to JSON. + yq < schema/synapse-config.schema.yaml \ + > book/schema/synapse-config.schema.json + # Deploy to the target directory. - name: Deploy to gh pages uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 909b0a847f..d73df1e2e7 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -6,6 +6,11 @@ name: Attempt to automatically fix linting errors on: workflow_dispatch: +env: + # We use nightly so that `fmt` correctly groups together imports, and + # clippy correctly fixes up the benchmarks. + RUST_VERSION: nightly-2025-06-24 + jobs: fixup: name: Fix up @@ -13,21 +18,20 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: - # We use nightly so that `fmt` correctly groups together imports, and - # clippy correctly fixes up the benchmarks. - toolchain: nightly-2022-12-01 - components: rustfmt - - uses: Swatinem/rust-cache@v2 + toolchain: ${{ env.RUST_VERSION }} + components: clippy, rustfmt + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Setup Poetry - uses: matrix-org/setup-python-poetry@v1 + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: install-project: "false" + poetry-version: "2.1.1" - name: Run ruff check continue-on-error: true @@ -43,6 +47,6 @@ jobs: - run: cargo fmt continue-on-error: true - - uses: stefanzweifel/git-auto-commit-action@v5 + - uses: stefanzweifel/git-auto-commit-action@778341af668090896ca464160c2def5d1d1a3eb0 # v6.0.1 with: commit_message: "Attempt to fix linting" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 3884b6d402..c1c3d8199c 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -21,6 +21,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: 1.87.0 + jobs: check_repo: # Prevent this workflow from running on any fork of Synapse other than element-hq/synapse, as it is @@ -39,23 +42,25 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: "all" # Dump installed versions for debugging. - run: poetry run pip list > before.txt # Upgrade all runtime dependencies only. This is intended to mimic a fresh # `pip install matrix-synapse[all]` as closely as possible. - - run: poetry update --no-dev + - run: poetry update --without dev - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) - name: Remove unhelpful options from mypy config run: sed -e '/warn_unused_ignores = True/d' -e '/warn_redundant_casts = True/d' -i mypy.ini @@ -72,11 +77,13 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.postgres-version }} @@ -86,7 +93,7 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@v5 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" - run: pip install .[all,test] @@ -145,11 +152,13 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Ensure sytest runs `pip install` # Delete the lockfile so sytest will `pip install` rather than `poetry install` @@ -164,7 +173,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) @@ -192,15 +201,15 @@ jobs: database: Postgres steps: - - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@v4 + - name: Check out synapse codebase + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: path: synapse - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@v5 + - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod @@ -225,7 +234,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 496e536b93..19468c2d92 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -16,8 +16,8 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: '3.x' - run: pip install tomli diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 6fbd2ed015..bfafe5642a 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,29 +33,29 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 if: github.event_name == 'push' with: ref: master - name: Login to registry - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Work out labels for complement image id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 with: images: ghcr.io/${{ github.repository }}/complement-synapse tags: | diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index c0aff79141..1217171b5a 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,10 +27,10 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - python-version: '3.x' + python-version: "3.x" - id: set-distros run: | # if we're running from a tag, get the full list of distros; otherwise just use debian:sid @@ -55,18 +55,18 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: path: src - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 with: install: true - name: Set up docker layer caching - uses: actions/cache@v4 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} @@ -74,9 +74,9 @@ jobs: ${{ runner.os }}-buildx- - name: Set up python - uses: actions/setup-python@v5 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - python-version: '3.x' + python-version: "3.x" - name: Build the packages # see https://github.com/docker/build-push-action/issues/252 @@ -101,18 +101,21 @@ jobs: echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT" - name: Upload debs as artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: debs/* build-wheels: - name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }} + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, macos-13] - arch: [x86_64, aarch64] + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + - macos-13 # This uses x86-64 + - macos-14 # This uses arm64 # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. is_pr: @@ -122,38 +125,27 @@ jobs: # Don't build macos wheels on PR CI. - is_pr: true os: "macos-13" - # Don't build aarch64 wheels on mac. - - os: "macos-13" - arch: aarch64 + - is_pr: true + os: "macos-14" # Don't build aarch64 wheels on PR CI. - is_pr: true - arch: aarch64 + os: "ubuntu-24.04-arm" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: # setup-python@v4 doesn't impose a default python version. Need to use 3.x # here, because `python` on osx points to Python 2.7. python-version: "3.x" - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.19.1 - - - name: Set up QEMU to emulate aarch64 - if: matrix.arch == 'aarch64' - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - - name: Build aarch64 wheels - if: matrix.arch == 'aarch64' - run: echo 'CIBW_ARCHS_LINUX=aarch64' >> $GITHUB_ENV + run: python -m pip install cibuildwheel==3.0.0 - name: Only build a single wheel on PR if: startsWith(github.ref, 'refs/pull/') - run: echo "CIBW_BUILD="cp39-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV + run: echo "CIBW_BUILD="cp39-manylinux_*"" >> $GITHUB_ENV - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse @@ -161,13 +153,10 @@ jobs: # Skip testing for platforms which various libraries don't have wheels # for, and so need extra build deps. CIBW_TEST_SKIP: pp3*-* *i686* *musl* - # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583 - CARGO_NET_GIT_FETCH_WITH_CLI: true - CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: Wheel-${{ matrix.os }}-${{ matrix.arch }} + name: Wheel-${{ matrix.os }} path: ./wheelhouse/*.whl build-sdist: @@ -176,22 +165,21 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - python-version: '3.10' + python-version: "3.10" - run: pip install build - name: Build sdist run: python -m build --sdist - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: Sdist path: dist/*.tar.gz - # if it's a tag, create a release and attach the artifacts to it attach-assets: name: "Attach assets to release" @@ -203,7 +191,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download all workflow run artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 - name: Build a tarball for the debs # We need to merge all the debs uploads into one folder, then compress # that. @@ -212,7 +200,8 @@ jobs: mv debs*/* debs/ tar -cvJf debs.tar.xz debs - name: Attach to release - uses: softprops/action-gh-release@v2 + # Pinned to work around https://github.com/softprops/action-gh-release/issues/445 + uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda # v0.1.15 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -220,3 +209,7 @@ jobs: Sdist/* Wheel*/* debs.tar.xz + # if it's not already published, keep the release as a draft. + draft: true + # mark it as a prerelease if the tag contains 'rc'. + prerelease: ${{ contains(github.ref, 'rc') }} diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml new file mode 100644 index 0000000000..6c416e762d --- /dev/null +++ b/.github/workflows/schema.yaml @@ -0,0 +1,57 @@ +name: Schema + +on: + pull_request: + paths: + - schema/** + - docs/usage/configuration/config_documentation.md + push: + branches: ["develop", "release-*"] + workflow_dispatch: + +jobs: + validate-schema: + name: Ensure Synapse config schema is valid + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: "3.x" + - name: Install check-jsonschema + run: pip install check-jsonschema==0.33.0 + + - name: Validate meta schema + run: check-jsonschema --check-metaschema schema/v*/meta.schema.json + - name: Validate schema + run: |- + # Please bump on introduction of a new meta schema. + LATEST_META_SCHEMA_VERSION=v1 + check-jsonschema \ + --schemafile="schema/$LATEST_META_SCHEMA_VERSION/meta.schema.json" \ + schema/synapse-config.schema.yaml + - name: Validate default config + # Populates the empty instance with default values and checks against the schema. + run: |- + echo "{}" | check-jsonschema \ + --fill-defaults --schemafile=schema/synapse-config.schema.yaml - + + check-doc-generation: + name: Ensure generated documentation is up-to-date + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: "3.x" + - name: Install PyYAML + run: pip install PyYAML==6.0.2 + + - name: Regenerate config documentation + run: | + scripts-dev/gen_config_documentation.py \ + schema/synapse-config.schema.yaml \ + > docs/usage/configuration/config_documentation.md + - name: Error in case of any differences + # Errors if there are now any modified files (untracked files are ignored). + run: 'git diff --exit-code' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d91f9c2918..216c7da6be 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: 1.87.0 + jobs: # Job to detect what has changed so we don't run e.g. Rust checks on PRs that # don't modify Rust code. @@ -23,7 +26,7 @@ jobs: linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }} linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }} steps: - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter # We only check on PRs if: startsWith(github.ref, 'refs/pull/') @@ -83,14 +86,16 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 - - uses: matrix-org/setup-python-poetry@v1 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: "all" - run: poetry run scripts-dev/generate_sample_config.sh --check - run: poetry run scripts-dev/config-lint.sh @@ -101,8 +106,8 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" @@ -111,8 +116,8 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" - run: .ci/scripts/check_lockfile.py @@ -124,11 +129,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Setup Poetry - uses: matrix-org/setup-python-poetry@v1 + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: + poetry-version: "2.1.1" install-project: "false" - name: Run ruff check @@ -145,14 +151,16 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Setup Poetry - uses: matrix-org/setup-python-poetry@v1 + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: # We want to make use of type hints in optional dependencies too. extras: all @@ -161,11 +169,12 @@ jobs: # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775 # To make CI green, err towards caution and install the project. install-project: "true" + poetry-version: "2.1.1" # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@v4 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: | .mypy_cache @@ -178,7 +187,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -186,11 +195,11 @@ jobs: if: ${{ (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" - run: "pip install 'towncrier>=18.6.0rc1'" @@ -204,15 +213,17 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 - - uses: matrix-org/setup-python-poetry@v1 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: - poetry-version: "1.3.2" + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + poetry-version: "2.1.1" extras: "all" - run: poetry run scripts-dev/check_pydantic_models.py @@ -222,13 +233,14 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: components: clippy - - uses: Swatinem/rust-cache@v2 + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo clippy -- -D warnings @@ -240,32 +252,70 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: - toolchain: nightly-2022-12-01 + toolchain: nightly-2025-04-23 components: clippy - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo clippy --all-features -- -D warnings + lint-rust: + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Install Rust + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + + - name: Setup Poetry + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + # Install like a normal project from source with all optional dependencies + extras: all + install-project: "true" + poetry-version: "2.1.1" + + - name: Ensure `Cargo.lock` is up to date (no stray changes after install) + # The `::error::` syntax is using GitHub Actions' error annotations, see + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions + run: | + if git diff --quiet Cargo.lock; then + echo "Cargo.lock is up to date" + else + echo "::error::Cargo.lock has uncommitted changes after install. Please run 'poetry install --extras all' and commit the Cargo.lock changes." + git diff --exit-code Cargo.lock + exit 1 + fi + + # This job is split from `lint-rust` because it requires a nightly Rust toolchain + # for some of the unstable options we use in `.rustfmt.toml`. lint-rustfmt: runs-on: ubuntu-latest needs: changes if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: - # We use nightly so that it correctly groups together imports - toolchain: nightly-2022-12-01 + # We use nightly so that we can use some unstable options that we use in + # `.rustfmt.toml`. + toolchain: nightly-2025-04-23 components: rustfmt - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo fmt --check @@ -276,8 +326,8 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" - run: "pip install rstcheck" @@ -297,11 +347,12 @@ jobs: - check-lockfile - lint-clippy - lint-clippy-nightly + - lint-rust - lint-rustfmt - lint-readme runs-on: ubuntu-latest steps: - - uses: matrix-org/done-action@v3 + - uses: matrix-org/done-action@3409aa904e8a2aaf2220f09bc954d3d0b0a2ee67 # v3 with: needs: ${{ toJSON(needs) }} @@ -315,6 +366,7 @@ jobs: lint-pydantic lint-clippy lint-clippy-nightly + lint-rust lint-rustfmt lint-readme @@ -324,8 +376,8 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" - id: get-matrix @@ -345,7 +397,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -360,13 +412,15 @@ jobs: postgres:${{ matrix.job.postgres-version }} - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.job.python-version }} - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: ${{ matrix.job.extras }} - name: Await PostgreSQL if: ${{ matrix.job.postgres-version }} @@ -399,11 +453,13 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 # There aren't wheels for some of the older deps, so we need to install # their build dependencies @@ -412,7 +468,7 @@ jobs: sudo apt-get -qq install build-essential libffi-dev python3-dev \ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - uses: actions/setup-python@v5 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: '3.9' @@ -462,13 +518,13 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.python-version }} - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: ${{ matrix.extras }} - run: poetry run trial --jobs=2 tests - name: Dump logs @@ -512,13 +568,15 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Run SyTest run: /bootstrap.sh synapse @@ -527,7 +585,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) @@ -557,11 +615,11 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: "postgres" - run: .ci/scripts/test_export_data_command.sh env: @@ -581,7 +639,7 @@ jobs: matrix: include: - python-version: "3.9" - postgres-version: "11" + postgres-version: "13" - python-version: "3.13" postgres-version: "17" @@ -601,7 +659,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -612,10 +670,10 @@ jobs: wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.python-version }} - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: "postgres" - run: .ci/scripts/test_synapse_port_db.sh id: run_tester_script @@ -625,7 +683,7 @@ jobs: PGPASSWORD: postgres PGDATABASE: postgres - name: "Upload schema differences" - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} with: name: Schema dumps @@ -655,19 +713,21 @@ jobs: database: Postgres steps: - - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@v4 + - name: Checkout synapse codebase + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: path: synapse - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@v5 + - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod @@ -690,11 +750,13 @@ jobs: - changes steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo test @@ -708,13 +770,13 @@ jobs: - changes steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: nightly-2022-12-01 - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo bench --no-run @@ -733,7 +795,7 @@ jobs: - linting-done runs-on: ubuntu-latest steps: - - uses: matrix-org/done-action@v3 + - uses: matrix-org/done-action@3409aa904e8a2aaf2220f09bc954d3d0b0a2ee67 # v3 with: needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/triage-incoming.yml b/.github/workflows/triage-incoming.yml index 7a369b77fe..1d291a319b 100644 --- a/.github/workflows/triage-incoming.yml +++ b/.github/workflows/triage-incoming.yml @@ -6,7 +6,7 @@ on: jobs: triage: - uses: matrix-org/backend-meta/.github/workflows/triage-incoming.yml@v2 + uses: matrix-org/backend-meta/.github/workflows/triage-incoming.yml@18beaf3c8e536108bd04d18e6c3dc40ba3931e28 # v2.0.3 with: project_id: 'PVT_kwDOAIB0Bs4AFDdZ' content_id: ${{ github.event.issue.node_id }} diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index d1ac4357b1..41f535a15d 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -11,11 +11,15 @@ jobs: if: > contains(github.event.issue.labels.*.name, 'X-Needs-Info') steps: - - uses: actions/add-to-project@main + - uses: actions/add-to-project@4515659e2b458b27365e167605ac44f219494b66 # v1.0.2 id: add_project with: project-url: "https://github.com/orgs/matrix-org/projects/67" github-token: ${{ secrets.ELEMENT_BOT_TOKEN }} + # This action will error if the issue already exists on the project. Which is + # common as `X-Needs-Info` will often be added to issues that are already in + # the triage queue. Prevent the whole job from failing in this case. + continue-on-error: true - name: Set status env: GITHUB_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }} diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index cdaa00ef90..edb3c44090 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -20,6 +20,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: 1.87.0 + jobs: check_repo: # Prevent this workflow from running on any fork of Synapse other than element-hq/synapse, as it is @@ -40,16 +43,19 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" extras: "all" + poetry-version: "2.1.1" - run: | poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref || 'trunk' }} @@ -64,17 +70,20 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" extras: "all test" + poetry-version: "2.1.1" - run: | poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk @@ -108,11 +117,13 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Patch dependencies # Note: The poetry commands want to create a virtualenv in /src/.venv/, @@ -136,7 +147,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) @@ -164,14 +175,14 @@ jobs: steps: - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: path: synapse - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@v5 + - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod @@ -181,11 +192,11 @@ jobs: run: | set -x DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx - pipx install poetry==1.3.2 + pipx install poetry==2.1.1 poetry remove -n twisted poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk - poetry lock --no-update + poetry lock working-directory: synapse - run: | @@ -206,7 +217,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index a89f149ec1..e333f2320b 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ __pycache__/ /.idea/ /.ropeproject/ /.vscode/ +/.zed/ # build products !/.coveragerc diff --git a/.rustfmt.toml b/.rustfmt.toml index bf96e7743d..29e67033cc 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1 +1,6 @@ +# Unstable options are only available on a nightly toolchain and must be opted into +unstable_features = true + +# `group_imports` is an unstable option that requires nightly Rust toolchain. Tracked by +# https://github.com/rust-lang/rustfmt/issues/5083 group_imports = "StdExternalCrate" diff --git a/CHANGES.md b/CHANGES.md index 31d9914b39..91fe74c60f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3671 +1,1033 @@ -# Synapse 1.120.0rc1 (2024-11-20) +# Synapse 1.138.0 (2025-09-09) -This release enables the enforcement of authenticated media by default, with exemptions for media that is already present in the -homeserver's media store. +No significant changes since 1.138.0rc1. -Most homeservers operating in the public federation will not be impacted by this change, given that -the large homeserver `matrix.org` enabled this in September 2024 and therefore most clients and servers -will already have updated as a result. -Some server administrators may still wish to disable this enforcement for the time being, in the interest of compatibility with older clients -and older federated homeservers. -See the [upgrade notes](https://element-hq.github.io/synapse/v1.120/upgrade.html#authenticated-media-is-now-enforced-by-default) for more information. + + +# Synapse 1.138.0rc1 (2025-09-02) ### Features -- Enforce authenticated media by default. Administrators can revert this by configuring `enable_authenticated_media` to `false`. In a future release of Synapse, this option will be removed and become always-on. ([\#17889](https://github.com/element-hq/synapse/issues/17889)) -- Add a one-off task to delete old One-Time Keys, to guard against us having old OTKs in the database that the client has long forgotten about. ([\#17934](https://github.com/element-hq/synapse/issues/17934)) +- Support for the stable endpoint and scopes of [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) & co. ([\#18549](https://github.com/element-hq/synapse/issues/18549)) + +### Bugfixes + +- Improve database performance of [MSC4293](https://github.com/matrix-org/matrix-spec-proposals/pull/4293) - Redact on Kick/Ban. ([\#18851](https://github.com/element-hq/synapse/issues/18851)) +- Do not throw an error when fetching a rejected delayed state event on startup. ([\#18858](https://github.com/element-hq/synapse/issues/18858)) ### Improved Documentation -- Clarify the semantics of the `enable_authenticated_media` configuration option. ([\#17913](https://github.com/element-hq/synapse/issues/17913)) -- Add documentation about backing up Synapse. ([\#17931](https://github.com/element-hq/synapse/issues/17931)) +- Fix worker documentation incorrectly indicating all room Admin API requests were capable of being handled by workers. ([\#18853](https://github.com/element-hq/synapse/issues/18853)) + +### Internal Changes + +- Instrument `_ByteProducer` with tracing to measure potential dead time while writing bytes to the request. ([\#18804](https://github.com/element-hq/synapse/issues/18804)) +- Switch to OpenTracing's `ContextVarsScopeManager` instead of our own custom `LogContextScopeManager`. ([\#18849](https://github.com/element-hq/synapse/issues/18849)) +- Trace how much work is being done while "recursively fetching redactions". ([\#18854](https://github.com/element-hq/synapse/issues/18854)) +- Link [upstream Twisted bug](https://github.com/twisted/twisted/issues/12498) tracking the problem that explains why we have to use a `Producer` to write bytes to the request. ([\#18855](https://github.com/element-hq/synapse/issues/18855)) +- Introduce `EventPersistencePair` type. ([\#18857](https://github.com/element-hq/synapse/issues/18857)) + + + +### Updates to locked dependencies + +* Bump actions/add-to-project from c0c5949b017d0d4a39f7ba888255881bdac2a823 to 4515659e2b458b27365e167605ac44f219494b66. ([\#18863](https://github.com/element-hq/synapse/issues/18863)) +* Bump actions/checkout from 4.3.0 to 5.0.0. ([\#18834](https://github.com/element-hq/synapse/issues/18834)) +* Bump anyhow from 1.0.98 to 1.0.99. ([\#18841](https://github.com/element-hq/synapse/issues/18841)) +* Bump docker/login-action from 3.4.0 to 3.5.0. ([\#18835](https://github.com/element-hq/synapse/issues/18835)) +* Bump dtolnay/rust-toolchain from b3b07ba8b418998c39fb20f53e8b695cdcc8de1b to e97e2d8cc328f1b50210efc529dca0028893a2d9. ([\#18862](https://github.com/element-hq/synapse/issues/18862)) +* Bump phonenumbers from 9.0.11 to 9.0.12. ([\#18837](https://github.com/element-hq/synapse/issues/18837)) +* Bump regex from 1.11.1 to 1.11.2. ([\#18864](https://github.com/element-hq/synapse/issues/18864)) +* Bump reqwest from 0.12.22 to 0.12.23. ([\#18842](https://github.com/element-hq/synapse/issues/18842)) +* Bump ruff from 0.12.7 to 0.12.10. ([\#18865](https://github.com/element-hq/synapse/issues/18865)) +* Bump serde_json from 1.0.142 to 1.0.143. ([\#18866](https://github.com/element-hq/synapse/issues/18866)) +* Bump types-bleach from 6.2.0.20250514 to 6.2.0.20250809. ([\#18838](https://github.com/element-hq/synapse/issues/18838)) +* Bump types-jsonschema from 4.25.0.20250720 to 4.25.1.20250822. ([\#18867](https://github.com/element-hq/synapse/issues/18867)) +* Bump types-psycopg2 from 2.9.21.20250718 to 2.9.21.20250809. ([\#18836](https://github.com/element-hq/synapse/issues/18836)) + +# Synapse 1.137.0 (2025-08-26) + +No significant changes since 1.137.0rc1. + + + + +# Synapse 1.137.0rc1 (2025-08-19) + +### Bugfixes + +- Fix a bug which could corrupt auth chains making it impossible to perform state resolution. ([\#18746](https://github.com/element-hq/synapse/issues/18746)) +- Fix error message in `register_new_matrix_user` utility script for empty `registration_shared_secret`. ([\#18780](https://github.com/element-hq/synapse/issues/18780)) +- Allow enabling [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108) when the stable Matrix Authentication Service integration is enabled. ([\#18832](https://github.com/element-hq/synapse/issues/18832)) + +### Improved Documentation + +- Include IPv6 networks in `denied-peer-ips` of coturn setup. Contributed by @litetex. ([\#18781](https://github.com/element-hq/synapse/issues/18781)) + +### Internal Changes + +- Update tests to ensure all database tables are emptied when purging a room. ([\#18794](https://github.com/element-hq/synapse/issues/18794)) +- Instrument the `encode_response` part of Sliding Sync requests for more complete traces in Jaeger. ([\#18815](https://github.com/element-hq/synapse/issues/18815)) +- Tag Sliding Sync traces when we `wait_for_events`. ([\#18816](https://github.com/element-hq/synapse/issues/18816)) +- Fix `portdb` CI by hardcoding the new `pg_dump` restrict key that was added due to [CVE-2025-8714](https://nvd.nist.gov/vuln/detail/cve-2025-8714). ([\#18824](https://github.com/element-hq/synapse/issues/18824)) + + + +### Updates to locked dependencies + +* Bump actions/add-to-project from 5b1a254a3546aef88e0a7724a77a623fa2e47c36 to 0c37450c4be3b6a7582b2fb013c9ebfd9c8e9300. ([\#18557](https://github.com/element-hq/synapse/issues/18557)) +* Bump actions/cache from 4.2.3 to 4.2.4. ([\#18799](https://github.com/element-hq/synapse/issues/18799)) +* Bump actions/checkout from 4.2.2 to 4.3.0. ([\#18800](https://github.com/element-hq/synapse/issues/18800)) +* Bump actions/download-artifact from 4.3.0 to 5.0.0. ([\#18801](https://github.com/element-hq/synapse/issues/18801)) +* Bump docker/metadata-action from 5.7.0 to 5.8.0. ([\#18773](https://github.com/element-hq/synapse/issues/18773)) +* Bump mypy from 1.16.1 to 1.17.1. ([\#18775](https://github.com/element-hq/synapse/issues/18775)) +* Bump phonenumbers from 9.0.10 to 9.0.11. ([\#18797](https://github.com/element-hq/synapse/issues/18797)) +* Bump pygithub from 2.6.1 to 2.7.0. ([\#18779](https://github.com/element-hq/synapse/issues/18779)) +* Bump serde_json from 1.0.141 to 1.0.142. ([\#18776](https://github.com/element-hq/synapse/issues/18776)) +* Bump slab from 0.4.10 to 0.4.11. ([\#18809](https://github.com/element-hq/synapse/issues/18809)) +* Bump tokio from 1.47.0 to 1.47.1. ([\#18774](https://github.com/element-hq/synapse/issues/18774)) +* Bump types-pyyaml from 6.0.12.20250516 to 6.0.12.20250809. ([\#18798](https://github.com/element-hq/synapse/issues/18798)) +* Bump types-setuptools from 80.9.0.20250529 to 80.9.0.20250809. ([\#18796](https://github.com/element-hq/synapse/issues/18796)) + +# Synapse 1.136.0 (2025-08-12) + +Note: This release includes the security fixes from `1.135.2` and `1.136.0rc2`, detailed below. + +### Bugfixes + +- Fix bug introduced in 1.135.2 and 1.136.0rc2 where the [Make Room Admin API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#make-room-admin-api) would not treat a room v12's creator power level as the highest in room. ([\#18805](https://github.com/element-hq/synapse/issues/18805)) + + +# Synapse 1.135.2 (2025-08-11) + +This is the Synapse portion of the [Matrix coordinated security release](https://matrix.org/blog/2025/07/security-predisclosure/). This release includes support for [room version](https://spec.matrix.org/v1.15/rooms/) 12 which fixes a number of security vulnerabilities, including [CVE-2025-49090](https://www.cve.org/CVERecord?id=CVE-2025-49090). + +The default room version is not changed. Not all clients will support room version 12 immediately, and not all users will be using the latest version of their clients. Large, public rooms are advised to wait a few weeks before upgrading to room version 12 to allow users throughout the Matrix ecosystem to update their clients. + +Note: release 1.135.1 was skipped due to issues discovered during the release process. + +Two patched Synapse releases are now available: + +* `1.135.2`: stable release comprised of `1.135.0` + security patches + * Upgrade to this release **if you are currently running 1.135.0 or below**. +* `1.136.0rc2`: unstable release candidate comprised of `1.136.0rc1` + security patches. + * Upgrade to this release **only if you are on 1.136.0rc1**. + +### Bugfixes + +- Fix invalidation of storage cache that was broken in 1.135.0. ([\#18786](https://github.com/element-hq/synapse/issues/18786)) + +### Internal Changes + +- Add a parameter to `upgrade_rooms(..)` to allow auto join local users. ([\#82](https://github.com/element-hq/synapse/issues/82)) +- Speed up upgrading a room with large numbers of banned users. ([\#18574](https://github.com/element-hq/synapse/issues/18574)) + + +# Synapse 1.136.0rc2 (2025-08-11) + +- Update MSC4293 redaction logic for room v12. ([\#80](https://github.com/element-hq/synapse/issues/80)) + +### Internal Changes + +- Add a parameter to `upgrade_rooms(..)` to allow auto join local users. ([\#83](https://github.com/element-hq/synapse/issues/83)) + + +# Synapse 1.136.0rc1 (2025-08-05) + +Please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11360) as this release contains changes to MAS support, metrics labels and the module API which may require your attention when upgrading. + +### Features + +- Add configurable rate limiting for the creation of rooms. ([\#18514](https://github.com/element-hq/synapse/issues/18514)) +- Add support for [MSC4293](https://github.com/matrix-org/matrix-spec-proposals/pull/4293) - Redact on Kick/Ban. ([\#18540](https://github.com/element-hq/synapse/issues/18540)) +- When admins enable themselves to see soft-failed events, they will also see if the cause is due to the policy server flagging them as spam via `unsigned`. ([\#18585](https://github.com/element-hq/synapse/issues/18585)) +- Add ability to configure forward/outbound proxy via homeserver config instead of environment variables. See `http_proxy`, `https_proxy`, `no_proxy_hosts`. ([\#18686](https://github.com/element-hq/synapse/issues/18686)) +- Advertise experimental support for [MSC4306](https://github.com/matrix-org/matrix-spec-proposals/pull/4306) (Thread Subscriptions) through `/_matrix/clients/versions` if enabled. ([\#18722](https://github.com/element-hq/synapse/issues/18722)) +- Stabilise support for delegating authentication to [Matrix Authentication Service](https://github.com/element-hq/matrix-authentication-service/). ([\#18759](https://github.com/element-hq/synapse/issues/18759)) +- Implement the push rules for experimental [MSC4306: Thread Subscriptions](https://github.com/matrix-org/matrix-doc/issues/4306). ([\#18762](https://github.com/element-hq/synapse/issues/18762)) + +### Bugfixes + +- Allow return code 403 (allowed by C2S Spec since v1.2) when fetching profiles via federation. ([\#18696](https://github.com/element-hq/synapse/issues/18696)) +- Register the MSC4306 (Thread Subscriptions) endpoints in the CS API when the experimental feature is enabled. ([\#18726](https://github.com/element-hq/synapse/issues/18726)) +- Fix a long-standing bug where suspended users could not have server notices sent to them (a 403 was returned to the admin). ([\#18750](https://github.com/element-hq/synapse/issues/18750)) +- Fix an issue that could cause logcontexts to be lost on rate-limited requests. Found by @realtyem. ([\#18763](https://github.com/element-hq/synapse/issues/18763)) +- Fix invalidation of storage cache that was broken in 1.135.0. ([\#18786](https://github.com/element-hq/synapse/issues/18786)) + +### Improved Documentation + +- Minor improvements to README. ([\#18700](https://github.com/element-hq/synapse/issues/18700)) +- Document that there can be multiple workers handling the `receipts` stream. ([\#18760](https://github.com/element-hq/synapse/issues/18760)) +- Improve worker documentation for some device paths. ([\#18761](https://github.com/element-hq/synapse/issues/18761)) ### Deprecations and Removals -- Remove support for [MSC3886: Simple client rendezvous capability](https://github.com/matrix-org/matrix-spec-proposals/pull/3886), which has been superseded by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108) and therefore closed. ([\#17638](https://github.com/element-hq/synapse/issues/17638)) +- Deprecate `run_as_background_process` exported as part of the module API interface in favor of `ModuleApi.run_as_background_process`. See [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11360) for more information. ([\#18737](https://github.com/element-hq/synapse/issues/18737)) ### Internal Changes -- Addressed some typos in docs and returned error message for unknown MXC ID. ([\#17865](https://github.com/element-hq/synapse/issues/17865)) -- Unpin the upload release GHA action. ([\#17923](https://github.com/element-hq/synapse/issues/17923)) -- Bump macOS version used to build wheels during release, as current version used is end-of-life. ([\#17924](https://github.com/element-hq/synapse/issues/17924)) -- Move server event filtering logic to Rust. ([\#17928](https://github.com/element-hq/synapse/issues/17928)) -- Support new package name of PyPI package `python-multipart` 0.0.13 so that distro packagers do not need to work around name conflict with PyPI package `multipart`. ([\#17932](https://github.com/element-hq/synapse/issues/17932)) -- Speed up slow initial sliding syncs on large servers. ([\#17946](https://github.com/element-hq/synapse/issues/17946)) +- Add debug logging for HMAC digest verification failures when using the admin API to register users. ([\#18474](https://github.com/element-hq/synapse/issues/18474)) +- Speed up upgrading a room with large numbers of banned users. ([\#18574](https://github.com/element-hq/synapse/issues/18574)) +- Fix config documentation generation script on Windows by enforcing UTF-8. ([\#18580](https://github.com/element-hq/synapse/issues/18580)) +- Refactor cache, background process, `Counter`, `LaterGauge`, `GaugeBucketCollector`, `Histogram`, and `Gauge` metrics to be homeserver-scoped. ([\#18656](https://github.com/element-hq/synapse/issues/18656), [\#18714](https://github.com/element-hq/synapse/issues/18714), [\#18715](https://github.com/element-hq/synapse/issues/18715), [\#18724](https://github.com/element-hq/synapse/issues/18724), [\#18753](https://github.com/element-hq/synapse/issues/18753), [\#18725](https://github.com/element-hq/synapse/issues/18725), [\#18670](https://github.com/element-hq/synapse/issues/18670), [\#18748](https://github.com/element-hq/synapse/issues/18748), [\#18751](https://github.com/element-hq/synapse/issues/18751)) +- Reduce database usage in Sliding Sync by not querying for background update completion after the update is known to be complete. ([\#18718](https://github.com/element-hq/synapse/issues/18718)) +- Improve order of validation and ratelimiting in room creation. ([\#18723](https://github.com/element-hq/synapse/issues/18723)) +- Bump minimum version bound on Twisted to 21.2.0. ([\#18727](https://github.com/element-hq/synapse/issues/18727), [\#18729](https://github.com/element-hq/synapse/issues/18729)) +- Use `twisted.internet.testing` module in tests instead of deprecated `twisted.test.proto_helpers`. ([\#18728](https://github.com/element-hq/synapse/issues/18728)) +- Remove obsolete `/send_event` replication endpoint. ([\#18730](https://github.com/element-hq/synapse/issues/18730)) +- Update metrics linting to be able to handle custom metrics. ([\#18733](https://github.com/element-hq/synapse/issues/18733)) +- Work around `twisted.protocols.amp.TooLong` error by reducing logging in some tests. ([\#18736](https://github.com/element-hq/synapse/issues/18736)) +- Prevent "Move labelled issues to correct projects" GitHub Actions workflow from failing when an issue is already on the project board. ([\#18755](https://github.com/element-hq/synapse/issues/18755)) +- Bump minimum supported Rust version (MSRV) to 1.82.0. Missed in [#18553](https://github.com/element-hq/synapse/pull/18553) (released in Synapse 1.134.0). ([\#18757](https://github.com/element-hq/synapse/issues/18757)) +- Make `Clock.sleep(...)` return a coroutine, so that mypy can catch places where we don't await on it. ([\#18772](https://github.com/element-hq/synapse/issues/18772)) +- Update implementation of [MSC4306: Thread Subscriptions](https://github.com/matrix-org/matrix-doc/issues/4306) to include automatic subscription conflict prevention as introduced in later drafts. ([\#18756](https://github.com/element-hq/synapse/issues/18756)) + + ### Updates to locked dependencies -* Bump anyhow from 1.0.92 to 1.0.93. ([\#17920](https://github.com/element-hq/synapse/issues/17920)) -* Bump bleach from 6.1.0 to 6.2.0. ([\#17918](https://github.com/element-hq/synapse/issues/17918)) -* Bump immutabledict from 4.2.0 to 4.2.1. ([\#17941](https://github.com/element-hq/synapse/issues/17941)) -* Bump packaging from 24.1 to 24.2. ([\#17940](https://github.com/element-hq/synapse/issues/17940)) -* Bump phonenumbers from 8.13.49 to 8.13.50. ([\#17942](https://github.com/element-hq/synapse/issues/17942)) -* Bump pygithub from 2.4.0 to 2.5.0. ([\#17917](https://github.com/element-hq/synapse/issues/17917)) -* Bump ruff from 0.7.2 to 0.7.3. ([\#17919](https://github.com/element-hq/synapse/issues/17919)) -* Bump serde from 1.0.214 to 1.0.215. ([\#17938](https://github.com/element-hq/synapse/issues/17938)) +* Bump gitpython from 3.1.44 to 3.1.45. ([\#18743](https://github.com/element-hq/synapse/issues/18743)) +* Bump mypy-zope from 1.0.12 to 1.0.13. ([\#18744](https://github.com/element-hq/synapse/issues/18744)) +* Bump phonenumbers from 9.0.9 to 9.0.10. ([\#18741](https://github.com/element-hq/synapse/issues/18741)) +* Bump ruff from 0.12.4 to 0.12.5. ([\#18742](https://github.com/element-hq/synapse/issues/18742)) +* Bump sentry-sdk from 2.32.0 to 2.33.2. ([\#18745](https://github.com/element-hq/synapse/issues/18745)) +* Bump tokio from 1.46.1 to 1.47.0. ([\#18740](https://github.com/element-hq/synapse/issues/18740)) +* Bump types-jsonschema from 4.24.0.20250708 to 4.25.0.20250720. ([\#18703](https://github.com/element-hq/synapse/issues/18703)) +* Bump types-psycopg2 from 2.9.21.20250516 to 2.9.21.20250718. ([\#18706](https://github.com/element-hq/synapse/issues/18706)) -# Synapse 1.119.0 (2024-11-13) +# Synapse 1.135.0 (2025-08-01) -No significant changes since 1.119.0rc2. - -### Python 3.8 support dropped - -Python 3.8 is [end-of-life](https://devguide.python.org/versions/) and is no longer supported by Synapse. The minimum supported Python version is now 3.9. - -If you are running Synapse with Python 3.8, please upgrade to Python 3.9 (or greater) before upgrading Synapse. +No significant changes since 1.135.0rc2. -# Synapse 1.119.0rc2 (2024-11-11) - -Note that due to packaging issues there was no v1.119.0rc1. -### Features - -- Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API. ([\#17374](https://github.com/element-hq/synapse/issues/17374)) -- Add experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2). ([\#17888](https://github.com/element-hq/synapse/issues/17888)) +# Synapse 1.135.0rc2 (2025-07-30) ### Bugfixes -- Fix bug with sliding sync where `$LAZY`-loading room members would not return `required_state` membership in incremental syncs. ([\#17809](https://github.com/element-hq/synapse/issues/17809)) -- Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi. ([\#17839](https://github.com/element-hq/synapse/issues/17839)) -- Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in - the config option `run_background_tasks_on`. ([\#17847](https://github.com/element-hq/synapse/issues/17847)) -- Fix bug where some presence and typing timeouts can expire early. ([\#17850](https://github.com/element-hq/synapse/issues/17850)) -- Fix detection when the built Rust library was outdated when using source installations. ([\#17861](https://github.com/element-hq/synapse/issues/17861)) -- Fix a long-standing bug in Synapse which could cause one-time keys to be issued in the incorrect order, causing message decryption failures. ([\#17903](https://github.com/element-hq/synapse/pull/17903)) -- Fix experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2) where we would return the full state on incremental syncs when using lazy loaded members and there were no new events in the timeline. ([\#17915](https://github.com/element-hq/synapse/pull/17915)) +- Fix user failing to deactivate with MAS when `/_synapse/mas` is handled by a worker. ([\#18716](https://github.com/element-hq/synapse/issues/18716)) ### Internal Changes -- Remove support for python 3.8. ([\#17908](https://github.com/element-hq/synapse/issues/17908)) -- Add a test for downloading and thumbnailing a CMYK JPEG. ([\#17786](https://github.com/element-hq/synapse/issues/17786)) -- Refactor database calls to remove `Generator` usage. ([\#17813](https://github.com/element-hq/synapse/issues/17813), [\#17814](https://github.com/element-hq/synapse/issues/17814), [\#17815](https://github.com/element-hq/synapse/issues/17815), [\#17816](https://github.com/element-hq/synapse/issues/17816), [\#17817](https://github.com/element-hq/synapse/issues/17817), [\#17818](https://github.com/element-hq/synapse/issues/17818), [\#17890](https://github.com/element-hq/synapse/issues/17890)) -- Include the destination in the error of 'Destination mismatch' on federation requests. ([\#17830](https://github.com/element-hq/synapse/issues/17830)) -- The nix flake inside the repository no longer tracks nixpkgs/master to not catch the latest bugs from a PR merged 5 minutes ago. ([\#17852](https://github.com/element-hq/synapse/issues/17852)) -- Minor speed-up of sliding sync by computing extensions results in parallel. ([\#17884](https://github.com/element-hq/synapse/issues/17884)) -- Bump the default Python version in the Synapse Dockerfile from 3.11 -> 3.12. ([\#17887](https://github.com/element-hq/synapse/issues/17887)) -- Remove usage of internal header encoding API. ([\#17894](https://github.com/element-hq/synapse/issues/17894)) -- Use unique name for each os.arch variant when uploading Wheel artifacts. ([\#17905](https://github.com/element-hq/synapse/issues/17905)) -- Fix tests to run with latest Twisted. ([\#17906](https://github.com/element-hq/synapse/pull/17906), [\#17907](https://github.com/element-hq/synapse/pull/17907), [\#17911](https://github.com/element-hq/synapse/pull/17911)) -- Update version constraint to allow the latest poetry-core 1.9.1. ([\#17902](https://github.com/element-hq/synapse/pull/17902)) -- Update the portdb CI to use Python 3.13 and Postgres 17 as latest dependencies. ([\#17909](https://github.com/element-hq/synapse/pull/17909)) -- Add an index to `current_state_delta_stream` table. ([\#17912](https://github.com/element-hq/synapse/issues/17912)) -- Fix building and attaching release artifacts during the release process. ([\#17921](https://github.com/element-hq/synapse/issues/17921)) - -### Updates to locked dependencies - -* Bump actions/download-artifact & actions/upload-artifact from 3 to 4 in /.github/workflows. ([\#17657](https://github.com/element-hq/synapse/issues/17657)) -* Bump anyhow from 1.0.89 to 1.0.92. ([\#17858](https://github.com/element-hq/synapse/issues/17858), [\#17876](https://github.com/element-hq/synapse/issues/17876), [\#17901](https://github.com/element-hq/synapse/issues/17901)) -* Bump bytes from 1.7.2 to 1.8.0. ([\#17877](https://github.com/element-hq/synapse/issues/17877)) -* Bump cryptography from 43.0.1 to 43.0.3. ([\#17853](https://github.com/element-hq/synapse/issues/17853)) -* Bump mypy-zope from 1.0.7 to 1.0.8. ([\#17898](https://github.com/element-hq/synapse/issues/17898)) -* Bump phonenumbers from 8.13.47 to 8.13.49. ([\#17880](https://github.com/element-hq/synapse/issues/17880), [\#17899](https://github.com/element-hq/synapse/issues/17899)) -* Bump python-multipart from 0.0.12 to 0.0.16. ([\#17879](https://github.com/element-hq/synapse/issues/17879)) -* Bump regex from 1.11.0 to 1.11.1. ([\#17874](https://github.com/element-hq/synapse/issues/17874)) -* Bump ruff from 0.6.9 to 0.7.2. ([\#17868](https://github.com/element-hq/synapse/issues/17868), [\#17897](https://github.com/element-hq/synapse/issues/17897)) -* Bump serde from 1.0.210 to 1.0.214. ([\#17875](https://github.com/element-hq/synapse/issues/17875), [\#17900](https://github.com/element-hq/synapse/issues/17900)) -* Bump serde_json from 1.0.128 to 1.0.132. ([\#17857](https://github.com/element-hq/synapse/issues/17857)) -* Bump types-psycopg2 from 2.9.21.20240819 to 2.9.21.20241019. ([\#17855](https://github.com/element-hq/synapse/issues/17855)) -* Bump types-setuptools from 75.1.0.20241014 to 75.2.0.20241019. ([\#17856](https://github.com/element-hq/synapse/issues/17856)) - -# Synapse 1.118.0 (2024-10-29) - -No significant changes since 1.118.0rc1. - -### Python 3.8 support will be dropped in the next release - -Python 3.8 is now [end-of-life](https://devguide.python.org/versions/). As per our [Deprecation Policy for Platform Dependencies](https://element-hq.github.io/synapse/latest/deprecation_policy.html#policy), Synapse will be dropping support for Python 3.8 in the next release; Synapse 1.119.0. - -Synapse 1.118.x will be the final release to support Python 3.8. If you are running Synapse with Python 3.8, please upgrade before the 1.119.0 release, due in less than one month. - -### Python 3.13 and PostgreSQL 17 support - -On the other end of the spectrum, Synapse 1.118.0 is the first release to support [Python 3.13](https://www.python.org/downloads/release/python-3130/)! [PostgreSQL 17](https://www.postgresql.org/about/news/postgresql-17-released-2936/) is also supported as of this release. +- Fix performance regression introduced in [#18238](https://github.com/element-hq/synapse/issues/18238) by adding a cache to `is_server_admin`. ([\#18747](https://github.com/element-hq/synapse/issues/18747)) -# Synapse 1.118.0rc1 (2024-10-22) + + +# Synapse 1.135.0rc1 (2025-07-22) ### Features -- Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload. ([\#17708](https://github.com/element-hq/synapse/issues/17708)) -- Implement [MSC4210](https://github.com/matrix-org/matrix-spec-proposals/pull/4210): Remove legacy mentions. Contributed by @tulir @ Beeper. ([\#17783](https://github.com/element-hq/synapse/issues/17783)) +- Add `recaptcha_private_key_path` and `recaptcha_public_key_path` config option. ([\#17984](https://github.com/element-hq/synapse/issues/17984), [\#18684](https://github.com/element-hq/synapse/issues/18684)) +- Add plain-text handling for rich-text topics as per [MSC3765](https://github.com/matrix-org/matrix-spec-proposals/pull/3765). ([\#18195](https://github.com/element-hq/synapse/issues/18195)) +- If enabled by the user, server admins will see [soft failed](https://spec.matrix.org/v1.13/server-server-api/#soft-failure) events over the Client-Server API. ([\#18238](https://github.com/element-hq/synapse/issues/18238)) +- Add experimental support for [MSC4277: Harmonizing the reporting endpoints](https://github.com/matrix-org/matrix-spec-proposals/pull/4277). ([\#18263](https://github.com/element-hq/synapse/issues/18263)) +- Add ability to limit amount of media uploaded by a user in a given time period. ([\#18527](https://github.com/element-hq/synapse/issues/18527)) +- Enable workers to write directly to the device lists stream and handle device list updates, reducing load on the main process. ([\#18581](https://github.com/element-hq/synapse/issues/18581)) +- Support arbitrary profile fields. Contributed by @clokep. ([\#18635](https://github.com/element-hq/synapse/issues/18635)) +- Advertise support for Matrix v1.12. ([\#18647](https://github.com/element-hq/synapse/issues/18647)) +- Add an option to issue redactions as an admin user via the [admin redaction endpoint](https://element-hq.github.io/synapse/latest/admin_api/user_admin_api.html#redact-all-the-events-of-a-user). ([\#18671](https://github.com/element-hq/synapse/issues/18671)) +- Add experimental and incomplete support for [MSC4306: Thread Subscriptions](https://github.com/matrix-org/matrix-spec-proposals/blob/rei/msc_thread_subscriptions/proposals/4306-thread-subscriptions.md). ([\#18674](https://github.com/element-hq/synapse/issues/18674)) +- Include `event_id` when getting state with `?format=event`. Contributed by @tulir @ Beeper. ([\#18675](https://github.com/element-hq/synapse/issues/18675)) ### Bugfixes -- Fix saving of PNG thumbnails, when the original image is in the CMYK color space. ([\#17736](https://github.com/element-hq/synapse/issues/17736)) -- Fix bug with sliding sync where the server would not return state that was added to the `required_state` config. ([\#17785](https://github.com/element-hq/synapse/issues/17785), [\#17805](https://github.com/element-hq/synapse/issues/17805)) -- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync that would cause rooms to stay forgotten and hidden even after rejoining. ([\#17835](https://github.com/element-hq/synapse/issues/17835)) +- Fix CPU and database spinning when retrying sending events to servers whilst at the same time purging those events. ([\#18499](https://github.com/element-hq/synapse/issues/18499)) +- Don't allow creation of tags with names longer than 255 bytes, [as per the spec](https://spec.matrix.org/v1.15/client-server-api/#events-14). ([\#18660](https://github.com/element-hq/synapse/issues/18660)) +- Fix `sliding_sync_connections`-related errors when porting from SQLite to Postgres. ([\#18677](https://github.com/element-hq/synapse/issues/18677)) +- Fix the MAS integration not working when Synapse is started with `--daemonize` or using `synctl`. ([\#18691](https://github.com/element-hq/synapse/issues/18691)) ### Improved Documentation -- Clarify when the `user_may_invite` and `user_may_send_3pid_invite` module callbacks are called. ([\#17627](https://github.com/element-hq/synapse/issues/17627)) -- Correct documentation to refer to the `--config-path` argument instead of `--config-file`. ([\#17802](https://github.com/element-hq/synapse/issues/17802)) -- Fix typo in `target_cache_memory_usage` docs. ([\#17825](https://github.com/element-hq/synapse/issues/17825)) +- Document that some config options for the user directory are in violation of the Matrix spec. ([\#18548](https://github.com/element-hq/synapse/issues/18548)) +- Update `rc_delayed_event_mgmt` docs to the actual nesting level. Contributed by @HarHarLinks. ([\#18692](https://github.com/element-hq/synapse/issues/18692)) ### Internal Changes -- Slight optimization when fetching state/events for Sliding Sync. ([\#17718](https://github.com/element-hq/synapse/issues/17718)) -- Add Python 3.13 and Postgres 17 to the test matrix. ([\#17752](https://github.com/element-hq/synapse/issues/17752)) -- Test github token before running release script steps. ([\#17803](https://github.com/element-hq/synapse/issues/17803)) -- Build debian packages for new Ubuntu versions, and stop building for no longer supported versions. ([\#17824](https://github.com/element-hq/synapse/issues/17824)) -- Enable the `.org.matrix.msc4028.encrypted_event` push rule by default in accordance with [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028). Note that the corresponding experimental feature must still be switched on for this push rule to have any effect. ([\#17826](https://github.com/element-hq/synapse/issues/17826)) -- Fix some typing issues uncovered by upgrading mypy to 1.11.x. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) +- Add a dedicated internal API for Matrix Authentication Service to Synapse communication. ([\#18520](https://github.com/element-hq/synapse/issues/18520)) +- Allow user registrations to be done on workers. ([\#18552](https://github.com/element-hq/synapse/issues/18552)) +- Remove unnecessary HTTP replication calls. ([\#18564](https://github.com/element-hq/synapse/issues/18564)) +- Refactor `Measure` block metrics to be homeserver-scoped. ([\#18601](https://github.com/element-hq/synapse/issues/18601)) +- Refactor cache metrics to be homeserver-scoped. ([\#18604](https://github.com/element-hq/synapse/issues/18604)) +- Unbreak "Latest dependencies" workflow by using the `--without dev` poetry option instead of removed `--no-dev`. ([\#18617](https://github.com/element-hq/synapse/issues/18617)) +- Update URL Preview code to work with `lxml` 6.0.0+. ([\#18622](https://github.com/element-hq/synapse/issues/18622)) +- Use `markdown-it-py` instead of `commonmark` in the release script. ([\#18637](https://github.com/element-hq/synapse/issues/18637)) +- Fix typing errors with upgraded mypy version. ([\#18653](https://github.com/element-hq/synapse/issues/18653)) +- Add doc comment explaining that config files are shallowly merged. ([\#18664](https://github.com/element-hq/synapse/issues/18664)) +- Minor speed up of insertion into `stream_positions` table. ([\#18672](https://github.com/element-hq/synapse/issues/18672)) +- Remove unused `allow_no_prev_events` option when creating an event. ([\#18676](https://github.com/element-hq/synapse/issues/18676)) +- Clean up `MetricsResource` and Prometheus hacks. ([\#18687](https://github.com/element-hq/synapse/issues/18687)) +- Fix dirty `Cargo.lock` changes appearing after install (`base64`). ([\#18689](https://github.com/element-hq/synapse/issues/18689)) +- Prevent dirty `Cargo.lock` changes from install. ([\#18693](https://github.com/element-hq/synapse/issues/18693)) +- Correct spelling of 'Admin token used' log line. ([\#18697](https://github.com/element-hq/synapse/issues/18697)) +- Reduce log spam when client stops downloading media while it is being streamed to them. ([\#18699](https://github.com/element-hq/synapse/issues/18699)) ### Updates to locked dependencies -* Bump mypy from 1.10.1 to 1.11.2. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) -* Bump mypy-zope from 1.0.5 to 1.0.7. ([\#17827](https://github.com/element-hq/synapse/issues/17827)) -* Bump phonenumbers from 8.13.46 to 8.13.47. ([\#17797](https://github.com/element-hq/synapse/issues/17797)) -* Bump psycopg2 from 2.9.9 to 2.9.10. ([\#17843](https://github.com/element-hq/synapse/issues/17843)) -* Bump ruff from 0.6.8 to 0.6.9. ([\#17794](https://github.com/element-hq/synapse/issues/17794)) -* Bump sentry-sdk from 2.14.0 to 2.15.0. ([\#17795](https://github.com/element-hq/synapse/issues/17795)) -* Bump sentry-sdk from 2.15.0 to 2.16.0. ([\#17829](https://github.com/element-hq/synapse/issues/17829)) -* Bump sentry-sdk from 2.16.0 to 2.17.0. ([\#17844](https://github.com/element-hq/synapse/issues/17844)) -* Bump sigstore/cosign-installer from 3.6.0 to 3.7.0. ([\#17798](https://github.com/element-hq/synapse/issues/17798)) -* Bump tomli from 2.0.1 to 2.0.2. ([\#17796](https://github.com/element-hq/synapse/issues/17796)) -* Bump types-requests from 2.32.0.20240914 to 2.32.0.20241016. ([\#17841](https://github.com/element-hq/synapse/issues/17841)) -* Bump types-setuptools from 75.1.0.20240917 to 75.1.0.20241014. ([\#17828](https://github.com/element-hq/synapse/issues/17828)) +* Bump authlib from 1.6.0 to 1.6.1. ([\#18704](https://github.com/element-hq/synapse/issues/18704)) +* Bump base64 from 0.21.7 to 0.22.1. ([\#18666](https://github.com/element-hq/synapse/issues/18666)) +* Bump jsonschema from 4.24.0 to 4.25.0. ([\#18707](https://github.com/element-hq/synapse/issues/18707)) +* Bump lxml from 5.4.0 to 6.0.0. ([\#18631](https://github.com/element-hq/synapse/issues/18631)) +* Bump mypy from 1.13.0 to 1.16.1. ([\#18653](https://github.com/element-hq/synapse/issues/18653)) +* Bump once_cell from 1.19.0 to 1.21.3. ([\#18710](https://github.com/element-hq/synapse/issues/18710)) +* Bump phonenumbers from 9.0.8 to 9.0.9. ([\#18681](https://github.com/element-hq/synapse/issues/18681)) +* Bump ruff from 0.12.2 to 0.12.5. ([\#18683](https://github.com/element-hq/synapse/issues/18683), [\#18705](https://github.com/element-hq/synapse/issues/18705)) +* Bump serde_json from 1.0.140 to 1.0.141. ([\#18709](https://github.com/element-hq/synapse/issues/18709)) +* Bump sigstore/cosign-installer from 3.9.1 to 3.9.2. ([\#18708](https://github.com/element-hq/synapse/issues/18708)) +* Bump types-jsonschema from 4.24.0.20250528 to 4.24.0.20250708. ([\#18682](https://github.com/element-hq/synapse/issues/18682)) -# Synapse 1.117.0 (2024-10-15) +# Synapse 1.134.0 (2025-07-15) -No significant changes since 1.117.0rc1. +No significant changes since 1.134.0rc1. -# Synapse 1.117.0rc1 (2024-10-08) +# Synapse 1.134.0rc1 (2025-07-09) ### Features -- Add config option `redis.password_path`. ([\#17717](https://github.com/element-hq/synapse/issues/17717)) +- Support for [MSC4235](https://github.com/matrix-org/matrix-spec-proposals/pull/4235): `via` query param for hierarchy endpoint. Contributed by Krishan (@kfiven). ([\#18070](https://github.com/element-hq/synapse/issues/18070)) +- Add `forget_forced_upon_leave` capability as per [MSC4267](https://github.com/matrix-org/matrix-spec-proposals/pull/4267). ([\#18196](https://github.com/element-hq/synapse/issues/18196)) +- Add `federated_user_may_invite` spam checker callback which receives the entire invite event. Contributed by @tulir @ Beeper. ([\#18241](https://github.com/element-hq/synapse/issues/18241)) ### Bugfixes -- Fix a rare bug introduced in v1.29.0 where invalidating a user's access token from a worker could raise an error. ([\#17779](https://github.com/element-hq/synapse/issues/17779)) -- In the response to `GET /_matrix/client/versions`, set the `unstable_features` flag for [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140) to `false` when server configuration disables support for delayed events. ([\#17780](https://github.com/element-hq/synapse/issues/17780)) -- Improve input validation and room membership checks in admin redaction API. ([\#17792](https://github.com/element-hq/synapse/issues/17792)) +- Fix `KeyError` on background updates when using split main/state databases. ([\#18509](https://github.com/element-hq/synapse/issues/18509)) +- Improve performance of device deletion by adding missing index. ([\#18582](https://github.com/element-hq/synapse/issues/18582)) +- Fix `avatar_url` and `displayname` being sent on federation profile queries when they are not set. ([\#18593](https://github.com/element-hq/synapse/issues/18593)) +- Respond with 401 & `M_USER_LOCKED` when a locked user calls `POST /login`, as per the spec. ([\#18594](https://github.com/element-hq/synapse/issues/18594)) +- Ensure policy servers are not asked to scan policy server change events, allowing rooms to disable the use of a policy server while the policy server is down. ([\#18605](https://github.com/element-hq/synapse/issues/18605)) ### Improved Documentation -- Clarify the docstring of `test_forget_when_not_left`. ([\#17628](https://github.com/element-hq/synapse/issues/17628)) -- Add documentation note about PYTHONMALLOC for accurate jemalloc memory tracking. Contributed by @hensg. ([\#17709](https://github.com/element-hq/synapse/issues/17709)) -- Remove spurious "TODO UPDATE ALL THIS" note in the Debian installation docs. ([\#17749](https://github.com/element-hq/synapse/issues/17749)) -- Explain how load balancing works for `federation_sender_instances`. ([\#17776](https://github.com/element-hq/synapse/issues/17776)) - -### Internal Changes - -- Minor performance increase for large accounts using sliding sync. ([\#17751](https://github.com/element-hq/synapse/issues/17751)) -- Increase performance of the notifier when there are many syncing users. ([\#17765](https://github.com/element-hq/synapse/issues/17765), [\#17766](https://github.com/element-hq/synapse/issues/17766)) -- Fix performance of streams that don't change often. ([\#17767](https://github.com/element-hq/synapse/issues/17767)) -- Improve performance of sliding sync connections that do not ask for any rooms. ([\#17768](https://github.com/element-hq/synapse/issues/17768)) -- Reduce overhead of sliding sync E2EE loops. ([\#17771](https://github.com/element-hq/synapse/issues/17771)) -- Sliding sync minor performance speed up using new table. ([\#17787](https://github.com/element-hq/synapse/issues/17787)) -- Sliding sync minor performance improvement by omitting unchanged data from incremental responses. ([\#17788](https://github.com/element-hq/synapse/issues/17788)) -- Speed up sliding sync when there are many active subscriptions. ([\#17789](https://github.com/element-hq/synapse/issues/17789)) -- Add missing license headers on new source files. ([\#17799](https://github.com/element-hq/synapse/issues/17799)) - - - -### Updates to locked dependencies - -* Bump phonenumbers from 8.13.45 to 8.13.46. ([\#17773](https://github.com/element-hq/synapse/issues/17773)) -* Bump python-multipart from 0.0.10 to 0.0.12. ([\#17772](https://github.com/element-hq/synapse/issues/17772)) -* Bump regex from 1.10.6 to 1.11.0. ([\#17770](https://github.com/element-hq/synapse/issues/17770)) -* Bump ruff from 0.6.7 to 0.6.8. ([\#17774](https://github.com/element-hq/synapse/issues/17774)) - -# Synapse 1.116.0 (2024-10-01) - -No significant changes since 1.116.0rc2. - - - - -# Synapse 1.116.0rc2 (2024-09-26) - -### Features - -- Add implementation of restricting who can overwrite a state event as proposed by [MSC3757](https://github.com/matrix-org/matrix-spec-proposals/pull/3757). ([\#17513](https://github.com/element-hq/synapse/issues/17513)) - - - - -# Synapse 1.116.0rc1 (2024-09-25) - -### Features - -- Add initial implementation of delayed events as proposed by [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). ([\#17326](https://github.com/element-hq/synapse/issues/17326)) -- Add an asynchronous Admin API endpoint [to redact all a user's events](https://element-hq.github.io/synapse/v1.116/admin_api/user_admin_api.html#redact-all-the-events-of-a-user), - and [an endpoint to check on the status of that redaction task](https://element-hq.github.io/synapse/v1.116/admin_api/user_admin_api.html#check-the-status-of-a-redaction-process). ([\#17506](https://github.com/element-hq/synapse/issues/17506)) -- Add support for the `tags` and `not_tags` filters for [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17662](https://github.com/element-hq/synapse/issues/17662)) -- Guests can use the new media endpoints to download media, as described by [MSC4189](https://github.com/matrix-org/matrix-spec-proposals/pull/4189). ([\#17675](https://github.com/element-hq/synapse/issues/17675)) -- Add config option `turn_shared_secret_path`. ([\#17690](https://github.com/element-hq/synapse/issues/17690)) -- Return room tags in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync account data extension. ([\#17707](https://github.com/element-hq/synapse/issues/17707)) - -### Bugfixes - -- Make sure we get up-to-date state information when using the new [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync tables to derive room membership. ([\#17692](https://github.com/element-hq/synapse/issues/17692)) -- Fix bug where room account data would not correctly be sent down [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync for old rooms. ([\#17695](https://github.com/element-hq/synapse/issues/17695)) -- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync which could prevent /sync from working for certain user accounts. ([\#17727](https://github.com/element-hq/synapse/issues/17727), [\#17733](https://github.com/element-hq/synapse/issues/17733)) -- Ignore invites from ignored users in Sliding Sync. ([\#17729](https://github.com/element-hq/synapse/issues/17729)) -- Fix bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync where the server would incorrectly return a negative bump stamp, which caused Element X apps to stop syncing. ([\#17748](https://github.com/element-hq/synapse/issues/17748)) - -### Internal Changes - -- Import pydantic objects from the `_pydantic_compat` module. - This allows `check_pydantic_models.py` to mock those pydantic objects - only in the synapse module, and not interfere with pydantic objects in - external dependencies. ([\#17667](https://github.com/element-hq/synapse/issues/17667)) -- Use [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync tables as a bulk shortcut for getting the max `event_stream_ordering` of rooms. ([\#17693](https://github.com/element-hq/synapse/issues/17693)) -- Speed up [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync requests a bit where there are many room changes. ([\#17696](https://github.com/element-hq/synapse/issues/17696)) -- Refactor [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync filter unit tests so the sliding sync API has better test coverage. ([\#17703](https://github.com/element-hq/synapse/issues/17703)) -- Fetch `bump_stamp`s more efficiently in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17723](https://github.com/element-hq/synapse/issues/17723)) -- Shortcut for checking if certain background updates have completed (utilized in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync). ([\#17724](https://github.com/element-hq/synapse/issues/17724)) -- More efficiently fetch rooms for [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17725](https://github.com/element-hq/synapse/issues/17725)) -- Fix `_bulk_get_max_event_pos` being inefficient. ([\#17728](https://github.com/element-hq/synapse/issues/17728)) -- Add cache to `get_tags_for_room(...)`. ([\#17730](https://github.com/element-hq/synapse/issues/17730)) -- Small performance improvement in speeding up [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17731](https://github.com/element-hq/synapse/issues/17731)) -- Minor speed up of initial [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync requests. ([\#17734](https://github.com/element-hq/synapse/issues/17734)) -- Remove usage of the deprecated `cgi` module, deprecated in Python 3.11 and removed in Python 3.13. ([\#17741](https://github.com/element-hq/synapse/issues/17741)) -- Fix typing of a variable that is not `Unknown` anymore after updating `treq`. ([\#17744](https://github.com/element-hq/synapse/issues/17744)) - - - -### Updates to locked dependencies - -* Bump anyhow from 1.0.86 to 1.0.89. ([\#17685](https://github.com/element-hq/synapse/issues/17685), [\#17716](https://github.com/element-hq/synapse/issues/17716)) -* Bump bytes from 1.7.1 to 1.7.2. ([\#17743](https://github.com/element-hq/synapse/issues/17743)) -* Bump cryptography from 43.0.0 to 43.0.1. ([\#17689](https://github.com/element-hq/synapse/issues/17689)) -* Bump idna from 3.8 to 3.10. ([\#17758](https://github.com/element-hq/synapse/issues/17758)) -* Bump msgpack from 1.0.8 to 1.1.0. ([\#17759](https://github.com/element-hq/synapse/issues/17759)) -* Bump phonenumbers from 8.13.44 to 8.13.45. ([\#17762](https://github.com/element-hq/synapse/issues/17762)) -* Bump prometheus-client from 0.20.0 to 0.21.0. ([\#17746](https://github.com/element-hq/synapse/issues/17746)) -* Bump pyasn1 from 0.6.0 to 0.6.1. ([\#17714](https://github.com/element-hq/synapse/issues/17714)) -* Bump pyasn1-modules from 0.4.0 to 0.4.1. ([\#17747](https://github.com/element-hq/synapse/issues/17747)) -* Bump pydantic from 2.8.2 to 2.9.2. ([\#17756](https://github.com/element-hq/synapse/issues/17756)) -* Bump python-multipart from 0.0.9 to 0.0.10. ([\#17745](https://github.com/element-hq/synapse/issues/17745)) -* Bump ruff from 0.6.4 to 0.6.7. ([\#17715](https://github.com/element-hq/synapse/issues/17715), [\#17760](https://github.com/element-hq/synapse/issues/17760)) -* Bump sentry-sdk from 2.13.0 to 2.14.0. ([\#17712](https://github.com/element-hq/synapse/issues/17712)) -* Bump serde from 1.0.209 to 1.0.210. ([\#17686](https://github.com/element-hq/synapse/issues/17686)) -* Bump serde_json from 1.0.127 to 1.0.128. ([\#17687](https://github.com/element-hq/synapse/issues/17687)) -* Bump treq from 23.11.0 to 24.9.1. ([\#17744](https://github.com/element-hq/synapse/issues/17744)) -* Bump types-pyyaml from 6.0.12.20240808 to 6.0.12.20240917. ([\#17755](https://github.com/element-hq/synapse/issues/17755)) -* Bump types-requests from 2.32.0.20240712 to 2.32.0.20240914. ([\#17713](https://github.com/element-hq/synapse/issues/17713)) -* Bump types-setuptools from 74.1.0.20240907 to 75.1.0.20240917. ([\#17757](https://github.com/element-hq/synapse/issues/17757)) - -# Synapse 1.115.0 (2024-09-17) - -No significant changes since 1.115.0rc2. - - - - -# Synapse 1.115.0rc2 (2024-09-12) - -### Internal Changes - -- Pre-populate room data used in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint for quick filtering/sorting. ([\#17652](https://github.com/element-hq/synapse/issues/17652)) -- Speed up sliding sync by reducing amount of data pulled out of the database for large rooms. ([\#17683](https://github.com/element-hq/synapse/issues/17683)) - - - - -# Synapse 1.115.0rc1 (2024-09-10) - -### Features - -- Improve cross-signing upload when using [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) to use a custom UIA flow stage, with web fallback support. ([\#17509](https://github.com/element-hq/synapse/issues/17509)) - -### Bugfixes - -- Return `400 M_BAD_JSON` upon attempting to complete various room actions with a non-local user ID and unknown room ID, rather than an internal server error. ([\#17607](https://github.com/element-hq/synapse/issues/17607)) -- Fix authenticated media responses using a wrong limit when following redirects over federation. ([\#17626](https://github.com/element-hq/synapse/issues/17626)) -- Fix bug where we returned the wrong `bump_stamp` for invites in sliding sync response, causing incorrect ordering of invites in the room list. ([\#17674](https://github.com/element-hq/synapse/issues/17674)) - -### Improved Documentation - -- Clarify that the admin api resource is only loaded on the main process and not workers. ([\#17590](https://github.com/element-hq/synapse/issues/17590)) -- Fixed typo in `saml2_config` config [example](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#saml2_config). ([\#17594](https://github.com/element-hq/synapse/issues/17594)) +- Fix documentation of the Delete Room Admin API's status field. ([\#18519](https://github.com/element-hq/synapse/issues/18519)) ### Deprecations and Removals -- Stabilise [MSC4156](https://github.com/matrix-org/matrix-spec-proposals/pull/4156) by removing the `msc4156_enabled` config setting and defaulting it to `true`. ([\#17650](https://github.com/element-hq/synapse/issues/17650)) +- Stop adding the "origin" field to newly-created events (PDUs). ([\#18418](https://github.com/element-hq/synapse/issues/18418)) ### Internal Changes -- Update [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation: load the issuer and account management URLs from OIDC discovery. ([\#17407](https://github.com/element-hq/synapse/issues/17407)) -- Pre-populate room data used in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint for quick filtering/sorting. ([\#17512](https://github.com/element-hq/synapse/issues/17512), [\#17632](https://github.com/element-hq/synapse/issues/17632), [\#17633](https://github.com/element-hq/synapse/issues/17633), [\#17634](https://github.com/element-hq/synapse/issues/17634), [\#17635](https://github.com/element-hq/synapse/issues/17635), [\#17636](https://github.com/element-hq/synapse/issues/17636), [\#17641](https://github.com/element-hq/synapse/issues/17641), [\#17654](https://github.com/element-hq/synapse/issues/17654), [\#17673](https://github.com/element-hq/synapse/issues/17673)) -- Store sliding sync per-connection state in the database. ([\#17599](https://github.com/element-hq/synapse/issues/17599), [\#17631](https://github.com/element-hq/synapse/issues/17631)) -- Make the sliding sync `PerConnectionState` class immutable. ([\#17600](https://github.com/element-hq/synapse/issues/17600)) -- Replace `isort` and `black` with `ruff`. ([\#17620](https://github.com/element-hq/synapse/issues/17620), [\#17643](https://github.com/element-hq/synapse/issues/17643)) -- Sliding Sync: Split up `get_room_membership_for_user_at_to_token`. ([\#17629](https://github.com/element-hq/synapse/issues/17629)) -- Use new database tables for sliding sync. ([\#17630](https://github.com/element-hq/synapse/issues/17630), [\#17649](https://github.com/element-hq/synapse/issues/17649)) -- Prevent duplicate tags being added to Sliding Sync traces. ([\#17655](https://github.com/element-hq/synapse/issues/17655)) -- Get `bump_stamp` from [new sliding sync tables](https://github.com/element-hq/synapse/pull/17512) which should be faster. ([\#17658](https://github.com/element-hq/synapse/issues/17658)) -- Speed up incremental Sliding Sync requests by avoiding extra work. ([\#17665](https://github.com/element-hq/synapse/issues/17665)) -- Small performance improvement in speeding up sliding sync. ([\#17666](https://github.com/element-hq/synapse/issues/17666), [\#17670](https://github.com/element-hq/synapse/issues/17670), [\#17672](https://github.com/element-hq/synapse/issues/17672)) -- Speed up sliding sync by reducing number of database calls. ([\#17684](https://github.com/element-hq/synapse/issues/17684)) -- Speed up sync by pulling out fewer events from the database. ([\#17688](https://github.com/element-hq/synapse/issues/17688)) +- Replace `PyICU` crate with equivalent `icu_segmenter` Rust crate. ([\#18553](https://github.com/element-hq/synapse/issues/18553), [\#18646](https://github.com/element-hq/synapse/issues/18646)) +- Improve docstring on `simple_upsert_many`. ([\#18573](https://github.com/element-hq/synapse/issues/18573)) +- Raise poetry-core version cap to 2.1.3. ([\#18575](https://github.com/element-hq/synapse/issues/18575)) +- Raise setuptools_rust version cap to 1.11.1. ([\#18576](https://github.com/element-hq/synapse/issues/18576)) +- Better handling of ratelimited requests. ([\#18595](https://github.com/element-hq/synapse/issues/18595), [\#18600](https://github.com/element-hq/synapse/issues/18600)) +- Update to Rust 1.87.0 in CI, and bump the pinned commit of the `dtolnay/rust-toolchain` GitHub Action to `b3b07ba8b418998c39fb20f53e8b695cdcc8de1b`. ([\#18596](https://github.com/element-hq/synapse/issues/18596)) +- Speed up bulk device deletion. ([\#18602](https://github.com/element-hq/synapse/issues/18602)) +- Speed up the building of arm-based wheels in CI. ([\#18618](https://github.com/element-hq/synapse/issues/18618)) +- Speed up the building of Docker images in CI. ([\#18620](https://github.com/element-hq/synapse/issues/18620)) +- Add `.zed/` directory to `.gitignore`. ([\#18623](https://github.com/element-hq/synapse/issues/18623)) +- Log the room ID we're purging state for. ([\#18625](https://github.com/element-hq/synapse/issues/18625)) ### Updates to locked dependencies -* Bump authlib from 1.3.1 to 1.3.2. ([\#17679](https://github.com/element-hq/synapse/issues/17679)) -* Bump idna from 3.7 to 3.8. ([\#17682](https://github.com/element-hq/synapse/issues/17682)) -* Bump ruff from 0.6.2 to 0.6.4. ([\#17680](https://github.com/element-hq/synapse/issues/17680)) -* Bump towncrier from 24.7.1 to 24.8.0. ([\#17645](https://github.com/element-hq/synapse/issues/17645)) -* Bump twisted from 24.7.0rc1 to 24.7.0. ([\#17647](https://github.com/element-hq/synapse/issues/17647)) -* Bump types-pillow from 10.2.0.20240520 to 10.2.0.20240822. ([\#17644](https://github.com/element-hq/synapse/issues/17644)) -* Bump types-psycopg2 from 2.9.21.20240417 to 2.9.21.20240819. ([\#17646](https://github.com/element-hq/synapse/issues/17646)) -* Bump types-setuptools from 71.1.0.20240818 to 74.1.0.20240907. ([\#17681](https://github.com/element-hq/synapse/issues/17681)) +* Bump Swatinem/rust-cache from 2.7.8 to 2.8.0. ([\#18612](https://github.com/element-hq/synapse/issues/18612)) +* Bump attrs from 24.2.0 to 25.3.0. ([\#18649](https://github.com/element-hq/synapse/issues/18649)) +* Bump authlib from 1.5.2 to 1.6.0. ([\#18642](https://github.com/element-hq/synapse/issues/18642)) +* Bump base64 from 0.21.7 to 0.22.1. ([\#18589](https://github.com/element-hq/synapse/issues/18589)) +* Bump base64 from 0.21.7 to 0.22.1. ([\#18629](https://github.com/element-hq/synapse/issues/18629)) +* Bump docker/build-push-action from 6.17.0 to 6.18.0. ([\#18497](https://github.com/element-hq/synapse/issues/18497)) +* Bump docker/setup-buildx-action from 3.10.0 to 3.11.1. ([\#18587](https://github.com/element-hq/synapse/issues/18587)) +* Bump hiredis from 3.1.0 to 3.2.1. ([\#18638](https://github.com/element-hq/synapse/issues/18638)) +* Bump ijson from 3.3.0 to 3.4.0. ([\#18650](https://github.com/element-hq/synapse/issues/18650)) +* Bump jsonschema from 4.23.0 to 4.24.0. ([\#18630](https://github.com/element-hq/synapse/issues/18630)) +* Bump msgpack from 1.1.0 to 1.1.1. ([\#18651](https://github.com/element-hq/synapse/issues/18651)) +* Bump mypy-zope from 1.0.11 to 1.0.12. ([\#18640](https://github.com/element-hq/synapse/issues/18640)) +* Bump phonenumbers from 9.0.2 to 9.0.8. ([\#18652](https://github.com/element-hq/synapse/issues/18652)) +* Bump pillow from 11.2.1 to 11.3.0. ([\#18624](https://github.com/element-hq/synapse/issues/18624)) +* Bump prometheus-client from 0.21.0 to 0.22.1. ([\#18609](https://github.com/element-hq/synapse/issues/18609)) +* Bump pyasn1-modules from 0.4.1 to 0.4.2. ([\#18495](https://github.com/element-hq/synapse/issues/18495)) +* Bump pydantic from 2.11.4 to 2.11.7. ([\#18639](https://github.com/element-hq/synapse/issues/18639)) +* Bump reqwest from 0.12.15 to 0.12.20. ([\#18590](https://github.com/element-hq/synapse/issues/18590)) +* Bump reqwest from 0.12.20 to 0.12.22. ([\#18627](https://github.com/element-hq/synapse/issues/18627)) +* Bump ruff from 0.11.11 to 0.12.1. ([\#18645](https://github.com/element-hq/synapse/issues/18645)) +* Bump ruff from 0.12.1 to 0.12.2. ([\#18657](https://github.com/element-hq/synapse/issues/18657)) +* Bump sentry-sdk from 2.22.0 to 2.32.0. ([\#18633](https://github.com/element-hq/synapse/issues/18633)) +* Bump setuptools-rust from 1.10.2 to 1.11.1. ([\#18655](https://github.com/element-hq/synapse/issues/18655)) +* Bump sigstore/cosign-installer from 3.8.2 to 3.9.0. ([\#18588](https://github.com/element-hq/synapse/issues/18588)) +* Bump sigstore/cosign-installer from 3.9.0 to 3.9.1. ([\#18608](https://github.com/element-hq/synapse/issues/18608)) +* Bump stefanzweifel/git-auto-commit-action from 5.2.0 to 6.0.1. ([\#18607](https://github.com/element-hq/synapse/issues/18607)) +* Bump tokio from 1.45.1 to 1.46.0. ([\#18628](https://github.com/element-hq/synapse/issues/18628)) +* Bump tokio from 1.46.0 to 1.46.1. ([\#18667](https://github.com/element-hq/synapse/issues/18667)) +* Bump treq from 24.9.1 to 25.5.0. ([\#18610](https://github.com/element-hq/synapse/issues/18610)) +* Bump types-bleach from 6.2.0.20241123 to 6.2.0.20250514. ([\#18634](https://github.com/element-hq/synapse/issues/18634)) +* Bump types-jsonschema from 4.23.0.20250516 to 4.24.0.20250528. ([\#18611](https://github.com/element-hq/synapse/issues/18611)) +* Bump types-opentracing from 2.4.10.6 to 2.4.10.20250622. ([\#18586](https://github.com/element-hq/synapse/issues/18586)) +* Bump types-psycopg2 from 2.9.21.20250318 to 2.9.21.20250516. ([\#18658](https://github.com/element-hq/synapse/issues/18658)) +* Bump types-pyyaml from 6.0.12.20241230 to 6.0.12.20250516. ([\#18643](https://github.com/element-hq/synapse/issues/18643)) +* Bump types-setuptools from 75.2.0.20241019 to 80.9.0.20250529. ([\#18644](https://github.com/element-hq/synapse/issues/18644)) +* Bump typing-extensions from 4.12.2 to 4.14.0. ([\#18654](https://github.com/element-hq/synapse/issues/18654)) +* Bump typing-extensions from 4.14.0 to 4.14.1. ([\#18668](https://github.com/element-hq/synapse/issues/18668)) +* Bump urllib3 from 2.2.2 to 2.5.0. ([\#18572](https://github.com/element-hq/synapse/issues/18572)) -# Synapse 1.114.0 (2024-09-02) +# Synapse 1.133.0 (2025-07-01) -This release enables support for -[MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) — -Simplified Sliding Sync. This allows using the upcoming releases of the Element -X mobile apps without having to run a Sliding Sync Proxy. +Pre-built wheels are now built using the [manylinux_2_28](https://github.com/pypa/manylinux#manylinux_2_28-almalinux-8-based) base, which is expected to be compatible with distros using glibc 2.28 or later, including: + - Debian 10+ + - Ubuntu 18.10+ + - Fedora 29+ + - CentOS/RHEL 8+ -### Features - -- Enable native sliding sync support ([MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) and [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186)) by default. ([\#17648](https://github.com/element-hq/synapse/issues/17648)) - - - - -# Synapse 1.114.0rc3 (2024-08-30) +Previously, wheels were built using the [manylinux2014](https://github.com/pypa/manylinux#manylinux2014-centos-7-based-glibc-217) base, which was expected to be compatible with distros using glibc 2.17 or later. ### Bugfixes -- Fix regression in v1.114.0rc2 that caused workers to fail to start. ([\#17626](https://github.com/element-hq/synapse/issues/17626)) +- Bump `cibuildwheel` to 3.0.0 to fix the `manylinux` wheel builds. ([\#18615](https://github.com/element-hq/synapse/issues/18615)) -# Synapse 1.114.0rc2 (2024-08-30) +# Synapse 1.133.0rc1 (2025-06-24) ### Features -- Improve cross-signing upload when using [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) to use a custom UIA flow stage, with web fallback support. ([\#17509](https://github.com/element-hq/synapse/issues/17509)) -- Make `hash_password` script accept password input from stdin. ([\#17608](https://github.com/element-hq/synapse/issues/17608)) +- Add support for the [MSC4260 user report API](https://github.com/matrix-org/matrix-spec-proposals/pull/4260). ([\#18120](https://github.com/element-hq/synapse/issues/18120)) ### Bugfixes -- Fix hierarchy returning 403 when room is accessible through federation. Contributed by Krishan (@kfiven). ([\#17194](https://github.com/element-hq/synapse/issues/17194)) -- Fix content-length on federation `/thumbnail` responses. ([\#17532](https://github.com/element-hq/synapse/issues/17532)) -- Fix authenticated media responses using a wrong limit when following redirects over federation. ([\#17543](https://github.com/element-hq/synapse/issues/17543)) - -### Internal Changes - -- MSC3861: load the issuer and account management URLs from OIDC discovery. ([\#17407](https://github.com/element-hq/synapse/issues/17407)) -- Refactor sliding sync class into multiple files. ([\#17595](https://github.com/element-hq/synapse/issues/17595)) -- Store sliding sync per-connection state in the database. ([\#17599](https://github.com/element-hq/synapse/issues/17599)) -- Make the sliding sync `PerConnectionState` class immutable. ([\#17600](https://github.com/element-hq/synapse/issues/17600)) -- Add support to `@tag_args` for standalone functions. ([\#17604](https://github.com/element-hq/synapse/issues/17604)) -- Speed up incremental syncs in sliding sync by adding some more caching. ([\#17606](https://github.com/element-hq/synapse/issues/17606)) -- Always return the user's own read receipts in sliding sync. ([\#17617](https://github.com/element-hq/synapse/issues/17617)) -- Replace `isort` and `black` with `ruff`. ([\#17620](https://github.com/element-hq/synapse/issues/17620)) -- Refactor sliding sync code to move room list logic out into a separate class. ([\#17622](https://github.com/element-hq/synapse/issues/17622)) - - - -### Updates to locked dependencies - -* Bump attrs from 23.2.0 to 24.2.0. ([\#17609](https://github.com/element-hq/synapse/issues/17609)) -* Bump cryptography from 42.0.8 to 43.0.0. ([\#17584](https://github.com/element-hq/synapse/issues/17584)) -* Bump phonenumbers from 8.13.43 to 8.13.44. ([\#17610](https://github.com/element-hq/synapse/issues/17610)) -* Bump pygithub from 2.3.0 to 2.4.0. ([\#17612](https://github.com/element-hq/synapse/issues/17612)) -* Bump pyyaml from 6.0.1 to 6.0.2. ([\#17611](https://github.com/element-hq/synapse/issues/17611)) -* Bump sentry-sdk from 2.12.0 to 2.13.0. ([\#17585](https://github.com/element-hq/synapse/issues/17585)) -* Bump serde from 1.0.206 to 1.0.208. ([\#17581](https://github.com/element-hq/synapse/issues/17581)) -* Bump serde from 1.0.208 to 1.0.209. ([\#17613](https://github.com/element-hq/synapse/issues/17613)) -* Bump serde_json from 1.0.124 to 1.0.125. ([\#17582](https://github.com/element-hq/synapse/issues/17582)) -* Bump serde_json from 1.0.125 to 1.0.127. ([\#17614](https://github.com/element-hq/synapse/issues/17614)) -* Bump types-jsonschema from 4.23.0.20240712 to 4.23.0.20240813. ([\#17583](https://github.com/element-hq/synapse/issues/17583)) -* Bump types-setuptools from 71.1.0.20240726 to 71.1.0.20240818. ([\#17586](https://github.com/element-hq/synapse/issues/17586)) - -# Synapse 1.114.0rc1 (2024-08-20) - -### Features - -- Add a flag to `/versions`, `org.matrix.simplified_msc3575`, to indicate whether experimental sliding sync support has been enabled. ([\#17571](https://github.com/element-hq/synapse/issues/17571)) -- Handle changes in `timeline_limit` in experimental sliding sync. ([\#17579](https://github.com/element-hq/synapse/issues/17579)) -- Correctly track read receipts that should be sent down in experimental sliding sync. ([\#17575](https://github.com/element-hq/synapse/issues/17575), [\#17589](https://github.com/element-hq/synapse/issues/17589), [\#17592](https://github.com/element-hq/synapse/issues/17592)) - -### Bugfixes - -- Start handlers for new media endpoints when media resource configured. ([\#17483](https://github.com/element-hq/synapse/issues/17483)) -- Fix timeline ordering (using `stream_ordering` instead of topological ordering) in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17510](https://github.com/element-hq/synapse/issues/17510)) -- Fix experimental sliding sync implementation to remember any updates in rooms that were not sent down immediately. ([\#17535](https://github.com/element-hq/synapse/issues/17535)) -- Better exclude partially stated rooms if we must await full state in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17538](https://github.com/element-hq/synapse/issues/17538)) -- Handle lower-case http headers in `_Mulitpart_Parser_Protocol`. ([\#17545](https://github.com/element-hq/synapse/issues/17545)) -- Fix fetching federation signing keys from servers that omit `old_verify_keys`. Contributed by @tulir @ Beeper. ([\#17568](https://github.com/element-hq/synapse/issues/17568)) -- Fix bug where we would respond with an error when a remote server asked for media that had a length of 0, using the new multipart federation media endpoint. ([\#17570](https://github.com/element-hq/synapse/issues/17570)) +- Fix an issue where, during state resolution for v11 rooms, Synapse would incorrectly calculate the power level of the creator when there was no power levels event in the room. ([\#18534](https://github.com/element-hq/synapse/issues/18534), [\#18547](https://github.com/element-hq/synapse/issues/18547)) +- Fix long-standing bug where sliding sync did not honour the `room_id_to_include` config option. ([\#18535](https://github.com/element-hq/synapse/issues/18535)) +- Fix an issue where "Lock timeout is getting excessive" warnings would be logged even when the lock timeout was <10 minutes. ([\#18543](https://github.com/element-hq/synapse/issues/18543)) +- Fix an issue where Synapse could calculate the wrong power level for the creator of the room if there was no power levels event. ([\#18545](https://github.com/element-hq/synapse/issues/18545)) ### Improved Documentation -- Clarify default behaviour of the - [`auto_accept_invites.worker_to_run_on`](https://element-hq.github.io/synapse/develop/usage/configuration/config_documentation.html#auto-accept-invites) - option. ([\#17515](https://github.com/element-hq/synapse/issues/17515)) -- Improve docstrings for profile methods. ([\#17559](https://github.com/element-hq/synapse/issues/17559)) +- Generate config documentation from JSON Schema file. ([\#18528](https://github.com/element-hq/synapse/issues/18528)) +- Fix typo in user type documentation. ([\#18568](https://github.com/element-hq/synapse/issues/18568)) ### Internal Changes -- Add more tracing to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17514](https://github.com/element-hq/synapse/issues/17514)) -- Fixup comment in sliding sync implementation. ([\#17531](https://github.com/element-hq/synapse/issues/17531)) -- Replace override of deprecated method `HTTPAdapter.get_connection` with `get_connection_with_tls_context`. ([\#17536](https://github.com/element-hq/synapse/issues/17536)) -- Fix performance of device lists in `/key/changes` and sliding sync. ([\#17537](https://github.com/element-hq/synapse/issues/17537), [\#17548](https://github.com/element-hq/synapse/issues/17548)) -- Bump setuptools from 67.6.0 to 72.1.0. ([\#17542](https://github.com/element-hq/synapse/issues/17542)) -- Add a utility function for generating random event IDs. ([\#17557](https://github.com/element-hq/synapse/issues/17557)) -- Speed up responding to media requests. ([\#17558](https://github.com/element-hq/synapse/issues/17558), [\#17561](https://github.com/element-hq/synapse/issues/17561), [\#17564](https://github.com/element-hq/synapse/issues/17564), [\#17566](https://github.com/element-hq/synapse/issues/17566), [\#17567](https://github.com/element-hq/synapse/issues/17567), [\#17569](https://github.com/element-hq/synapse/issues/17569)) -- Test github token before running release script steps. ([\#17562](https://github.com/element-hq/synapse/issues/17562)) -- Reduce log spam of multipart files. ([\#17563](https://github.com/element-hq/synapse/issues/17563)) -- Refactor per-connection state in experimental sliding sync handler. ([\#17574](https://github.com/element-hq/synapse/issues/17574)) -- Add histogram metrics for sliding sync processing time. ([\#17593](https://github.com/element-hq/synapse/issues/17593)) +- Increase performance of introspecting access tokens when using delegated auth. ([\#18357](https://github.com/element-hq/synapse/issues/18357), [\#18561](https://github.com/element-hq/synapse/issues/18561)) +- Log user deactivations. ([\#18541](https://github.com/element-hq/synapse/issues/18541)) +- Enable [`flake8-logging`](https://docs.astral.sh/ruff/rules/#flake8-logging-log) and [`flake8-logging-format`](https://docs.astral.sh/ruff/rules/#flake8-logging-format-g) rules in Ruff and fix related issues throughout the codebase. ([\#18542](https://github.com/element-hq/synapse/issues/18542)) +- Clean up old, unused rows from the `device_federation_inbox` table. ([\#18546](https://github.com/element-hq/synapse/issues/18546)) +- Run config schema CI on develop and release branches. ([\#18551](https://github.com/element-hq/synapse/issues/18551)) +- Add support for Twisted `25.5.0`+ releases. ([\#18577](https://github.com/element-hq/synapse/issues/18577)) +- Update PyO3 to version 0.25. ([\#18578](https://github.com/element-hq/synapse/issues/18578)) ### Updates to locked dependencies -* Bump bytes from 1.6.1 to 1.7.1. ([\#17526](https://github.com/element-hq/synapse/issues/17526)) -* Bump lxml from 5.2.2 to 5.3.0. ([\#17550](https://github.com/element-hq/synapse/issues/17550)) -* Bump phonenumbers from 8.13.42 to 8.13.43. ([\#17551](https://github.com/element-hq/synapse/issues/17551)) -* Bump regex from 1.10.5 to 1.10.6. ([\#17527](https://github.com/element-hq/synapse/issues/17527)) -* Bump sentry-sdk from 2.10.0 to 2.12.0. ([\#17553](https://github.com/element-hq/synapse/issues/17553)) -* Bump serde from 1.0.204 to 1.0.206. ([\#17556](https://github.com/element-hq/synapse/issues/17556)) -* Bump serde_json from 1.0.122 to 1.0.124. ([\#17555](https://github.com/element-hq/synapse/issues/17555)) -* Bump sigstore/cosign-installer from 3.5.0 to 3.6.0. ([\#17549](https://github.com/element-hq/synapse/issues/17549)) -* Bump types-pyyaml from 6.0.12.20240311 to 6.0.12.20240808. ([\#17552](https://github.com/element-hq/synapse/issues/17552)) -* Bump types-requests from 2.31.0.20240406 to 2.32.0.20240712. ([\#17524](https://github.com/element-hq/synapse/issues/17524)) +* Bump actions/setup-python from 5.5.0 to 5.6.0. ([\#18555](https://github.com/element-hq/synapse/issues/18555)) +* Bump base64 from 0.21.7 to 0.22.1. ([\#18559](https://github.com/element-hq/synapse/issues/18559)) +* Bump dawidd6/action-download-artifact from 9 to 11. ([\#18556](https://github.com/element-hq/synapse/issues/18556)) +* Bump headers from 0.4.0 to 0.4.1. ([\#18529](https://github.com/element-hq/synapse/issues/18529)) +* Bump requests from 2.32.2 to 2.32.4. ([\#18533](https://github.com/element-hq/synapse/issues/18533)) +* Bump types-requests from 2.32.0.20250328 to 2.32.4.20250611. ([\#18558](https://github.com/element-hq/synapse/issues/18558)) -# Synapse 1.113.0 (2024-08-13) - -No significant changes since 1.113.0rc1. - - - - -# Synapse 1.113.0rc1 (2024-08-06) - -### Features - -- Track which rooms have been sent to clients in the experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17447](https://github.com/element-hq/synapse/issues/17447)) -- Add Account Data extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17477](https://github.com/element-hq/synapse/issues/17477)) -- Add receipts extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17489](https://github.com/element-hq/synapse/issues/17489)) -- Add typing notification extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17505](https://github.com/element-hq/synapse/issues/17505)) - -### Bugfixes - -- Update experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint to handle invite/knock rooms when filtering. ([\#17450](https://github.com/element-hq/synapse/issues/17450)) -- Fix a bug introduced in v1.110.0 which caused `/keys/query` to return incomplete results, leading to high network activity and CPU usage on Matrix clients. ([\#17499](https://github.com/element-hq/synapse/issues/17499)) +# Synapse 1.132.0 (2025-06-17) ### Improved Documentation -- Update the [`allowed_local_3pids`](https://element-hq.github.io/synapse/v1.112/usage/configuration/config_documentation.html#allowed_local_3pids) config option's msisdn address to a working example. ([\#17476](https://github.com/element-hq/synapse/issues/17476)) +- Improvements to generate config documentation from JSON Schema file. ([\#18522](https://github.com/element-hq/synapse/issues/18522)) + + + + +# Synapse 1.132.0rc1 (2025-06-10) + +### Features + +- Add support for [MSC4155](https://github.com/matrix-org/matrix-spec-proposals/pull/4155) Invite Filtering. ([\#18288](https://github.com/element-hq/synapse/issues/18288)) +- Add experimental `user_may_send_state_event` module API callback. ([\#18455](https://github.com/element-hq/synapse/issues/18455)) +- Add experimental `get_media_config_for_user` and `is_user_allowed_to_upload_media_of_size` module API callbacks that allow overriding of media repository maximum upload size. ([\#18457](https://github.com/element-hq/synapse/issues/18457)) +- Add experimental `get_ratelimit_override_for_user` module API callback that allows overriding of per-user ratelimits. ([\#18458](https://github.com/element-hq/synapse/issues/18458)) +- Pass `room_config` argument to `user_may_create_room` spam checker module callback. ([\#18486](https://github.com/element-hq/synapse/issues/18486)) +- Support configuration of default and extra user types. ([\#18456](https://github.com/element-hq/synapse/issues/18456)) +- Successful requests to `/_matrix/app/v1/ping` will now force Synapse to reattempt delivering transactions to appservices. ([\#18521](https://github.com/element-hq/synapse/issues/18521)) +- Support the import of the `RatelimitOverride` type from `synapse.module_api` in modules and rename `messages_per_second` to `per_second`. ([\#18513](https://github.com/element-hq/synapse/issues/18513)) + +### Bugfixes + +- Remove destinations from sending if not whitelisted. ([\#18484](https://github.com/element-hq/synapse/issues/18484)) +- Fixed room summary API incorrectly returning that a room is private in the room summary response when the join rule is omitted by the remote server. Contributed by @nexy7574. ([\#18493](https://github.com/element-hq/synapse/issues/18493)) +- Prevent users from adding themselves to their own user ignore list. ([\#18508](https://github.com/element-hq/synapse/issues/18508)) + +### Improved Documentation + +- Generate config documentation from JSON Schema file. ([\#17892](https://github.com/element-hq/synapse/issues/17892)) +- Mention `CAP_NET_BIND_SERVICE` as an alternative to running Synapse as root in order to bind to a privileged port. ([\#18408](https://github.com/element-hq/synapse/issues/18408)) +- Surface hidden Admin API documentation regarding fetching of scheduled tasks. ([\#18516](https://github.com/element-hq/synapse/issues/18516)) +- Mark the new module APIs in this release as experimental. ([\#18536](https://github.com/element-hq/synapse/issues/18536)) ### Internal Changes -- Change sliding sync to use their own token format in preparation for storing per-connection state. ([\#17452](https://github.com/element-hq/synapse/issues/17452)) -- Ensure we don't send down negative `bump_stamp` in experimental sliding sync endpoint. ([\#17478](https://github.com/element-hq/synapse/issues/17478)) -- Do not send down empty room entries down experimental sliding sync endpoint. ([\#17479](https://github.com/element-hq/synapse/issues/17479)) -- Refactor Sliding Sync tests to better utilize the `SlidingSyncBase`. ([\#17481](https://github.com/element-hq/synapse/issues/17481), [\#17482](https://github.com/element-hq/synapse/issues/17482)) -- Add some opentracing tags and logging to the experimental sliding sync implementation. ([\#17501](https://github.com/element-hq/synapse/issues/17501)) -- Split and move Sliding Sync tests so we have some more sane test file sizes. ([\#17504](https://github.com/element-hq/synapse/issues/17504)) -- Update the `limited` field description in the Sliding Sync response to accurately describe what it actually represents. ([\#17507](https://github.com/element-hq/synapse/issues/17507)) -- Easier to understand `timeline` assertions in Sliding Sync tests. ([\#17511](https://github.com/element-hq/synapse/issues/17511)) -- Reset the sliding sync connection if we don't recognize the per-connection state position. ([\#17529](https://github.com/element-hq/synapse/issues/17529)) +- Mark dehydrated devices in the [List All User Devices Admin API](https://element-hq.github.io/synapse/latest/admin_api/user_admin_api.html#list-all-devices). ([\#18252](https://github.com/element-hq/synapse/issues/18252)) +- Reduce disk wastage by cleaning up `received_transactions` older than 1 day, rather than 30 days. ([\#18310](https://github.com/element-hq/synapse/issues/18310)) +- Distinguish all vs local events being persisted in the "Event Send Time Quantiles" graph (Grafana). ([\#18510](https://github.com/element-hq/synapse/issues/18510)) + + + + +# Synapse 1.131.0 (2025-06-03) + +No significant changes since 1.131.0rc1. + +# Synapse 1.131.0rc1 (2025-05-28) + +### Features + +- Add `msc4263_limit_key_queries_to_users_who_share_rooms` config option as per [MSC4263](https://github.com/matrix-org/matrix-spec-proposals/pull/4263). ([\#18180](https://github.com/element-hq/synapse/issues/18180)) +- Add option to allow registrations that begin with `_`. Contributed by `_` (@hex5f). ([\#18262](https://github.com/element-hq/synapse/issues/18262)) +- Include room ID in response to the [Room Deletion Status Admin API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#status-of-deleting-rooms). ([\#18318](https://github.com/element-hq/synapse/issues/18318)) +- Add support for calling Policy Servers ([MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284)) to mark events as spam. ([\#18387](https://github.com/element-hq/synapse/issues/18387)) + +### Bugfixes + +- Prevent race-condition in `_maybe_retry_device_resync` entrance. ([\#18391](https://github.com/element-hq/synapse/issues/18391)) +- Fix the `tests.handlers.test_worker_lock.WorkerLockTestCase.test_lock_contention` test which could spuriously time out on RISC-V architectures due to performance differences. ([\#18430](https://github.com/element-hq/synapse/issues/18430)) +- Fix admin redaction endpoint not redacting encrypted messages. ([\#18434](https://github.com/element-hq/synapse/issues/18434)) + +### Improved Documentation + +- Update `room_list_publication_rules` docs to consider defaults that changed in v1.126.0. Contributed by @HarHarLinks. ([\#18286](https://github.com/element-hq/synapse/issues/18286)) +- Add advice for upgrading between major PostgreSQL versions to the database documentation. ([\#18445](https://github.com/element-hq/synapse/issues/18445)) + +### Internal Changes + +- Fix a memory leak in `_NotifierUserStream`. ([\#18380](https://github.com/element-hq/synapse/issues/18380)) +- Fix a couple type annotations in the `RootConfig`/`Config`. ([\#18409](https://github.com/element-hq/synapse/issues/18409)) +- Explicitly enable PyPy builds in `cibuildwheel`s config to avoid it being disabled on a future upgrade to `cibuildwheel` v3. ([\#18417](https://github.com/element-hq/synapse/issues/18417)) +- Update the PR review template to remove an erroneous line break from the final bullet point. ([\#18419](https://github.com/element-hq/synapse/issues/18419)) +- Explain why we `flush_buffer()` for Python `print(...)` output. ([\#18420](https://github.com/element-hq/synapse/issues/18420)) +- Add lint to ensure we don't add a `CREATE/DROP INDEX` in a schema delta. ([\#18440](https://github.com/element-hq/synapse/issues/18440)) +- Allow checking only for the existence of a field in an SSO provider's response, rather than requiring the value(s) to check. ([\#18454](https://github.com/element-hq/synapse/issues/18454)) +- Add unit tests for homeserver usage statistics. ([\#18463](https://github.com/element-hq/synapse/issues/18463)) +- Don't move invited users to new room when shutting down room. ([\#18471](https://github.com/element-hq/synapse/issues/18471)) ### Updates to locked dependencies -* Bump bcrypt from 4.1.3 to 4.2.0. ([\#17495](https://github.com/element-hq/synapse/issues/17495)) -* Bump black from 24.4.2 to 24.8.0. ([\#17522](https://github.com/element-hq/synapse/issues/17522)) -* Bump phonenumbers from 8.13.39 to 8.13.42. ([\#17521](https://github.com/element-hq/synapse/issues/17521)) -* Bump ruff from 0.5.4 to 0.5.5. ([\#17494](https://github.com/element-hq/synapse/issues/17494)) -* Bump serde_json from 1.0.120 to 1.0.121. ([\#17493](https://github.com/element-hq/synapse/issues/17493)) -* Bump serde_json from 1.0.121 to 1.0.122. ([\#17525](https://github.com/element-hq/synapse/issues/17525)) -* Bump towncrier from 23.11.0 to 24.7.1. ([\#17523](https://github.com/element-hq/synapse/issues/17523)) -* Bump types-pyopenssl from 24.1.0.20240425 to 24.1.0.20240722. ([\#17496](https://github.com/element-hq/synapse/issues/17496)) -* Bump types-setuptools from 70.1.0.20240627 to 71.1.0.20240726. ([\#17497](https://github.com/element-hq/synapse/issues/17497)) +* Bump actions/setup-python from 5.5.0 to 5.6.0. ([\#18398](https://github.com/element-hq/synapse/issues/18398)) +* Bump authlib from 1.5.1 to 1.5.2. ([\#18452](https://github.com/element-hq/synapse/issues/18452)) +* Bump docker/build-push-action from 6.15.0 to 6.17.0. ([\#18397](https://github.com/element-hq/synapse/issues/18397), [\#18449](https://github.com/element-hq/synapse/issues/18449)) +* Bump lxml from 5.3.0 to 5.4.0. ([\#18480](https://github.com/element-hq/synapse/issues/18480)) +* Bump mypy-zope from 1.0.9 to 1.0.11. ([\#18428](https://github.com/element-hq/synapse/issues/18428)) +* Bump pyo3 from 0.23.5 to 0.24.2. ([\#18460](https://github.com/element-hq/synapse/issues/18460)) +* Bump pyo3-log from 0.12.3 to 0.12.4. ([\#18453](https://github.com/element-hq/synapse/issues/18453)) +* Bump pyopenssl from 25.0.0 to 25.1.0. ([\#18450](https://github.com/element-hq/synapse/issues/18450)) +* Bump ruff from 0.7.3 to 0.11.11. ([\#18451](https://github.com/element-hq/synapse/issues/18451), [\#18482](https://github.com/element-hq/synapse/issues/18482)) +* Bump tornado from 6.4.2 to 6.5.0. ([\#18459](https://github.com/element-hq/synapse/issues/18459)) +* Bump setuptools from 72.1.0 to 78.1.1. ([\#18461](https://github.com/element-hq/synapse/issues/18461)) +* Bump types-jsonschema from 4.23.0.20241208 to 4.23.0.20250516. ([\#18481](https://github.com/element-hq/synapse/issues/18481)) +* Bump types-requests from 2.32.0.20241016 to 2.32.0.20250328. ([\#18427](https://github.com/element-hq/synapse/issues/18427)) -# Synapse 1.112.0 (2024-07-30) +# Synapse 1.130.0 (2025-05-20) -This security release is to update our locked dependency on Twisted to 24.7.0rc1, which includes a security fix for [CVE-2024-41671 / GHSA-c8m8-j448-xjx7: Disordered HTTP pipeline response in twisted.web, again](https://github.com/twisted/twisted/security/advisories/GHSA-c8m8-j448-xjx7). +### Bugfixes -Note that this security fix is also available as **Synapse 1.111.1**, which does not include the rest of the changes in Synapse 1.112.0. - -This issue means that, if multiple HTTP requests are pipelined in the same TCP connection, Synapse can send responses to the wrong HTTP request. -If a reverse proxy was configured to use HTTP pipelining, this could result in responses being sent to the wrong user, severely harming confidentiality. - -With that said, despite being a high severity issue, **we consider it unlikely that Synapse installations will be affected**. -The use of HTTP pipelining in this fashion would cause worse performance for clients (request-response latencies would be increased as users' responses would be artificially blocked behind other users' slow requests). Further, Nginx and Haproxy, two common reverse proxies, do not appear to support configuring their upstreams to use HTTP pipelining and thus would not be affected. For both of these reasons, we consider it unlikely that a Synapse deployment would be set up in such a configuration. - -Despite that, we cannot rule out that some installations may exist with this unusual setup and so we are releasing this security update today. - -**pip users:** Note that by default, upgrading Synapse using pip will not automatically upgrade Twisted. **Please manually install the new version of Twisted** using `pip install Twisted==24.7.0rc1`. Note also that even the `--upgrade-strategy=eager` flag to `pip install -U matrix-synapse` will not upgrade Twisted to a patched version because it is only a release candidate at this time. - -### Internal Changes - -- Upgrade locked dependency on Twisted to 24.7.0rc1. ([\#17502](https://github.com/element-hq/synapse/issues/17502)) +- Fix startup being blocked on creating a new index that was introduced in v1.130.0rc1. ([\#18439](https://github.com/element-hq/synapse/issues/18439)) +- Fix the ordering of local messages in rooms that were affected by [GHSA-v56r-hwv5-mxg6](https://github.com/advisories/GHSA-v56r-hwv5-mxg6). ([\#18447](https://github.com/element-hq/synapse/issues/18447)) -# Synapse 1.112.0rc1 (2024-07-23) -Please note that this release candidate does not include the security dependency update -included in version 1.111.1 as this version was released before 1.111.1. -The same security fix can be found in the full release of 1.112.0. + +# Synapse 1.130.0rc1 (2025-05-13) ### Features -- Add to-device extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17416](https://github.com/element-hq/synapse/issues/17416)) -- Populate `name`/`avatar` fields in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17418](https://github.com/element-hq/synapse/issues/17418)) -- Populate `heroes` and room summary fields (`joined_count`, `invited_count`) in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17419](https://github.com/element-hq/synapse/issues/17419)) -- Populate `is_dm` room field in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17429](https://github.com/element-hq/synapse/issues/17429)) -- Add room subscriptions to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17432](https://github.com/element-hq/synapse/issues/17432)) -- Prepare for authenticated media freeze. ([\#17433](https://github.com/element-hq/synapse/issues/17433)) -- Add E2EE extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17454](https://github.com/element-hq/synapse/issues/17454)) +- Add an Admin API endpoint `GET /_synapse/admin/v1/scheduled_tasks` to fetch scheduled tasks. ([\#18214](https://github.com/element-hq/synapse/issues/18214)) +- Add config option `user_directory.exclude_remote_users` which, when enabled, excludes remote users from user directory search results. ([\#18300](https://github.com/element-hq/synapse/issues/18300)) +- Add support for handling `GET /devices/` on workers. ([\#18355](https://github.com/element-hq/synapse/issues/18355)) ### Bugfixes -- Add configurable option to always include offline users in presence sync results. Contributed by @Michael-Hollister. ([\#17231](https://github.com/element-hq/synapse/issues/17231)) -- Fix bug in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint when using room type filters and the user has one or more remote invites. ([\#17434](https://github.com/element-hq/synapse/issues/17434)) -- Order `heroes` by `stream_ordering` as the Matrix specification states (applies to `/sync`). ([\#17435](https://github.com/element-hq/synapse/issues/17435)) -- Fix rare bug where `/sync` would break for a user when using workers with multiple stream writers. ([\#17438](https://github.com/element-hq/synapse/issues/17438)) - -### Improved Documentation - -- Update the readme image to have a white background, so that it is readable in dark mode. ([\#17387](https://github.com/element-hq/synapse/issues/17387)) -- Add Red Hat Enterprise Linux and Rocky Linux 8 and 9 installation instructions. ([\#17423](https://github.com/element-hq/synapse/issues/17423)) -- Improve documentation for the [`default_power_level_content_override`](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#default_power_level_content_override) config option. ([\#17451](https://github.com/element-hq/synapse/issues/17451)) - -### Internal Changes - -- Make sure we always use the right logic for enabling the media repo. ([\#17424](https://github.com/element-hq/synapse/issues/17424)) -- Fix argument documentation for method `RateLimiter.record_action`. ([\#17426](https://github.com/element-hq/synapse/issues/17426)) -- Reduce volume of 'Waiting for current token' logs, which were introduced in v1.109.0. ([\#17428](https://github.com/element-hq/synapse/issues/17428)) -- Limit concurrent remote downloads to 6 per IP address, and decrement remote downloads without a content-length from the ratelimiter after the download is complete. ([\#17439](https://github.com/element-hq/synapse/issues/17439)) -- Remove unnecessary call to resume producing in fake channel. ([\#17449](https://github.com/element-hq/synapse/issues/17449)) -- Update experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint to bump room when it is created. ([\#17453](https://github.com/element-hq/synapse/issues/17453)) -- Speed up generating sliding sync responses. ([\#17458](https://github.com/element-hq/synapse/issues/17458)) -- Add cache to `get_rooms_for_local_user_where_membership_is` to speed up sliding sync. ([\#17460](https://github.com/element-hq/synapse/issues/17460)) -- Speed up fetching room keys from backup. ([\#17461](https://github.com/element-hq/synapse/issues/17461)) -- Speed up sorting of the room list in sliding sync. ([\#17468](https://github.com/element-hq/synapse/issues/17468)) -- Implement handling of `$ME` as a state key in sliding sync. ([\#17469](https://github.com/element-hq/synapse/issues/17469)) - - - -### Updates to locked dependencies - -* Bump bytes from 1.6.0 to 1.6.1. ([\#17441](https://github.com/element-hq/synapse/issues/17441)) -* Bump hiredis from 2.3.2 to 3.0.0. ([\#17464](https://github.com/element-hq/synapse/issues/17464)) -* Bump jsonschema from 4.22.0 to 4.23.0. ([\#17444](https://github.com/element-hq/synapse/issues/17444)) -* Bump matrix-org/done-action from 2 to 3. ([\#17440](https://github.com/element-hq/synapse/issues/17440)) -* Bump mypy from 1.9.0 to 1.10.1. ([\#17445](https://github.com/element-hq/synapse/issues/17445)) -* Bump pyopenssl from 24.1.0 to 24.2.1. ([\#17465](https://github.com/element-hq/synapse/issues/17465)) -* Bump ruff from 0.5.0 to 0.5.4. ([\#17466](https://github.com/element-hq/synapse/issues/17466)) -* Bump sentry-sdk from 2.6.0 to 2.8.0. ([\#17456](https://github.com/element-hq/synapse/issues/17456)) -* Bump sentry-sdk from 2.8.0 to 2.10.0. ([\#17467](https://github.com/element-hq/synapse/issues/17467)) -* Bump setuptools from 67.6.0 to 70.0.0. ([\#17448](https://github.com/element-hq/synapse/issues/17448)) -* Bump twine from 5.1.0 to 5.1.1. ([\#17443](https://github.com/element-hq/synapse/issues/17443)) -* Bump types-jsonschema from 4.22.0.20240610 to 4.23.0.20240712. ([\#17446](https://github.com/element-hq/synapse/issues/17446)) -* Bump ulid from 1.1.2 to 1.1.3. ([\#17442](https://github.com/element-hq/synapse/issues/17442)) -* Bump zipp from 3.15.0 to 3.19.1. ([\#17427](https://github.com/element-hq/synapse/issues/17427)) - - -# Synapse 1.111.1 (2024-07-30) - -This security release is to update our locked dependency on Twisted to 24.7.0rc1, which includes a security fix for [CVE-2024-41671 / GHSA-c8m8-j448-xjx7: Disordered HTTP pipeline response in twisted.web, again](https://github.com/twisted/twisted/security/advisories/GHSA-c8m8-j448-xjx7). - -This issue means that, if multiple HTTP requests are pipelined in the same TCP connection, Synapse can send responses to the wrong HTTP request. -If a reverse proxy was configured to use HTTP pipelining, this could result in responses being sent to the wrong user, severely harming confidentiality. - -With that said, despite being a high severity issue, **we consider it unlikely that Synapse installations will be affected**. -The use of HTTP pipelining in this fashion would cause worse performance for clients (request-response latencies would be increased as users' responses would be artificially blocked behind other users' slow requests). Further, Nginx and Haproxy, two common reverse proxies, do not appear to support configuring their upstreams to use HTTP pipelining and thus would not be affected. For both of these reasons, we consider it unlikely that a Synapse deployment would be set up in such a configuration. - -Despite that, we cannot rule out that some installations may exist with this unusual setup and so we are releasing this security update today. - -**pip users:** Note that by default, upgrading Synapse using pip will not automatically upgrade Twisted. **Please manually install the new version of Twisted** using `pip install Twisted==24.7.0rc1`. Note also that even the `--upgrade-strategy=eager` flag to `pip install -U matrix-synapse` will not upgrade Twisted to a patched version because it is only a release candidate at this time. - - -### Internal Changes - -- Upgrade locked dependency on Twisted to 24.7.0rc1. ([\#17502](https://github.com/element-hq/synapse/issues/17502)) - - -# Synapse 1.111.0 (2024-07-16) - -No significant changes since 1.111.0rc2. - - - - -# Synapse 1.111.0rc2 (2024-07-10) - -### Bugfixes - -- Fix bug where using `synapse.app.media_repository` worker configuration would break the new media endpoints. ([\#17420](https://github.com/element-hq/synapse/issues/17420)) - -### Improved Documentation - -- Document the new federation media worker endpoints in the [upgrade notes](https://element-hq.github.io/synapse/v1.111/upgrade.html) and [worker docs](https://element-hq.github.io/synapse/v1.111/workers.html). ([\#17421](https://github.com/element-hq/synapse/issues/17421)) - -### Internal Changes - -- Route authenticated federation media requests to media repository workers in Complement tests. ([\#17422](https://github.com/element-hq/synapse/issues/17422)) - - - - -# Synapse 1.111.0rc1 (2024-07-09) - -### Features - -- Add `rooms` data to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17320](https://github.com/element-hq/synapse/issues/17320)) -- Add `room_types`/`not_room_types` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17337](https://github.com/element-hq/synapse/issues/17337)) -- Return "required state" in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17342](https://github.com/element-hq/synapse/issues/17342)) -- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3916-authentication-for-media.md) by adding [`_matrix/client/v1/media/download`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediadownloadservernamemediaid) endpoint. ([\#17365](https://github.com/element-hq/synapse/issues/17365)) -- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/rav/authentication-for-media/proposals/3916-authentication-for-media.md) - by adding [`_matrix/client/v1/media/thumbnail`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediathumbnailservernamemediaid), [`_matrix/federation/v1/media/thumbnail`](https://spec.matrix.org/v1.11/server-server-api/#get_matrixfederationv1mediathumbnailmediaid) endpoints and stabilizing the - remaining [`_matrix/client/v1/media`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediaconfig) endpoints. ([\#17388](https://github.com/element-hq/synapse/issues/17388)) -- Add `rooms.bump_stamp` for easier client-side sorting in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17395](https://github.com/element-hq/synapse/issues/17395)) -- Forget all of a user's rooms upon deactivation, preventing local room purges from being blocked on deactivated users. ([\#17400](https://github.com/element-hq/synapse/issues/17400)) -- Declare support for [Matrix 1.11](https://matrix.org/blog/2024/06/20/matrix-v1.11-release/). ([\#17403](https://github.com/element-hq/synapse/issues/17403)) -- [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861): allow overriding the introspection endpoint. ([\#17406](https://github.com/element-hq/synapse/issues/17406)) - -### Bugfixes - -- Fix rare race which caused no new to-device messages to be received from remote server. ([\#17362](https://github.com/element-hq/synapse/issues/17362)) -- Fix bug in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint when using an old database. ([\#17398](https://github.com/element-hq/synapse/issues/17398)) - -### Improved Documentation - -- Clarify that `url_preview_url_blacklist` is a usability feature. ([\#17356](https://github.com/element-hq/synapse/issues/17356)) -- Fix broken links in README. ([\#17379](https://github.com/element-hq/synapse/issues/17379)) -- Clarify that changelog content *and file extension* need to match in order for entries to merge. ([\#17399](https://github.com/element-hq/synapse/issues/17399)) - -### Internal Changes - -- Make the release script create a release branch for Complement as well. ([\#17318](https://github.com/element-hq/synapse/issues/17318)) -- Fix uploading packages to PyPi. ([\#17363](https://github.com/element-hq/synapse/issues/17363)) -- Add CI check for the README. ([\#17367](https://github.com/element-hq/synapse/issues/17367)) -- Fix linting errors from new `ruff` version. ([\#17381](https://github.com/element-hq/synapse/issues/17381), [\#17411](https://github.com/element-hq/synapse/issues/17411)) -- Fix building debian packages on non-clean checkouts. ([\#17390](https://github.com/element-hq/synapse/issues/17390)) -- Finish up work to allow per-user feature flags. ([\#17392](https://github.com/element-hq/synapse/issues/17392), [\#17410](https://github.com/element-hq/synapse/issues/17410)) -- Allow enabling sliding sync per-user. ([\#17393](https://github.com/element-hq/synapse/issues/17393)) - - - -### Updates to locked dependencies - -* Bump certifi from 2023.7.22 to 2024.7.4. ([\#17404](https://github.com/element-hq/synapse/issues/17404)) -* Bump cryptography from 42.0.7 to 42.0.8. ([\#17382](https://github.com/element-hq/synapse/issues/17382)) -* Bump ijson from 3.2.3 to 3.3.0. ([\#17413](https://github.com/element-hq/synapse/issues/17413)) -* Bump log from 0.4.21 to 0.4.22. ([\#17384](https://github.com/element-hq/synapse/issues/17384)) -* Bump mypy-zope from 1.0.4 to 1.0.5. ([\#17414](https://github.com/element-hq/synapse/issues/17414)) -* Bump pillow from 10.3.0 to 10.4.0. ([\#17412](https://github.com/element-hq/synapse/issues/17412)) -* Bump pydantic from 2.7.1 to 2.8.2. ([\#17415](https://github.com/element-hq/synapse/issues/17415)) -* Bump ruff from 0.3.7 to 0.5.0. ([\#17381](https://github.com/element-hq/synapse/issues/17381)) -* Bump serde from 1.0.203 to 1.0.204. ([\#17409](https://github.com/element-hq/synapse/issues/17409)) -* Bump serde_json from 1.0.117 to 1.0.120. ([\#17385](https://github.com/element-hq/synapse/issues/17385), [\#17408](https://github.com/element-hq/synapse/issues/17408)) -* Bump types-setuptools from 69.5.0.20240423 to 70.1.0.20240627. ([\#17380](https://github.com/element-hq/synapse/issues/17380)) - -# Synapse 1.110.0 (2024-07-03) - -No significant changes since 1.110.0rc3. - - - - -# Synapse 1.110.0rc3 (2024-07-02) - -### Bugfixes - -- Fix bug where `/sync` requests could get blocked indefinitely after an upgrade from Synapse versions before v1.109.0. ([\#17386](https://github.com/element-hq/synapse/issues/17386), [\#17391](https://github.com/element-hq/synapse/issues/17391)) - -### Internal Changes - -- Limit size of presence EDUs to 50 entries. ([\#17371](https://github.com/element-hq/synapse/issues/17371)) -- Fix building debian package for debian sid. ([\#17389](https://github.com/element-hq/synapse/issues/17389)) - - - - -# Synapse 1.110.0rc2 (2024-06-26) - -### Internal Changes - -- Fix uploading packages to PyPi. ([\#17363](https://github.com/element-hq/synapse/issues/17363)) - - - - -# Synapse 1.110.0rc1 (2024-06-26) - -### Features - -- Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17187](https://github.com/element-hq/synapse/issues/17187)) -- Add experimental support for [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823) - Account suspension. ([\#17255](https://github.com/element-hq/synapse/issues/17255)) -- Improve ratelimiting in Synapse. ([\#17256](https://github.com/element-hq/synapse/issues/17256)) -- Add support for the unstable [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151) report room API. ([\#17270](https://github.com/element-hq/synapse/issues/17270), [\#17296](https://github.com/element-hq/synapse/issues/17296)) -- Filter for public and empty rooms added to Admin-API [List Room API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#list-room-api). ([\#17276](https://github.com/element-hq/synapse/issues/17276)) -- Add `is_dm` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17277](https://github.com/element-hq/synapse/issues/17277)) -- Add `is_encrypted` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17281](https://github.com/element-hq/synapse/issues/17281)) -- Include user membership in events served to clients, per [MSC4115](https://github.com/matrix-org/matrix-spec-proposals/pull/4115). ([\#17282](https://github.com/element-hq/synapse/issues/17282)) -- Do not require user-interactive authentication for uploading cross-signing keys for the first time, per [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967). ([\#17284](https://github.com/element-hq/synapse/issues/17284)) -- Add `stream_ordering` sort to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17293](https://github.com/element-hq/synapse/issues/17293)) -- `register_new_matrix_user` now supports a --password-file flag, which - is useful for scripting. ([\#17294](https://github.com/element-hq/synapse/issues/17294)) -- `register_new_matrix_user` now supports a --exists-ok flag to allow registration of users that already exist in the database. - This is useful for scripts that bootstrap user accounts with initial passwords. ([\#17304](https://github.com/element-hq/synapse/issues/17304)) -- Add support for via query parameter from [MSC4156](https://github.com/matrix-org/matrix-spec-proposals/pull/4156). ([\#17322](https://github.com/element-hq/synapse/issues/17322)) -- Add `is_invite` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17335](https://github.com/element-hq/synapse/issues/17335)) -- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3916-authentication-for-media.md) by adding a federation /download endpoint. ([\#17350](https://github.com/element-hq/synapse/issues/17350)) - -### Bugfixes - -- Fix searching for users with their exact localpart whose ID includes a hyphen. ([\#17254](https://github.com/element-hq/synapse/issues/17254)) -- Fix wrong retention policy being used when filtering events. ([\#17272](https://github.com/element-hq/synapse/issues/17272)) -- Fix bug where OTKs were not always included in `/sync` response when using workers. ([\#17275](https://github.com/element-hq/synapse/issues/17275)) -- Fix a long-standing bug where an invalid 'from' parameter to [`/notifications`](https://spec.matrix.org/v1.10/client-server-api/#get_matrixclientv3notifications) would result in an Internal Server Error. ([\#17283](https://github.com/element-hq/synapse/issues/17283)) -- Fix edge case in `/sync` returning the wrong the state when using sharded event persisters. ([\#17295](https://github.com/element-hq/synapse/issues/17295)) -- Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17301](https://github.com/element-hq/synapse/issues/17301)) -- Fix email notification subject when invited to a space. ([\#17336](https://github.com/element-hq/synapse/issues/17336)) - -### Improved Documentation - -- Add missing quotes for example for `exclude_rooms_from_sync`. ([\#17308](https://github.com/element-hq/synapse/issues/17308)) -- Update header in the README to visually fix the the auto-generated table of contents. ([\#17329](https://github.com/element-hq/synapse/issues/17329)) -- Fix stale references to the Foundation's Security Disclosure Policy. ([\#17341](https://github.com/element-hq/synapse/issues/17341)) -- Add default values for `rc_invites.per_issuer` to docs. ([\#17347](https://github.com/element-hq/synapse/issues/17347)) -- Fix an error in the docs for `search_all_users` parameter under `user_directory`. ([\#17348](https://github.com/element-hq/synapse/issues/17348)) - -### Internal Changes - -- Remove unused `expire_access_token` option in the Synapse Docker config file. Contributed by @AaronDewes. ([\#17198](https://github.com/element-hq/synapse/issues/17198)) -- Use fully-qualified `PersistedEventPosition` when returning `RoomsForUser` to facilitate proper comparisons and `RoomStreamToken` generation. ([\#17265](https://github.com/element-hq/synapse/issues/17265)) -- Add debug logging for when room keys are uploaded, including whether they are replacing other room keys. ([\#17266](https://github.com/element-hq/synapse/issues/17266)) -- Handle OTK uploads off master. ([\#17271](https://github.com/element-hq/synapse/issues/17271)) -- Don't try and resync devices for remote users whose servers are marked as down. ([\#17273](https://github.com/element-hq/synapse/issues/17273)) -- Re-organize Pydantic models and types used in handlers. ([\#17279](https://github.com/element-hq/synapse/issues/17279)) -- Expose the worker instance that persisted the event on `event.internal_metadata.instance_name`. ([\#17300](https://github.com/element-hq/synapse/issues/17300)) -- Update the README with Element branding, improve headers and fix the #synapse:matrix.org support room link rendering. ([\#17324](https://github.com/element-hq/synapse/issues/17324)) -- Change path of the experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync implementation to `/org.matrix.simplified_msc3575/sync` since our simplified API is slightly incompatible with what's in the current MSC. ([\#17331](https://github.com/element-hq/synapse/issues/17331)) -- Handle device lists notifications for large accounts more efficiently in worker mode. ([\#17333](https://github.com/element-hq/synapse/issues/17333), [\#17358](https://github.com/element-hq/synapse/issues/17358)) -- Do not block event sending/receiving while calculating large event auth chains. ([\#17338](https://github.com/element-hq/synapse/issues/17338)) -- Tidy up `parse_integer` docs and call sites to reflect the fact that they require non-negative integers by default, and bring `parse_integer_from_args` default in alignment. Contributed by Denis Kasak (@dkasak). ([\#17339](https://github.com/element-hq/synapse/issues/17339)) - - - -### Updates to locked dependencies - -* Bump authlib from 1.3.0 to 1.3.1. ([\#17343](https://github.com/element-hq/synapse/issues/17343)) -* Bump dawidd6/action-download-artifact from 3.1.4 to 5. ([\#17289](https://github.com/element-hq/synapse/issues/17289)) -* Bump dawidd6/action-download-artifact from 5 to 6. ([\#17313](https://github.com/element-hq/synapse/issues/17313)) -* Bump docker/build-push-action from 5 to 6. ([\#17312](https://github.com/element-hq/synapse/issues/17312)) -* Bump jinja2 from 3.1.3 to 3.1.4. ([\#17287](https://github.com/element-hq/synapse/issues/17287)) -* Bump lazy_static from 1.4.0 to 1.5.0. ([\#17355](https://github.com/element-hq/synapse/issues/17355)) -* Bump msgpack from 1.0.7 to 1.0.8. ([\#17317](https://github.com/element-hq/synapse/issues/17317)) -* Bump netaddr from 1.2.1 to 1.3.0. ([\#17353](https://github.com/element-hq/synapse/issues/17353)) -* Bump packaging from 24.0 to 24.1. ([\#17352](https://github.com/element-hq/synapse/issues/17352)) -* Bump phonenumbers from 8.13.37 to 8.13.39. ([\#17315](https://github.com/element-hq/synapse/issues/17315)) -* Bump regex from 1.10.4 to 1.10.5. ([\#17290](https://github.com/element-hq/synapse/issues/17290)) -* Bump requests from 2.31.0 to 2.32.2. ([\#17345](https://github.com/element-hq/synapse/issues/17345)) -* Bump sentry-sdk from 2.1.1 to 2.3.1. ([\#17263](https://github.com/element-hq/synapse/issues/17263)) -* Bump sentry-sdk from 2.3.1 to 2.6.0. ([\#17351](https://github.com/element-hq/synapse/issues/17351)) -* Bump tornado from 6.4 to 6.4.1. ([\#17344](https://github.com/element-hq/synapse/issues/17344)) -* Bump mypy from 1.8.0 to 1.9.0. ([\#17297](https://github.com/element-hq/synapse/issues/17297)) -* Bump types-jsonschema from 4.21.0.20240311 to 4.22.0.20240610. ([\#17288](https://github.com/element-hq/synapse/issues/17288)) -* Bump types-netaddr from 1.2.0.20240219 to 1.3.0.20240530. ([\#17314](https://github.com/element-hq/synapse/issues/17314)) -* Bump types-pillow from 10.2.0.20240423 to 10.2.0.20240520. ([\#17285](https://github.com/element-hq/synapse/issues/17285)) -* Bump types-pyyaml from 6.0.12.12 to 6.0.12.20240311. ([\#17316](https://github.com/element-hq/synapse/issues/17316)) -* Bump typing-extensions from 4.11.0 to 4.12.2. ([\#17354](https://github.com/element-hq/synapse/issues/17354)) -* Bump urllib3 from 2.0.7 to 2.2.2. ([\#17346](https://github.com/element-hq/synapse/issues/17346)) - -# Synapse 1.109.0 (2024-06-18) - -### Internal Changes - -- Fix the building of binary wheels for macOS by switching to macOS 12 CI runners. ([\#17319](https://github.com/element-hq/synapse/issues/17319)) - - - - -# Synapse 1.109.0rc3 (2024-06-17) - -### Bugfixes - -- When rolling back to a previous Synapse version and then forwards again to this release, don't require server operators to manually run SQL. ([\#17305](https://github.com/element-hq/synapse/issues/17305), [\#17309](https://github.com/element-hq/synapse/issues/17309)) - -### Internal Changes - -- Use the release branch for sytest in release-branch PRs. ([\#17306](https://github.com/element-hq/synapse/issues/17306)) - - - - -# Synapse 1.109.0rc2 (2024-06-11) - -### Bugfixes - -- Fix bug where one-time-keys were not always included in `/sync` response when using workers. Introduced in v1.109.0rc1. ([\#17275](https://github.com/element-hq/synapse/issues/17275)) -- Fix bug where `/sync` could get stuck due to edge case in device lists handling. Introduced in v1.109.0rc1. ([\#17292](https://github.com/element-hq/synapse/issues/17292)) - - - - -# Synapse 1.109.0rc1 (2024-06-04) - -### Features - -- Add the ability to auto-accept invites on the behalf of users. See the [`auto_accept_invites`](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#auto-accept-invites) config option for details. ([\#17147](https://github.com/element-hq/synapse/issues/17147)) -- Add experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync/e2ee` endpoint for to-device messages and device encryption info. ([\#17167](https://github.com/element-hq/synapse/issues/17167)) -- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/issues/3916) by adding unstable media endpoints to `/_matrix/client`. ([\#17213](https://github.com/element-hq/synapse/issues/17213)) -- Add logging to tasks managed by the task scheduler, showing CPU and database usage. ([\#17219](https://github.com/element-hq/synapse/issues/17219)) - -### Bugfixes - -- Fix deduplicating of membership events to not create unused state groups. ([\#17164](https://github.com/element-hq/synapse/issues/17164)) -- Fix bug where duplicate events could be sent down sync when using workers that are overloaded. ([\#17215](https://github.com/element-hq/synapse/issues/17215)) -- Ignore attempts to send to-device messages to bad users, to avoid log spam when we try to connect to the bad server. ([\#17240](https://github.com/element-hq/synapse/issues/17240)) -- Fix handling of duplicate concurrent uploading of device one-time-keys. ([\#17241](https://github.com/element-hq/synapse/issues/17241)) -- Fix reporting of default tags to Sentry, such as worker name. Broke in v1.108.0. ([\#17251](https://github.com/element-hq/synapse/issues/17251)) -- Fix bug where typing updates would not be sent when using workers after a restart. ([\#17252](https://github.com/element-hq/synapse/issues/17252)) - -### Improved Documentation - -- Update the LemonLDAP documentation to say that claims should be explicitly included in the returned `id_token`, as Synapse won't request them. ([\#17204](https://github.com/element-hq/synapse/issues/17204)) - -### Internal Changes - -- Improve DB usage when fetching related events. ([\#17083](https://github.com/element-hq/synapse/issues/17083)) -- Log exceptions when failing to auto-join new user according to the `auto_join_rooms` option. ([\#17176](https://github.com/element-hq/synapse/issues/17176)) -- Reduce work of calculating outbound device lists updates. ([\#17211](https://github.com/element-hq/synapse/issues/17211)) -- Improve performance of calculating device lists changes in `/sync`. ([\#17216](https://github.com/element-hq/synapse/issues/17216)) -- Move towards using `MultiWriterIdGenerator` everywhere. ([\#17226](https://github.com/element-hq/synapse/issues/17226)) -- Replaces all usages of `StreamIdGenerator` with `MultiWriterIdGenerator`. ([\#17229](https://github.com/element-hq/synapse/issues/17229)) -- Change the `allow_unsafe_locale` config option to also apply when setting up new databases. ([\#17238](https://github.com/element-hq/synapse/issues/17238)) -- Fix errors in logs about closing incorrect logging contexts when media gets rejected by a module. ([\#17239](https://github.com/element-hq/synapse/issues/17239), [\#17246](https://github.com/element-hq/synapse/issues/17246)) -- Clean out invalid destinations from `device_federation_outbox` table. ([\#17242](https://github.com/element-hq/synapse/issues/17242)) -- Stop logging errors when receiving invalid User IDs in key querys requests. ([\#17250](https://github.com/element-hq/synapse/issues/17250)) - - - -### Updates to locked dependencies - -* Bump anyhow from 1.0.83 to 1.0.86. ([\#17220](https://github.com/element-hq/synapse/issues/17220)) -* Bump bcrypt from 4.1.2 to 4.1.3. ([\#17224](https://github.com/element-hq/synapse/issues/17224)) -* Bump lxml from 5.2.1 to 5.2.2. ([\#17261](https://github.com/element-hq/synapse/issues/17261)) -* Bump mypy-zope from 1.0.3 to 1.0.4. ([\#17262](https://github.com/element-hq/synapse/issues/17262)) -* Bump phonenumbers from 8.13.35 to 8.13.37. ([\#17235](https://github.com/element-hq/synapse/issues/17235)) -* Bump prometheus-client from 0.19.0 to 0.20.0. ([\#17233](https://github.com/element-hq/synapse/issues/17233)) -* Bump pyasn1 from 0.5.1 to 0.6.0. ([\#17223](https://github.com/element-hq/synapse/issues/17223)) -* Bump pyicu from 2.13 to 2.13.1. ([\#17236](https://github.com/element-hq/synapse/issues/17236)) -* Bump pyopenssl from 24.0.0 to 24.1.0. ([\#17234](https://github.com/element-hq/synapse/issues/17234)) -* Bump serde from 1.0.201 to 1.0.202. ([\#17221](https://github.com/element-hq/synapse/issues/17221)) -* Bump serde from 1.0.202 to 1.0.203. ([\#17232](https://github.com/element-hq/synapse/issues/17232)) -* Bump twine from 5.0.0 to 5.1.0. ([\#17225](https://github.com/element-hq/synapse/issues/17225)) -* Bump types-psycopg2 from 2.9.21.20240311 to 2.9.21.20240417. ([\#17222](https://github.com/element-hq/synapse/issues/17222)) -* Bump types-pyopenssl from 24.0.0.20240311 to 24.1.0.20240425. ([\#17260](https://github.com/element-hq/synapse/issues/17260)) - -# Synapse 1.108.0 (2024-05-28) - -No significant changes since 1.108.0rc1. - - - - -# Synapse 1.108.0rc1 (2024-05-21) - -### Features - -- Add a feature that allows clients to query the configured federation whitelist. Disabled by default. ([\#16848](https://github.com/element-hq/synapse/issues/16848), [\#17199](https://github.com/element-hq/synapse/issues/17199)) -- Add the ability to allow numeric user IDs with a specific prefix when in the CAS flow. Contributed by Aurélien Grimpard. ([\#17098](https://github.com/element-hq/synapse/issues/17098)) - -### Bugfixes - -- Fix bug where push rules would be empty in `/sync` for some accounts. Introduced in v1.93.0. ([\#17142](https://github.com/element-hq/synapse/issues/17142)) -- Add support for optional whitespace around the Federation API's `Authorization` header's parameter commas. ([\#17145](https://github.com/element-hq/synapse/issues/17145)) -- Fix bug where disabling room publication prevented public rooms being created on workers. ([\#17177](https://github.com/element-hq/synapse/issues/17177), [\#17184](https://github.com/element-hq/synapse/issues/17184)) - -### Improved Documentation - -- Document [`/v1/make_knock`](https://spec.matrix.org/v1.10/server-server-api/#get_matrixfederationv1make_knockroomiduserid) and [`/v1/send_knock/`](https://spec.matrix.org/v1.10/server-server-api/#put_matrixfederationv1send_knockroomideventid) federation endpoints as worker-compatible. ([\#17058](https://github.com/element-hq/synapse/issues/17058)) -- Update User Admin API with note about prefixing OIDC external_id providers. ([\#17139](https://github.com/element-hq/synapse/issues/17139)) -- Clarify the state of the created room when using the `autocreate_auto_join_room_preset` config option. ([\#17150](https://github.com/element-hq/synapse/issues/17150)) -- Update the Admin FAQ with the current libjemalloc version for latest Debian stable. Additionally update the name of the "push_rules" stream in the Workers documentation. ([\#17171](https://github.com/element-hq/synapse/issues/17171)) - -### Internal Changes - -- Add note to reflect that [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) is closed but will remain supported for some time. ([\#17151](https://github.com/element-hq/synapse/issues/17151)) -- Update dependency PyO3 to 0.21. ([\#17162](https://github.com/element-hq/synapse/issues/17162)) -- Fixes linter errors found in PR #17147. ([\#17166](https://github.com/element-hq/synapse/issues/17166)) -- Bump black from 24.2.0 to 24.4.2. ([\#17170](https://github.com/element-hq/synapse/issues/17170)) -- Cache literal sync filter validation for performance. ([\#17186](https://github.com/element-hq/synapse/issues/17186)) -- Improve performance by fixing a reactor pause. ([\#17192](https://github.com/element-hq/synapse/issues/17192)) -- Route `/make_knock` and `/send_knock` federation APIs to the federation reader worker in Complement test runs. ([\#17195](https://github.com/element-hq/synapse/issues/17195)) -- Prepare sync handler to be able to return different sync responses (`SyncVersion`). ([\#17200](https://github.com/element-hq/synapse/issues/17200)) -- Organize the sync cache key parameter outside of the sync config (separate concerns). ([\#17201](https://github.com/element-hq/synapse/issues/17201)) -- Refactor `SyncResultBuilder` assembly to its own function. ([\#17202](https://github.com/element-hq/synapse/issues/17202)) -- Rename to be obvious: `joined_rooms` -> `joined_room_ids`. ([\#17203](https://github.com/element-hq/synapse/issues/17203), [\#17208](https://github.com/element-hq/synapse/issues/17208)) -- Add a short pause when rate-limiting a request. ([\#17210](https://github.com/element-hq/synapse/issues/17210)) - - - -### Updates to locked dependencies - -* Bump cryptography from 42.0.5 to 42.0.7. ([\#17180](https://github.com/element-hq/synapse/issues/17180)) -* Bump gitpython from 3.1.41 to 3.1.43. ([\#17181](https://github.com/element-hq/synapse/issues/17181)) -* Bump immutabledict from 4.1.0 to 4.2.0. ([\#17179](https://github.com/element-hq/synapse/issues/17179)) -* Bump sentry-sdk from 1.40.3 to 2.1.1. ([\#17178](https://github.com/element-hq/synapse/issues/17178)) -* Bump serde from 1.0.200 to 1.0.201. ([\#17183](https://github.com/element-hq/synapse/issues/17183)) -* Bump serde_json from 1.0.116 to 1.0.117. ([\#17182](https://github.com/element-hq/synapse/issues/17182)) - -Synapse 1.107.0 (2024-05-14) -============================ - -No significant changes since 1.107.0rc1. - - -# Synapse 1.107.0rc1 (2024-05-07) - -### Features - -- Add preliminary support for [MSC3823: Account Suspension](https://github.com/matrix-org/matrix-spec-proposals/pull/3823). ([\#17051](https://github.com/element-hq/synapse/issues/17051)) -- Declare support for [Matrix v1.10](https://matrix.org/blog/2024/03/22/matrix-v1.10-release/). Contributed by @clokep. ([\#17082](https://github.com/element-hq/synapse/issues/17082)) -- Add support for [MSC4115: membership metadata on events](https://github.com/matrix-org/matrix-spec-proposals/pull/4115). ([\#17104](https://github.com/element-hq/synapse/issues/17104), [\#17137](https://github.com/element-hq/synapse/issues/17137)) - -### Bugfixes - -- Fixed search feature of Element Android on homesevers using SQLite by returning search terms as search highlights. ([\#17000](https://github.com/element-hq/synapse/issues/17000)) -- Fixes a bug introduced in v1.52.0 where the `destination` query parameter for the [Destination Rooms Admin API](https://element-hq.github.io/synapse/v1.105/usage/administration/admin_api/federation.html#destination-rooms) failed to actually filter returned rooms. ([\#17077](https://github.com/element-hq/synapse/issues/17077)) -- For MSC3266 room summaries, support queries at the recommended endpoint of `/_matrix/client/unstable/im.nheko.summary/summary/{roomIdOrAlias}`. The existing endpoint of `/_matrix/client/unstable/im.nheko.summary/rooms/{roomIdOrAlias}/summary` is deprecated. ([\#17078](https://github.com/element-hq/synapse/issues/17078)) -- Apply user email & picture during OIDC registration if present & selected. ([\#17120](https://github.com/element-hq/synapse/issues/17120)) -- Improve error message for cross signing reset with [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) enabled. ([\#17121](https://github.com/element-hq/synapse/issues/17121)) -- Fix a bug which meant that to-device messages received over federation could be dropped when the server was under load or networking problems caused problems between Synapse processes or the database. ([\#17127](https://github.com/element-hq/synapse/issues/17127)) -- Fix bug where `StreamChangeCache` would not respect configured cache factors. ([\#17152](https://github.com/element-hq/synapse/issues/17152)) +- Fix a longstanding bug where Synapse would immediately retry a failing push endpoint when a new event is received, ignoring any backoff timers. ([\#18363](https://github.com/element-hq/synapse/issues/18363)) +- Pass leave from remote invite rejection down Sliding Sync. ([\#18375](https://github.com/element-hq/synapse/issues/18375)) ### Updates to the Docker image -- Correct licensing metadata on Docker image. ([\#17141](https://github.com/element-hq/synapse/issues/17141)) +- In `configure_workers_and_start.py`, use the same absolute path of Python in the interpreter shebang, and invoke child Python processes with `sys.executable`. ([\#18291](https://github.com/element-hq/synapse/issues/18291)) +- Optimize the build of the workers image. ([\#18292](https://github.com/element-hq/synapse/issues/18292)) +- In `start_for_complement.sh`, replace some external program calls with shell builtins. ([\#18293](https://github.com/element-hq/synapse/issues/18293)) +- When generating container scripts from templates, don't add a leading newline so that their shebangs may be handled correctly. ([\#18295](https://github.com/element-hq/synapse/issues/18295)) ### Improved Documentation -- Update the `event_cache_size` and `global_factor` configuration options' documentation. ([\#17071](https://github.com/element-hq/synapse/issues/17071)) -- Remove broken sphinx docs. ([\#17073](https://github.com/element-hq/synapse/issues/17073), [\#17148](https://github.com/element-hq/synapse/issues/17148)) -- Add RuntimeDirectory to example matrix-synapse.service systemd unit. ([\#17084](https://github.com/element-hq/synapse/issues/17084)) -- Fix various small typos throughout the docs. ([\#17114](https://github.com/element-hq/synapse/issues/17114)) -- Update enable_notifs configuration documentation. ([\#17116](https://github.com/element-hq/synapse/issues/17116)) -- Update the Upgrade Notes with the latest minimum supported Rust version of 1.66.0. Contributed by @jahway603. ([\#17140](https://github.com/element-hq/synapse/issues/17140)) +- Improve formatting of the README file. ([\#18218](https://github.com/element-hq/synapse/issues/18218)) +- Add documentation for configuring [Pocket ID](https://github.com/pocket-id/pocket-id) as an OIDC provider. ([\#18237](https://github.com/element-hq/synapse/issues/18237)) +- Fix typo in docs about the `push` config option. Contributed by @HarHarLinks. ([\#18320](https://github.com/element-hq/synapse/issues/18320)) +- Add `/_matrix/federation/v1/version` to list of federation endpoints that can be handled by workers. ([\#18377](https://github.com/element-hq/synapse/issues/18377)) +- Add an Admin API endpoint `GET /_synapse/admin/v1/scheduled_tasks` to fetch scheduled tasks. ([\#18384](https://github.com/element-hq/synapse/issues/18384)) ### Internal Changes -- Enable [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) by default in the Synapse Complement image. ([\#17105](https://github.com/element-hq/synapse/issues/17105)) -- Add optimisation to `StreamChangeCache.get_entities_changed(..)`. ([\#17130](https://github.com/element-hq/synapse/issues/17130)) +- Return specific error code when adding an email address / phone number to account is not supported ([MSC4178](https://github.com/matrix-org/matrix-spec-proposals/pull/4178)). ([\#17578](https://github.com/element-hq/synapse/issues/17578)) +- Stop auto-provisionning missing users & devices when delegating auth to Matrix Authentication Service. Requires MAS 0.13.0 or later. ([\#18181](https://github.com/element-hq/synapse/issues/18181)) +- Apply file hashing and existing quarantines to media downloaded for URL previews. ([\#18297](https://github.com/element-hq/synapse/issues/18297)) +- Allow a few admin APIs used by matrix-authentication-service to run on workers. ([\#18313](https://github.com/element-hq/synapse/issues/18313)) +- Apply `should_drop_federated_event` to federation invites. ([\#18330](https://github.com/element-hq/synapse/issues/18330)) +- Allow `/rooms/` admin API to be run on workers. ([\#18360](https://github.com/element-hq/synapse/issues/18360)) +- Minor performance improvements to the notifier. ([\#18367](https://github.com/element-hq/synapse/issues/18367)) +- Slight performance increase when using the ratelimiter. ([\#18369](https://github.com/element-hq/synapse/issues/18369)) +- Don't validate the `at_hash` (access token hash) field in OIDC ID Tokens if we don't end up actually using the OIDC Access Token. ([\#18374](https://github.com/element-hq/synapse/issues/18374), [\#18385](https://github.com/element-hq/synapse/issues/18385)) +- Fixed test failures when using authlib 1.5.2. ([\#18390](https://github.com/element-hq/synapse/issues/18390)) +- Refactor [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Simplified Sliding Sync room list tests to cover both new and fallback logic paths. ([\#18399](https://github.com/element-hq/synapse/issues/18399)) ### Updates to locked dependencies -* Bump furo from 2024.1.29 to 2024.4.27. ([\#17133](https://github.com/element-hq/synapse/issues/17133)) -* Bump idna from 3.6 to 3.7. ([\#17136](https://github.com/element-hq/synapse/issues/17136)) -* Bump jsonschema from 4.21.1 to 4.22.0. ([\#17157](https://github.com/element-hq/synapse/issues/17157)) -* Bump lxml from 5.1.0 to 5.2.1. ([\#17158](https://github.com/element-hq/synapse/issues/17158)) -* Bump phonenumbers from 8.13.29 to 8.13.35. ([\#17106](https://github.com/element-hq/synapse/issues/17106)) -- Bump pillow from 10.2.0 to 10.3.0. ([\#17146](https://github.com/element-hq/synapse/issues/17146)) -* Bump pydantic from 2.6.4 to 2.7.0. ([\#17107](https://github.com/element-hq/synapse/issues/17107)) -* Bump pydantic from 2.7.0 to 2.7.1. ([\#17160](https://github.com/element-hq/synapse/issues/17160)) -* Bump pyicu from 2.12 to 2.13. ([\#17109](https://github.com/element-hq/synapse/issues/17109)) -* Bump serde from 1.0.197 to 1.0.198. ([\#17111](https://github.com/element-hq/synapse/issues/17111)) -* Bump serde from 1.0.198 to 1.0.199. ([\#17132](https://github.com/element-hq/synapse/issues/17132)) -* Bump serde from 1.0.199 to 1.0.200. ([\#17161](https://github.com/element-hq/synapse/issues/17161)) -* Bump serde_json from 1.0.115 to 1.0.116. ([\#17112](https://github.com/element-hq/synapse/issues/17112)) -- Update `tornado` Python dependency from 6.2 to 6.4. ([\#17131](https://github.com/element-hq/synapse/issues/17131)) -* Bump twisted from 23.10.0 to 24.3.0. ([\#17135](https://github.com/element-hq/synapse/issues/17135)) -* Bump types-bleach from 6.1.0.1 to 6.1.0.20240331. ([\#17110](https://github.com/element-hq/synapse/issues/17110)) -* Bump types-pillow from 10.2.0.20240415 to 10.2.0.20240423. ([\#17159](https://github.com/element-hq/synapse/issues/17159)) -* Bump types-setuptools from 69.0.0.20240125 to 69.5.0.20240423. ([\#17134](https://github.com/element-hq/synapse/issues/17134)) +* Bump actions/add-to-project from 280af8ae1f83a494cfad2cb10f02f6d13529caa9 to 5b1a254a3546aef88e0a7724a77a623fa2e47c36. ([\#18365](https://github.com/element-hq/synapse/issues/18365)) +* Bump actions/download-artifact from 4.2.1 to 4.3.0. ([\#18364](https://github.com/element-hq/synapse/issues/18364)) +* Bump actions/setup-go from 5.4.0 to 5.5.0. ([\#18426](https://github.com/element-hq/synapse/issues/18426)) +* Bump anyhow from 1.0.97 to 1.0.98. ([\#18336](https://github.com/element-hq/synapse/issues/18336)) +* Bump packaging from 24.2 to 25.0. ([\#18393](https://github.com/element-hq/synapse/issues/18393)) +* Bump pillow from 11.1.0 to 11.2.1. ([\#18429](https://github.com/element-hq/synapse/issues/18429)) +* Bump pydantic from 2.10.3 to 2.11.4. ([\#18394](https://github.com/element-hq/synapse/issues/18394)) +* Bump pyo3-log from 0.12.2 to 0.12.3. ([\#18317](https://github.com/element-hq/synapse/issues/18317)) +* Bump pyopenssl from 24.3.0 to 25.0.0. ([\#18315](https://github.com/element-hq/synapse/issues/18315)) +* Bump sha2 from 0.10.8 to 0.10.9. ([\#18395](https://github.com/element-hq/synapse/issues/18395)) +* Bump sigstore/cosign-installer from 3.8.1 to 3.8.2. ([\#18366](https://github.com/element-hq/synapse/issues/18366)) +* Bump softprops/action-gh-release from 1 to 2. ([\#18264](https://github.com/element-hq/synapse/issues/18264)) +* Bump stefanzweifel/git-auto-commit-action from 5.1.0 to 5.2.0. ([\#18354](https://github.com/element-hq/synapse/issues/18354)) +* Bump txredisapi from 1.4.10 to 1.4.11. ([\#18392](https://github.com/element-hq/synapse/issues/18392)) +* Bump types-jsonschema from 4.23.0.20240813 to 4.23.0.20241208. ([\#18305](https://github.com/element-hq/synapse/issues/18305)) +* Bump types-psycopg2 from 2.9.21.20250121 to 2.9.21.20250318. ([\#18316](https://github.com/element-hq/synapse/issues/18316)) -# Synapse 1.106.0 (2024-04-30) +# Synapse 1.129.0 (2025-05-06) -No significant changes since 1.106.0rc1. +No significant changes since 1.129.0rc2. -# Synapse 1.106.0rc1 (2024-04-25) +# Synapse 1.129.0rc2 (2025-04-30) -### Features - -- Send an email if the address is already bound to an user account. ([\#16819](https://github.com/element-hq/synapse/issues/16819)) -- Implement the rendezvous mechanism described by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/issues/4108). ([\#17056](https://github.com/element-hq/synapse/issues/17056)) -- Support delegating the rendezvous mechanism described [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/issues/4108) to an external implementation. ([\#17086](https://github.com/element-hq/synapse/issues/17086)) - -### Bugfixes - -- Add validation to ensure that the `limit` parameter on `/publicRooms` is non-negative. ([\#16920](https://github.com/element-hq/synapse/issues/16920)) -- Return `400 M_NOT_JSON` upon receiving invalid JSON in query parameters across various client and admin endpoints, rather than an internal server error. ([\#16923](https://github.com/element-hq/synapse/issues/16923)) -- Make the CSAPI endpoint `/keys/device_signing/upload` idempotent. ([\#16943](https://github.com/element-hq/synapse/issues/16943)) -- Redact membership events if the user requested erasure upon deactivating. ([\#17076](https://github.com/element-hq/synapse/issues/17076)) - -### Improved Documentation - -- Add a prompt in the contributing guide to manually configure icu4c. ([\#17069](https://github.com/element-hq/synapse/issues/17069)) -- Clarify what part of message retention is still experimental. ([\#17099](https://github.com/element-hq/synapse/issues/17099)) +Synapse 1.129.0rc1 was never formally released due to regressions discovered during the release process. 1.129.0rc2 fixes those regressions by reverting the affected PRs. ### Internal Changes -- Use new receipts column to optimise receipt and push action SQL queries. Contributed by Nick @ Beeper (@fizzadar). ([\#17032](https://github.com/element-hq/synapse/issues/17032), [\#17096](https://github.com/element-hq/synapse/issues/17096)) -- Fix mypy with latest Twisted release. ([\#17036](https://github.com/element-hq/synapse/issues/17036)) -- Bump minimum supported Rust version to 1.66.0. ([\#17079](https://github.com/element-hq/synapse/issues/17079)) -- Add helpers to transform Twisted requests to Rust http Requests/Responses. ([\#17081](https://github.com/element-hq/synapse/issues/17081)) -- Fix type annotation for `visited_chains` after `mypy` upgrade. ([\#17125](https://github.com/element-hq/synapse/issues/17125)) - - - -### Updates to locked dependencies - -* Bump anyhow from 1.0.81 to 1.0.82. ([\#17095](https://github.com/element-hq/synapse/issues/17095)) -* Bump peaceiris/actions-gh-pages from 3.9.3 to 4.0.0. ([\#17087](https://github.com/element-hq/synapse/issues/17087)) -* Bump peaceiris/actions-mdbook from 1.2.0 to 2.0.0. ([\#17089](https://github.com/element-hq/synapse/issues/17089)) -* Bump pyasn1-modules from 0.3.0 to 0.4.0. ([\#17093](https://github.com/element-hq/synapse/issues/17093)) -* Bump pygithub from 2.2.0 to 2.3.0. ([\#17092](https://github.com/element-hq/synapse/issues/17092)) -* Bump ruff from 0.3.5 to 0.3.7. ([\#17094](https://github.com/element-hq/synapse/issues/17094)) -* Bump sigstore/cosign-installer from 3.4.0 to 3.5.0. ([\#17088](https://github.com/element-hq/synapse/issues/17088)) -* Bump twine from 4.0.2 to 5.0.0. ([\#17091](https://github.com/element-hq/synapse/issues/17091)) -* Bump types-pillow from 10.2.0.20240406 to 10.2.0.20240415. ([\#17090](https://github.com/element-hq/synapse/issues/17090)) - -# Synapse 1.105.1 (2024-04-23) - -## Security advisory - -The following issues are fixed in 1.105.1. - -- [GHSA-3h7q-rfh9-xm4v](https://github.com/element-hq/synapse/security/advisories/GHSA-3h7q-rfh9-xm4v) / [CVE-2024-31208](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-31208) — High Severity - - Weakness in auth chain indexing allows DoS from remote room members through disk fill and high CPU usage. - -See the advisories for more details. If you have any questions, email security@element.io. - - - -# Synapse 1.105.0 (2024-04-16) - -No significant changes since 1.105.0rc1. +- Revert the slow background update introduced by [\#18068](https://github.com/element-hq/synapse/issues/18068) in v1.128.0. ([\#18372](https://github.com/element-hq/synapse/issues/18372)) +- Revert "Add total event, unencrypted message, and e2ee event counts to stats reporting", added in v1.129.0rc1. ([\#18373](https://github.com/element-hq/synapse/issues/18373)) -# Synapse 1.105.0rc1 (2024-04-11) +# Synapse 1.129.0rc1 (2025-04-15) ### Features -- Stabilize support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which clarifies the interaction of push rules and account data. Contributed by @clokep. ([\#17022](https://github.com/element-hq/synapse/issues/17022)) -- Stabilize support for [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981): `/relations` recursion. Contributed by @clokep. ([\#17023](https://github.com/element-hq/synapse/issues/17023)) -- Add support for moving `/pushrules` off of main process. ([\#17037](https://github.com/element-hq/synapse/issues/17037), [\#17038](https://github.com/element-hq/synapse/issues/17038)) +- Add `passthrough_authorization_parameters` in OIDC configuration to allow passing parameters to the authorization grant URL. ([\#18232](https://github.com/element-hq/synapse/issues/18232)) +- Add `total_event_count`, `total_message_count`, and `total_e2ee_event_count` fields to the homeserver usage statistics. ([\#18260](https://github.com/element-hq/synapse/issues/18260)) ### Bugfixes -- Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16930](https://github.com/element-hq/synapse/issues/16930), [\#16932](https://github.com/element-hq/synapse/issues/16932), [\#16942](https://github.com/element-hq/synapse/issues/16942), [\#17064](https://github.com/element-hq/synapse/issues/17064), [\#17065](https://github.com/element-hq/synapse/issues/17065), [\#17066](https://github.com/element-hq/synapse/issues/17066)) -- Fix server notice rooms not always being created as unencrypted rooms, even when `encryption_enabled_by_default_for_room_type` is in use (server notices are always unencrypted). ([\#17033](https://github.com/element-hq/synapse/issues/17033)) -- Fix the `.m.rule.encrypted_room_one_to_one` and `.m.rule.room_one_to_one` default underride push rules being in the wrong order. Contributed by @Sumpy1. ([\#17043](https://github.com/element-hq/synapse/issues/17043)) - -### Internal Changes - -- Refactor auth chain fetching to reduce duplication. ([\#17044](https://github.com/element-hq/synapse/issues/17044)) -- Improve database performance by adding a missing index to `access_tokens.refresh_token_id`. ([\#17045](https://github.com/element-hq/synapse/issues/17045), [\#17054](https://github.com/element-hq/synapse/issues/17054)) -- Improve database performance by reducing number of receipts fetched when sending push notifications. ([\#17049](https://github.com/element-hq/synapse/issues/17049)) - - - -### Updates to locked dependencies - -* Bump packaging from 23.2 to 24.0. ([\#17027](https://github.com/element-hq/synapse/issues/17027)) -* Bump regex from 1.10.3 to 1.10.4. ([\#17028](https://github.com/element-hq/synapse/issues/17028)) -* Bump ruff from 0.3.2 to 0.3.5. ([\#17060](https://github.com/element-hq/synapse/issues/17060)) -* Bump serde_json from 1.0.114 to 1.0.115. ([\#17041](https://github.com/element-hq/synapse/issues/17041)) -* Bump types-pillow from 10.2.0.20240125 to 10.2.0.20240406. ([\#17061](https://github.com/element-hq/synapse/issues/17061)) -* Bump types-requests from 2.31.0.20240125 to 2.31.0.20240406. ([\#17063](https://github.com/element-hq/synapse/issues/17063)) -* Bump typing-extensions from 4.9.0 to 4.11.0. ([\#17062](https://github.com/element-hq/synapse/issues/17062)) - -# Synapse 1.104.0 (2024-04-02) - -### Bugfixes - -- Fix regression when using OIDC provider. Introduced in v1.104.0rc1. ([\#17031](https://github.com/element-hq/synapse/issues/17031)) - - -# Synapse 1.104.0rc1 (2024-03-26) - -### Features - -- Add an OIDC config to specify extra parameters for the authorization grant URL. IT can be useful to pass an ACR value for example. ([\#16971](https://github.com/element-hq/synapse/issues/16971)) -- Add support for OIDC provider returning JWT. ([\#16972](https://github.com/element-hq/synapse/issues/16972), [\#17031](https://github.com/element-hq/synapse/issues/17031)) - -### Bugfixes - -- Fix a bug which meant that, under certain circumstances, we might never retry sending events or to-device messages over federation after a failure. ([\#16925](https://github.com/element-hq/synapse/issues/16925)) -- Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16949](https://github.com/element-hq/synapse/issues/16949)) -- Fix case in which `m.fully_read` marker would not get updated. Contributed by @SpiritCroc. ([\#16990](https://github.com/element-hq/synapse/issues/16990)) -- Fix bug which did not retract a user's pending knocks at rooms when their account was deactivated. Contributed by @hanadi92. ([\#17010](https://github.com/element-hq/synapse/issues/17010)) +- Fix `force_tracing_for_users` config when using delegated auth. ([\#18334](https://github.com/element-hq/synapse/issues/18334)) +- Fix the token introspection cache logging access tokens when MAS integration is in use. ([\#18335](https://github.com/element-hq/synapse/issues/18335)) +- Stop caching introspection failures when delegating auth to MAS. ([\#18339](https://github.com/element-hq/synapse/issues/18339)) +- Fix `ExternalIDReuse` exception after migrating to MAS on workers with a high traffic. ([\#18342](https://github.com/element-hq/synapse/issues/18342)) +- Fix minor performance regression caused by tracking of room participation. Regressed in v1.128.0. ([\#18345](https://github.com/element-hq/synapse/issues/18345)) ### Updates to the Docker image -- Updated `start.py` to generate config using the correct user ID when running as root (fixes [\#16824](https://github.com/element-hq/synapse/issues/16824), [\#15202](https://github.com/element-hq/synapse/issues/15202)). ([\#16978](https://github.com/element-hq/synapse/issues/16978)) - -### Improved Documentation - -- Add a query to force a refresh of a remote user's device list to the "Useful SQL for Admins" documentation page. ([\#16892](https://github.com/element-hq/synapse/issues/16892)) -- Minor grammatical corrections to the upgrade documentation. ([\#16965](https://github.com/element-hq/synapse/issues/16965)) -- Fix the sort order for the documentation version picker, so that newer releases appear above older ones. ([\#16966](https://github.com/element-hq/synapse/issues/16966)) -- Remove recommendation for a specific poetry version from contributing guide. ([\#17002](https://github.com/element-hq/synapse/issues/17002)) +- Optimize the build of the complement-synapse image. ([\#18294](https://github.com/element-hq/synapse/issues/18294)) ### Internal Changes -- Improve lock performance when a lot of locks are all waiting for a single lock to be released. ([\#16840](https://github.com/element-hq/synapse/issues/16840)) -- Update power level default for public rooms. ([\#16907](https://github.com/element-hq/synapse/issues/16907)) -- Improve event validation. ([\#16908](https://github.com/element-hq/synapse/issues/16908)) -- Multi-worker-docker-container: disable log buffering. ([\#16919](https://github.com/element-hq/synapse/issues/16919)) -- Refactor state delta calculation in `/sync` handler. ([\#16929](https://github.com/element-hq/synapse/issues/16929)) -- Clarify docs for some room state functions. ([\#16950](https://github.com/element-hq/synapse/issues/16950)) -- Specify IP subnets in canonical form. ([\#16953](https://github.com/element-hq/synapse/issues/16953)) -- As done for SAML mapping provider, let's pass the module API to the OIDC one so the mapper can do more logic in its code. ([\#16974](https://github.com/element-hq/synapse/issues/16974)) -- Allow containers building on top of Synapse's Complement container is use the included PostgreSQL cluster. ([\#16985](https://github.com/element-hq/synapse/issues/16985)) -- Raise poetry-core version cap to 1.9.0. ([\#16986](https://github.com/element-hq/synapse/issues/16986)) -- Patch the db conn pool sooner in tests. ([\#17017](https://github.com/element-hq/synapse/issues/17017)) - - - -### Updates to locked dependencies - -* Bump anyhow from 1.0.80 to 1.0.81. ([\#17009](https://github.com/element-hq/synapse/issues/17009)) -* Bump black from 23.10.1 to 24.2.0. ([\#16936](https://github.com/element-hq/synapse/issues/16936)) -* Bump cryptography from 41.0.7 to 42.0.5. ([\#16958](https://github.com/element-hq/synapse/issues/16958)) -* Bump dawidd6/action-download-artifact from 3.1.1 to 3.1.2. ([\#16960](https://github.com/element-hq/synapse/issues/16960)) -* Bump dawidd6/action-download-artifact from 3.1.2 to 3.1.4. ([\#17008](https://github.com/element-hq/synapse/issues/17008)) -* Bump jinja2 from 3.1.2 to 3.1.3. ([\#17005](https://github.com/element-hq/synapse/issues/17005)) -* Bump log from 0.4.20 to 0.4.21. ([\#16977](https://github.com/element-hq/synapse/issues/16977)) -* Bump mypy from 1.5.1 to 1.8.0. ([\#16901](https://github.com/element-hq/synapse/issues/16901)) -* Bump netaddr from 0.9.0 to 1.2.1. ([\#17006](https://github.com/element-hq/synapse/issues/17006)) -* Bump pydantic from 2.6.0 to 2.6.4. ([\#17004](https://github.com/element-hq/synapse/issues/17004)) -* Bump pyo3 from 0.20.2 to 0.20.3. ([\#16962](https://github.com/element-hq/synapse/issues/16962)) -* Bump ruff from 0.1.14 to 0.3.2. ([\#16994](https://github.com/element-hq/synapse/issues/16994)) -* Bump serde from 1.0.196 to 1.0.197. ([\#16963](https://github.com/element-hq/synapse/issues/16963)) -* Bump serde_json from 1.0.113 to 1.0.114. ([\#16961](https://github.com/element-hq/synapse/issues/16961)) -* Bump types-jsonschema from 4.21.0.20240118 to 4.21.0.20240311. ([\#17007](https://github.com/element-hq/synapse/issues/17007)) -* Bump types-psycopg2 from 2.9.21.16 to 2.9.21.20240311. ([\#16995](https://github.com/element-hq/synapse/issues/16995)) -* Bump types-pyopenssl from 23.3.0.0 to 24.0.0.20240311. ([\#17003](https://github.com/element-hq/synapse/issues/17003)) - -# Synapse 1.103.0 (2024-03-19) - -No significant changes since 1.103.0rc1. +- Disable statement timeout during room purge. ([\#18133](https://github.com/element-hq/synapse/issues/18133)) +- Add cache to storage functions used to auth requests when using delegated auth. ([\#18337](https://github.com/element-hq/synapse/issues/18337)) -# Synapse 1.103.0rc1 (2024-03-12) +# Synapse 1.128.0 (2025-04-08) + +No significant changes since 1.128.0rc1. + + + + +# Synapse 1.128.0rc1 (2025-04-01) ### Features -- Add a new [List Accounts v3](https://element-hq.github.io/synapse/v1.103/admin_api/user_admin_api.html#list-accounts-v3) Admin API with improved deactivated user filtering capabilities. ([\#16874](https://github.com/element-hq/synapse/issues/16874)) -- Include `Retry-After` header by default per [MSC4041](https://github.com/matrix-org/matrix-spec-proposals/pull/4041). Contributed by @clokep. ([\#16947](https://github.com/element-hq/synapse/issues/16947)) +- Add an access token introspection cache to make Matrix Authentication Service integration ([MSC3861](https://github.com/matrix-org/matrix-doc/pull/3861)) more efficient. ([\#18231](https://github.com/element-hq/synapse/issues/18231)) +- Add background job to clear unreferenced state groups. ([\#18254](https://github.com/element-hq/synapse/issues/18254)) +- Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. ([\#18277](https://github.com/element-hq/synapse/issues/18277), [\#18302](https://github.com/element-hq/synapse/issues/18302), [\#18296](https://github.com/element-hq/synapse/issues/18296)) ### Bugfixes -- Fix joining remote rooms when a module uses the `on_new_event` callback. This callback may now pass partial state events instead of the full state for remote rooms. Introduced in v1.76.0. ([\#16973](https://github.com/element-hq/synapse/issues/16973)) -- Fix performance issue when joining very large rooms that can cause the server to lock up. Introduced in v1.100.0. Contributed by @ggogel. ([\#16968](https://github.com/element-hq/synapse/issues/16968)) - -### Improved Documentation - -- Add HAProxy example for single port operation to reverse proxy documentation. Contributed by Georg Pfuetzenreuter (@tacerus). ([\#16768](https://github.com/element-hq/synapse/issues/16768)) -- Improve the documentation around running Complement tests with new configuration parameters. ([\#16946](https://github.com/element-hq/synapse/issues/16946)) -- Add docs on upgrading from a very old version. ([\#16951](https://github.com/element-hq/synapse/issues/16951)) - - -### Updates to locked dependencies - -* Bump JasonEtco/create-an-issue from 2.9.1 to 2.9.2. ([\#16934](https://github.com/element-hq/synapse/issues/16934)) -* Bump anyhow from 1.0.79 to 1.0.80. ([\#16935](https://github.com/element-hq/synapse/issues/16935)) -* Bump dawidd6/action-download-artifact from 3.0.0 to 3.1.1. ([\#16933](https://github.com/element-hq/synapse/issues/16933)) -* Bump furo from 2023.9.10 to 2024.1.29. ([\#16939](https://github.com/element-hq/synapse/issues/16939)) -* Bump pyopenssl from 23.3.0 to 24.0.0. ([\#16937](https://github.com/element-hq/synapse/issues/16937)) -* Bump types-netaddr from 0.10.0.20240106 to 1.2.0.20240219. ([\#16938](https://github.com/element-hq/synapse/issues/16938)) - - -# Synapse 1.102.0 (2024-03-05) - -### Bugfixes - -- Revert https://github.com/element-hq/synapse/pull/16756, which caused incorrect notification counts on mobile clients since v1.100.0. ([\#16979](https://github.com/element-hq/synapse/issues/16979)) - - -# Synapse 1.102.0rc1 (2024-02-20) - -### Features - -- A metric was added for emails sent by Synapse, broken down by type: `synapse_emails_sent_total`. Contributed by Remi Rampin. ([\#16881](https://github.com/element-hq/synapse/issues/16881)) - -### Bugfixes - -- Do not send multiple concurrent requests for keys for the same server. ([\#16894](https://github.com/element-hq/synapse/issues/16894)) -- Fix performance issue when joining very large rooms that can cause the server to lock up. Introduced in v1.100.0. ([\#16903](https://github.com/element-hq/synapse/issues/16903)) -- Always prefer unthreaded receipt when >1 exist ([MSC4102](https://github.com/matrix-org/matrix-spec-proposals/pull/4102)). ([\#16927](https://github.com/element-hq/synapse/issues/16927)) - -### Improved Documentation - -- Fix a small typo in the Rooms section of the Admin API documentation. Contributed by @RainerZufall187. ([\#16857](https://github.com/element-hq/synapse/issues/16857)) - -### Internal Changes - -- Don't invalidate the entire event cache when we purge history. ([\#16905](https://github.com/element-hq/synapse/issues/16905)) -- Add experimental config option to not send device list updates for specific users. ([\#16909](https://github.com/element-hq/synapse/issues/16909)) -- Fix incorrect docker hub link in release script. ([\#16910](https://github.com/element-hq/synapse/issues/16910)) - - - -### Updates to locked dependencies - -* Bump attrs from 23.1.0 to 23.2.0. ([\#16899](https://github.com/element-hq/synapse/issues/16899)) -* Bump bcrypt from 4.0.1 to 4.1.2. ([\#16900](https://github.com/element-hq/synapse/issues/16900)) -* Bump pygithub from 2.1.1 to 2.2.0. ([\#16902](https://github.com/element-hq/synapse/issues/16902)) -* Bump sentry-sdk from 1.40.0 to 1.40.3. ([\#16898](https://github.com/element-hq/synapse/issues/16898)) - -# Synapse 1.101.0 (2024-02-13) - -### Bugfixes - -- Fix performance regression when fetching auth chains from the DB. Introduced in v1.100.0. ([\#16893](https://github.com/element-hq/synapse/issues/16893)) - - - - -# Synapse 1.101.0rc1 (2024-02-06) - -### Improved Documentation - -- Fix broken links in the documentation. ([\#16853](https://github.com/element-hq/synapse/issues/16853)) -- Update MacOS installation instructions to mention that libicu is optional. ([\#16854](https://github.com/element-hq/synapse/issues/16854)) -- The version picker now correctly lists versions after `v1.98.0`. ([\#16880](https://github.com/element-hq/synapse/issues/16880)) - -### Internal Changes - -- Add support for stabilised [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981) that adds a `recurse` parameter on the `/relations` API. ([\#16842](https://github.com/element-hq/synapse/issues/16842)) - - - -### Updates to locked dependencies - -* Bump dorny/paths-filter from 2 to 3. ([\#16869](https://github.com/element-hq/synapse/issues/16869)) -* Bump gitpython from 3.1.40 to 3.1.41. ([\#16850](https://github.com/element-hq/synapse/issues/16850)) -* Bump hiredis from 2.2.3 to 2.3.2. ([\#16862](https://github.com/element-hq/synapse/issues/16862)) -* Bump jsonschema from 4.20.0 to 4.21.1. ([\#16887](https://github.com/element-hq/synapse/issues/16887)) -* Bump lxml-stubs from 0.4.0 to 0.5.1. ([\#16885](https://github.com/element-hq/synapse/issues/16885)) -* Bump mypy-zope from 1.0.1 to 1.0.3. ([\#16865](https://github.com/element-hq/synapse/issues/16865)) -* Bump phonenumbers from 8.13.26 to 8.13.29. ([\#16868](https://github.com/element-hq/synapse/issues/16868)) -* Bump pydantic from 2.5.3 to 2.6.0. ([\#16888](https://github.com/element-hq/synapse/issues/16888)) -* Bump sentry-sdk from 1.39.1 to 1.40.0. ([\#16889](https://github.com/element-hq/synapse/issues/16889)) -* Bump serde from 1.0.195 to 1.0.196. ([\#16867](https://github.com/element-hq/synapse/issues/16867)) -* Bump serde_json from 1.0.111 to 1.0.113. ([\#16866](https://github.com/element-hq/synapse/issues/16866)) -* Bump sigstore/cosign-installer from 3.3.0 to 3.4.0. ([\#16890](https://github.com/element-hq/synapse/issues/16890)) -* Bump types-pillow from 10.1.0.2 to 10.2.0.20240125. ([\#16864](https://github.com/element-hq/synapse/issues/16864)) -* Bump types-requests from 2.31.0.10 to 2.31.0.20240125. ([\#16886](https://github.com/element-hq/synapse/issues/16886)) -* Bump types-setuptools from 69.0.0.0 to 69.0.0.20240125. ([\#16863](https://github.com/element-hq/synapse/issues/16863)) - -# Synapse 1.100.0 (2024-01-30) - -No significant changes since 1.100.0rc3. - - - - -# Synapse 1.100.0rc3 (2024-01-24) - -### Bugfixes - -- Fix database performance regression due to changing Postgres table statistics. Introduced in v1.100.0rc1. ([\#16849](https://github.com/element-hq/synapse/issues/16849)) - - - - -# Synapse 1.100.0rc2 (2024-01-24) - -This version is the same as 1.100.0rc1 but with fixes to the release process. - -### Internal Changes - -- Downgrade the `download-artifact` and `upload-artifact` actions to v3 due to breaking changes. ([\#16847](https://github.com/element-hq/synapse/issues/16847)) - - -# Synapse 1.100.0rc1 (2024-01-23) - -*This version was never released to PyPI or the Debian repository due to failures in the automatic part of the release process.* - -### Features - -- Advertise experimental support for [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028) through `/_matrix/clients/versions` if enabled. Contributed by @hanadi92. ([\#16787](https://github.com/element-hq/synapse/issues/16787)) - -### Bugfixes - -- Handle wildcard type filters properly for room messages endpoint. Contributed by Mo Balaa. ([\#14984](https://github.com/element-hq/synapse/issues/14984)) - -### Improved Documentation - -- Add a link to the "Request log format" explainer on the "Logging sample config" documentation page. ([\#16778](https://github.com/element-hq/synapse/issues/16778)) -- Fix broken links in issue templates and documentation. ([\#16810](https://github.com/element-hq/synapse/issues/16810)) -- NGINX listen http2 deprecation in documentation template for reverse proxy. ([\#16831](https://github.com/element-hq/synapse/issues/16831)) - -### Internal Changes - -- Faster partial join to room with complex auth graph. ([\#7](https://github.com/element-hq/synapse/issues/7)) -- Improve DB performance of calculating badge counts for push. ([\#16756](https://github.com/element-hq/synapse/issues/16756)) -- Split up deleting devices into batches. ([\#16766](https://github.com/element-hq/synapse/issues/16766)) -- Remove CI check for sign-off as we require a CLA signature instead. ([\#16776](https://github.com/element-hq/synapse/issues/16776)) -- Ensure CI fails when linting fails to make sure auto-merge does the correct thing. ([\#16781](https://github.com/element-hq/synapse/issues/16781)) -- Faster load recents for sync by reducing amount of state pulled out. ([\#16783](https://github.com/element-hq/synapse/issues/16783)) -- Reduce amount of state pulled out when querying federation hierachy. ([\#16785](https://github.com/element-hq/synapse/issues/16785)) -- Pull less state out of the DB when we retry fetching old events during backfill. ([\#16788](https://github.com/element-hq/synapse/issues/16788)) -- Optimize query for fetching to-device messages in `/sync`. ([\#16805](https://github.com/element-hq/synapse/issues/16805)) -- Reject OIDC config when `client_secret` isn't specified, but the auth method requires one. ([\#16806](https://github.com/element-hq/synapse/issues/16806)) -- Allow room creation but not publishing to continue if room publication rules are violated when creating - a new room. ([\#16811](https://github.com/element-hq/synapse/issues/16811)) -- Bump minimum supported Rust version to 1.65.0. ([\#16818](https://github.com/element-hq/synapse/issues/16818)) -- Fixup copyright lines in file headers after the licensing change. ([\#16820](https://github.com/element-hq/synapse/issues/16820)) -- Add a `--generate-only` option to the internal configuration/launch script for Complement. ([\#16828](https://github.com/element-hq/synapse/issues/16828)) -- Preparatory work for tweaking performance of auth chain lookups. ([\#16833](https://github.com/element-hq/synapse/issues/16833)) -- Speed up e2e device keys queries for bot accounts. ([\#16841](https://github.com/element-hq/synapse/issues/16841)) - -### Updates to locked dependencies - -* Bump actions/cache from 3 to 4. ([\#16832](https://github.com/element-hq/synapse/issues/16832)) -* Bump actions/download-artifact from 3 to 4. ([\#16795](https://github.com/element-hq/synapse/issues/16795)) -* Bump actions/upload-artifact from 3 to 4. ([\#16796](https://github.com/element-hq/synapse/issues/16796)) -* Bump anyhow from 1.0.75 to 1.0.79. ([\#16789](https://github.com/element-hq/synapse/issues/16789)) -* Bump authlib from 1.2.1 to 1.3.0. ([\#16801](https://github.com/element-hq/synapse/issues/16801)) -* Bump dawidd6/action-download-artifact from 2.28.0 to 3.0.0. ([\#16794](https://github.com/element-hq/synapse/issues/16794)) -* Bump immutabledict from 4.0.0 to 4.1.0. ([\#16812](https://github.com/element-hq/synapse/issues/16812)) -* Bump isort from 5.13.1 to 5.13.2. ([\#16835](https://github.com/element-hq/synapse/issues/16835)) -* Bump lxml from 4.9.3 to 5.1.0. ([\#16813](https://github.com/element-hq/synapse/issues/16813)) -* Bump pillow from 10.1.0 to 10.2.0. ([\#16802](https://github.com/element-hq/synapse/issues/16802)) -* Bump pydantic from 2.5.2 to 2.5.3. ([\#16836](https://github.com/element-hq/synapse/issues/16836)) -* Bump pyo3 from 0.20.0 to 0.20.2. ([\#16791](https://github.com/element-hq/synapse/issues/16791)) -* Bump regex from 1.9.6 to 1.10.3. ([\#16837](https://github.com/element-hq/synapse/issues/16837)) -* Bump ruff from 0.1.13 to 0.1.14. ([\#16838](https://github.com/element-hq/synapse/issues/16838)) -* Bump ruff from 0.1.7 to 0.1.13. ([\#16814](https://github.com/element-hq/synapse/issues/16814)) -* Bump sentry-sdk from 1.35.0 to 1.39.1. ([\#16799](https://github.com/element-hq/synapse/issues/16799)) -* Bump serde_json from 1.0.108 to 1.0.111. ([\#16792](https://github.com/element-hq/synapse/issues/16792)) -* Bump service-identity from 23.1.0 to 24.1.0. ([\#16816](https://github.com/element-hq/synapse/issues/16816)) -* Bump types-commonmark from 0.9.2.4 to 0.9.2.20240106. ([\#16797](https://github.com/element-hq/synapse/issues/16797)) -* Bump types-jsonschema from 4.20.0.0 to 4.20.0.20240105. ([\#16800](https://github.com/element-hq/synapse/issues/16800)) -* Bump types-jsonschema from 4.20.0.20240105 to 4.21.0.20240118. ([\#16834](https://github.com/element-hq/synapse/issues/16834)) -* Bump types-netaddr from 0.9.0.1 to 0.10.0.20240106. ([\#16839](https://github.com/element-hq/synapse/issues/16839)) -* Bump typing-extensions from 4.8.0 to 4.9.0. ([\#16815](https://github.com/element-hq/synapse/issues/16815)) - - -# Synapse 1.99.0 (2024-01-16) - -Synapse 1.99.0 is the first Synapse release under an AGPLv3.0 licence (with CLA to enable Element to sell AGPL -exceptions). You can read more about this here: - - - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ - - https://element.io/blog/element-to-adopt-agplv3/ - - https://element.io/blog/synapse-now-lives-at-github-com-element-hq-synapse/ - -No significant changes since 1.99.0rc1. - - -# Synapse 1.99.0rc1 (2024-01-09) - -### Features - -- Add [config options](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#server_notices) to set the avatar and the topic of the server notices room, as well as the avatar of the server notices user. ([\#16679](https://github.com/matrix-org/synapse/issues/16679)) -- Add config option [`email.notif_delay_before_mail`](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#email) to tweak the delay before an email is sent following a notification. ([\#16696](https://github.com/matrix-org/synapse/issues/16696)) -- Add new configuration option [`sentry.environment`](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#sentry) for improved system monitoring. Contributed by @zeeshanrafiqrana. ([\#16738](https://github.com/matrix-org/synapse/issues/16738)) -- Filter out rooms from the room directory being served to other homeservers when those rooms block that homeserver by their Access Control Lists. ([\#16759](https://github.com/element-hq/synapse/issues/16759)) - -### Bugfixes - -- Fix a long-standing bug where the signing keys generated by Synapse were world-readable. Contributed by Fabian Klemp. ([\#16740](https://github.com/matrix-org/synapse/issues/16740)) -- Fix email verification redirection. Contributed by Fadhlan Ridhwanallah. ([\#16761](https://github.com/element-hq/synapse/issues/16761)) -- Fixed a bug that prevented users from being queried by display name if it contains non-ASCII characters. ([\#16767](https://github.com/element-hq/synapse/issues/16767)) -- Allow reactivate user without password with Admin API in some edge cases. ([\#16770](https://github.com/element-hq/synapse/issues/16770)) -- Adds the `recursion_depth` parameter to the response of the /relations endpoint if MSC3981 recursion is being performed. ([\#16775](https://github.com/element-hq/synapse/issues/16775)) - -### Improved Documentation - -- Added version picker for Synapse documentation. Contributed by @Dmytro27Ind. ([\#16533](https://github.com/matrix-org/synapse/issues/16533)) -- Clarify that `password_config.enabled: "only_for_reauth"` does not allow new logins to be created using password auth. ([\#16737](https://github.com/matrix-org/synapse/issues/16737)) -- Remove value from header in configuration documentation for `refresh_token_lifetime`. ([\#16763](https://github.com/element-hq/synapse/issues/16763)) -- Add another custom statistics collection server to the documentation. Contributed by @loelkes. ([\#16769](https://github.com/element-hq/synapse/issues/16769)) - -### Internal Changes - -- Remove run-once workflow after adding the version picker to the documentation. ([\#9453](https://github.com/element-hq/synapse/issues/9453)) -- Update the implementation of [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965) (OIDC Provider discovery). ([\#16726](https://github.com/matrix-org/synapse/issues/16726)) -- Move the rust stubs inline for better IDE integration. ([\#16757](https://github.com/element-hq/synapse/issues/16757)) -- Fix sample config doc CI. ([\#16758](https://github.com/element-hq/synapse/issues/16758)) -- Simplify event internal metadata class. ([\#16762](https://github.com/element-hq/synapse/issues/16762), [\#16780](https://github.com/element-hq/synapse/issues/16780)) -- Sign the published docker image using [cosign](https://docs.sigstore.dev/). ([\#16774](https://github.com/element-hq/synapse/issues/16774)) -- Port `EventInternalMetadata` class to Rust. ([\#16782](https://github.com/element-hq/synapse/issues/16782)) - - - -### Updates to locked dependencies - -* Bump actions/setup-go from 4 to 5. ([\#16749](https://github.com/matrix-org/synapse/issues/16749)) -* Bump actions/setup-python from 4 to 5. ([\#16748](https://github.com/matrix-org/synapse/issues/16748)) -* Bump immutabledict from 3.0.0 to 4.0.0. ([\#16743](https://github.com/matrix-org/synapse/issues/16743)) -* Bump isort from 5.12.0 to 5.13.0. ([\#16745](https://github.com/matrix-org/synapse/issues/16745)) -* Bump isort from 5.13.0 to 5.13.1. ([\#16752](https://github.com/matrix-org/synapse/issues/16752)) -* Bump pydantic from 2.5.1 to 2.5.2. ([\#16747](https://github.com/matrix-org/synapse/issues/16747)) -* Bump ruff from 0.1.6 to 0.1.7. ([\#16746](https://github.com/matrix-org/synapse/issues/16746)) -* Bump types-setuptools from 68.2.0.2 to 69.0.0.0. ([\#16744](https://github.com/matrix-org/synapse/issues/16744)) - -# Synapse 1.98.0 (2023-12-12) - -Synapse 1.98.0 will be the last Synapse release in 2023; the regular release cadence will resume in January 2024. - -Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for -proprietary dual licensing). You can read more about this here: - - - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ - - https://element.io/blog/element-to-adopt-agplv3/ - -The Matrix.org Foundation copy of the project will be archived. Any changes needed -by server administrators will be communicated via our usual announcements channels, -but we are striving to make this as seamless as possible. - - -No significant changes since 1.98.0rc1. - - - -# Synapse 1.98.0rc1 (2023-12-05) - -### Features - -- Synapse now declares support for Matrix v1.7, v1.8, and v1.9. ([\#16707](https://github.com/matrix-org/synapse/issues/16707)) -- Add `on_user_login` [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) callback for when a user logs in. ([\#15207](https://github.com/matrix-org/synapse/issues/15207)) -- Support [MSC4069: Inhibit profile propagation](https://github.com/matrix-org/matrix-spec-proposals/pull/4069). ([\#16636](https://github.com/matrix-org/synapse/issues/16636)) -- Restore tracking of requests and monthly active users when delegating authentication via [MSC3861](https://github.com/matrix-org/synapse/pull/16672) to an OIDC provider. ([\#16672](https://github.com/matrix-org/synapse/issues/16672)) -- Add an autojoin setting for server notices rooms, so users may be joined directly instead of receiving an invite. ([\#16699](https://github.com/matrix-org/synapse/issues/16699)) -- Follow redirects when downloading media over federation (per [MSC3860](https://github.com/matrix-org/matrix-spec-proposals/pull/3860)). ([\#16701](https://github.com/matrix-org/synapse/issues/16701)) - -### Bugfixes - -- Enable refreshable tokens on the admin registration endpoint. ([\#16642](https://github.com/matrix-org/synapse/issues/16642)) -- Consistently bypass rate limits when using the server notice admin API. ([\#16670](https://github.com/matrix-org/synapse/issues/16670)) -- Fix a bug introduced in Synapse 1.7.2 where rooms whose power levels lacked an `events` field could not be upgraded. ([\#16725](https://github.com/matrix-org/synapse/issues/16725)) -- Fix `GET /_synapse/admin/v1/federation/destinations` [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) returning null (instead of 0) for `retry_last_ts` and `retry_interval`. ([\#16729](https://github.com/matrix-org/synapse/issues/16729)) - -### Improved Documentation - -- Add schema rollback information to documentation. ([\#16661](https://github.com/matrix-org/synapse/issues/16661)) -- Fix poetry version typo in the [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16695](https://github.com/matrix-org/synapse/issues/16695)) -- Switch the example UNIX socket paths to `/run`. Add HAProxy example configuration for UNIX sockets. ([\#16700](https://github.com/matrix-org/synapse/issues/16700)) -- Add documentation for how to validate the configuration file with `synapse.config` script. ([\#16714](https://github.com/matrix-org/synapse/issues/16714)) - -### Internal Changes - -- Clean-up unused tables. ([\#16522](https://github.com/matrix-org/synapse/issues/16522)) -- Reduce a little database load while processing state auth chains. ([\#16552](https://github.com/matrix-org/synapse/issues/16552)) -- Reduce database load of pruning old `user_ips`. ([\#16667](https://github.com/matrix-org/synapse/issues/16667)) -- Reduce DB load when forget on leave setting is disabled. ([\#16668](https://github.com/matrix-org/synapse/issues/16668)) -- Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway. ([\#16677](https://github.com/matrix-org/synapse/issues/16677)) -- Correctly read the to-device stream ID on startup using SQLite. ([\#16682](https://github.com/matrix-org/synapse/issues/16682)) -- Reoranganise test files. ([\#16684](https://github.com/matrix-org/synapse/issues/16684)) -- Remove old full schema dumps which are no longer used. ([\#16697](https://github.com/matrix-org/synapse/issues/16697)) -- Raise poetry-core upper bound to <=1.8.1. This allows contributors to import Synapse after `poetry install`ing with Poetry 1.6 and above. Contributed by Mo Balaa. ([\#16702](https://github.com/matrix-org/synapse/issues/16702)) -- Add a workflow to try and automatically fixup linting in a PR. ([\#16704](https://github.com/matrix-org/synapse/issues/16704)) - - -### Updates to locked dependencies - -* Bump cryptography from 41.0.5 to 41.0.6. ([\#16703](https://github.com/matrix-org/synapse/issues/16703)) -* Bump cryptography from 41.0.6 to 41.0.7. ([\#16721](https://github.com/matrix-org/synapse/issues/16721)) -* Bump idna from 3.4 to 3.6. ([\#16720](https://github.com/matrix-org/synapse/issues/16720)) -* Bump jsonschema from 4.19.1 to 4.20.0. ([\#16692](https://github.com/matrix-org/synapse/issues/16692)) -* Bump matrix-org/netlify-pr-preview from 2 to 3. ([\#16719](https://github.com/matrix-org/synapse/issues/16719)) -* Bump phonenumbers from 8.13.23 to 8.13.26. ([\#16722](https://github.com/matrix-org/synapse/issues/16722)) -* Bump prometheus-client from 0.18.0 to 0.19.0. ([\#16691](https://github.com/matrix-org/synapse/issues/16691)) -* Bump pyasn1 from 0.5.0 to 0.5.1. ([\#16689](https://github.com/matrix-org/synapse/issues/16689)) -* Bump pydantic from 2.4.2 to 2.5.1. ([\#16663](https://github.com/matrix-org/synapse/issues/16663)) -* Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). ([\#16673](https://github.com/matrix-org/synapse/issues/16673)) -* Bump pyopenssl from 23.2.0 to 23.3.0. ([\#16662](https://github.com/matrix-org/synapse/issues/16662)) -* Bump ruff from 0.1.4 to 0.1.6. ([\#16690](https://github.com/matrix-org/synapse/issues/16690)) -* Bump sentry-sdk from 1.32.0 to 1.35.0. ([\#16666](https://github.com/matrix-org/synapse/issues/16666)) -* Bump serde from 1.0.192 to 1.0.193. ([\#16693](https://github.com/matrix-org/synapse/issues/16693)) -* Bump sphinx-autodoc2 from 0.4.2 to 0.5.0. ([\#16723](https://github.com/matrix-org/synapse/issues/16723)) -* Bump types-jsonschema from 4.19.0.4 to 4.20.0.0. ([\#16724](https://github.com/matrix-org/synapse/issues/16724)) -* Bump types-pillow from 10.1.0.0 to 10.1.0.2. ([\#16664](https://github.com/matrix-org/synapse/issues/16664)) -* Bump types-psycopg2 from 2.9.21.15 to 2.9.21.16. ([\#16665](https://github.com/matrix-org/synapse/issues/16665)) -* Bump types-setuptools from 68.2.0.0 to 68.2.0.2. ([\#16688](https://github.com/matrix-org/synapse/issues/16688)) - -# Synapse 1.97.0 (2023-11-28) - -Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for -proprietary dual licensing). You can read more about this here: - - - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ - - https://element.io/blog/element-to-adopt-agplv3/ - -The Matrix.org Foundation copy of the project will be archived. Any changes needed -by server administrators will be communicated via our usual announcements channels, -but we are striving to make this as seamless as possible. - - -No significant changes since 1.97.0rc1. - - -# Synapse 1.97.0rc1 (2023-11-21) - -### Features - -- Add support for asynchronous uploads as defined by [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246). Contributed by @sumnerevans at @beeper. ([\#15503](https://github.com/matrix-org/synapse/issues/15503)) -- Improve the performance of some operations in multi-worker deployments. ([\#16613](https://github.com/matrix-org/synapse/issues/16613), [\#16616](https://github.com/matrix-org/synapse/issues/16616)) - -### Bugfixes - -- Fix a long-standing bug where some queries updated the same row twice. Introduced in Synapse 1.57.0. ([\#16609](https://github.com/matrix-org/synapse/issues/16609)) -- Fix a long-standing bug where Synapse would not unbind third-party identifiers for Application Service users when deactivated and would not emit a compliant response. ([\#16617](https://github.com/matrix-org/synapse/issues/16617)) -- Fix sending out of order `POSITION` over replication, causing additional database load. ([\#16639](https://github.com/matrix-org/synapse/issues/16639)) - -### Improved Documentation - -- Note that the option [`outbound_federation_restricted_to`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#outbound_federation_restricted_to) was added in Synapse 1.89.0, and fix a nearby formatting error. ([\#16628](https://github.com/matrix-org/synapse/issues/16628)) -- Update parameter information for the `/timestamp_to_event` admin API. ([\#16631](https://github.com/matrix-org/synapse/issues/16631)) -- Provide an example for a common encrypted media response from the admin user media API and mention possible null values. ([\#16654](https://github.com/matrix-org/synapse/issues/16654)) - -### Internal Changes - -- Remove whole table locks on push rule modifications. Contributed by Nick @ Beeper (@fizzadar). ([\#16051](https://github.com/matrix-org/synapse/issues/16051)) -- Support reactor tick timings on more types of event loops. ([\#16532](https://github.com/matrix-org/synapse/issues/16532)) -- Improve type hints. ([\#16564](https://github.com/matrix-org/synapse/issues/16564), [\#16611](https://github.com/matrix-org/synapse/issues/16611), [\#16612](https://github.com/matrix-org/synapse/issues/16612)) -- Avoid executing no-op queries. ([\#16583](https://github.com/matrix-org/synapse/issues/16583)) -- Simplify persistence code to be per-room. ([\#16584](https://github.com/matrix-org/synapse/issues/16584)) -- Use standard SQL helpers in persistence code. ([\#16585](https://github.com/matrix-org/synapse/issues/16585)) -- Avoid updating the stream cache unnecessarily. ([\#16586](https://github.com/matrix-org/synapse/issues/16586)) -- Improve performance when using opentracing. ([\#16589](https://github.com/matrix-org/synapse/issues/16589)) -- Run push rule evaluator setup in parallel. ([\#16590](https://github.com/matrix-org/synapse/issues/16590)) -- Improve tests of the SQL generator. ([\#16596](https://github.com/matrix-org/synapse/issues/16596)) -- Use more generic database methods. ([\#16615](https://github.com/matrix-org/synapse/issues/16615)) -- Use `dbname` instead of the deprecated `database` connection parameter for psycopg2. ([\#16618](https://github.com/matrix-org/synapse/issues/16618)) -- Add an internal [Admin API endpoint](https://matrix-org.github.io/synapse/v1.97/usage/configuration/config_documentation.html#allow-replacing-master-cross-signing-key-without-user-interactive-auth) to temporarily grant the ability to update an existing cross-signing key without UIA. ([\#16634](https://github.com/matrix-org/synapse/issues/16634)) -- Improve references to GitHub issues. ([\#16637](https://github.com/matrix-org/synapse/issues/16637), [\#16638](https://github.com/matrix-org/synapse/issues/16638)) -- More efficiently handle no-op `POSITION` over replication. ([\#16640](https://github.com/matrix-org/synapse/issues/16640), [\#16655](https://github.com/matrix-org/synapse/issues/16655)) -- Speed up deleting of device messages when deleting a device. ([\#16643](https://github.com/matrix-org/synapse/issues/16643)) -- Speed up persisting large number of outliers. ([\#16649](https://github.com/matrix-org/synapse/issues/16649)) -- Reduce max concurrency of background tasks, reducing potential max DB load. ([\#16656](https://github.com/matrix-org/synapse/issues/16656), [\#16660](https://github.com/matrix-org/synapse/issues/16660)) -- Speed up purge room by adding an index to `event_push_summary`. ([\#16657](https://github.com/matrix-org/synapse/issues/16657)) - - - -### Updates to locked dependencies - -* Bump prometheus-client from 0.17.1 to 0.18.0. ([\#16626](https://github.com/matrix-org/synapse/issues/16626)) -* Bump pyicu from 2.11 to 2.12. ([\#16603](https://github.com/matrix-org/synapse/issues/16603)) -* Bump requests-toolbelt from 0.10.1 to 1.0.0. ([\#16659](https://github.com/matrix-org/synapse/issues/16659)) -* Bump ruff from 0.0.292 to 0.1.4. ([\#16600](https://github.com/matrix-org/synapse/issues/16600)) -* Bump serde from 1.0.190 to 1.0.192. ([\#16627](https://github.com/matrix-org/synapse/issues/16627)) -* Bump serde_json from 1.0.107 to 1.0.108. ([\#16604](https://github.com/matrix-org/synapse/issues/16604)) -* Bump setuptools-rust from 1.8.0 to 1.8.1. ([\#16601](https://github.com/matrix-org/synapse/issues/16601)) -* Bump towncrier from 23.6.0 to 23.11.0. ([\#16622](https://github.com/matrix-org/synapse/issues/16622)) -* Bump treq from 22.2.0 to 23.11.0. ([\#16623](https://github.com/matrix-org/synapse/issues/16623)) -* Bump twisted from 23.8.0 to 23.10.0. ([\#16588](https://github.com/matrix-org/synapse/issues/16588)) -* Bump types-bleach from 6.1.0.0 to 6.1.0.1. ([\#16624](https://github.com/matrix-org/synapse/issues/16624)) -* Bump types-jsonschema from 4.19.0.3 to 4.19.0.4. ([\#16599](https://github.com/matrix-org/synapse/issues/16599)) -* Bump types-pyopenssl from 23.2.0.2 to 23.3.0.0. ([\#16625](https://github.com/matrix-org/synapse/issues/16625)) -* Bump types-pyyaml from 6.0.12.11 to 6.0.12.12. ([\#16602](https://github.com/matrix-org/synapse/issues/16602)) - -# Synapse 1.96.1 (2023-11-17) - -Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for -proprietary dual licensing). You can read more about this here: - -* https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ -* https://element.io/blog/element-to-adopt-agplv3/ - -The Matrix.org Foundation copy of the project will be archived. Any changes needed -by server administrators will be communicated via our usual -[announcements channels](https://matrix.to/#/#homeowners:matrix.org), but we are -striving to make this as seamless as possible. - -This minor release was needed only because of CI-related trouble on [v1.96.0](https://github.com/matrix-org/synapse/releases/tag/v1.96.0), which was never released. - -### Internal Changes - -- Fix building of wheels in CI. ([\#16653](https://github.com/matrix-org/synapse/issues/16653)) - -# Synapse 1.96.0 (2023-11-16) - -### Bugfixes - -- Fix "'int' object is not iterable" error in `set_device_id_for_pushers` background update introduced in Synapse 1.95.0. ([\#16594](https://github.com/matrix-org/synapse/issues/16594)) - -# Synapse 1.96.0rc1 (2023-10-31) - -### Features - -- Add experimental support to allow multiple workers to write to receipts stream. ([\#16432](https://github.com/matrix-org/synapse/issues/16432)) -- Add a new module API for controller presence. ([\#16544](https://github.com/matrix-org/synapse/issues/16544)) -- Add a new module API callback that allows adding extra fields to events' unsigned section when sent down to clients. ([\#16549](https://github.com/matrix-org/synapse/issues/16549)) -- Improve the performance of claiming encryption keys. ([\#16565](https://github.com/matrix-org/synapse/issues/16565), [\#16570](https://github.com/matrix-org/synapse/issues/16570)) - -### Bugfixes - -- Fixed a bug in the example Grafana dashboard that prevents it from finding the correct datasource. Contributed by @MichaelSasser. ([\#16471](https://github.com/matrix-org/synapse/issues/16471)) -- Fix a long-standing, exceedingly rare edge case where the first event persisted by a new event persister worker might not be sent down `/sync`. ([\#16473](https://github.com/matrix-org/synapse/issues/16473), [\#16557](https://github.com/matrix-org/synapse/issues/16557), [\#16561](https://github.com/matrix-org/synapse/issues/16561), [\#16578](https://github.com/matrix-org/synapse/issues/16578), [\#16580](https://github.com/matrix-org/synapse/issues/16580)) -- Fix long-standing bug where `/sync` incorrectly did not mark a room as `limited` in a sync requests when there were missing remote events. ([\#16485](https://github.com/matrix-org/synapse/issues/16485)) -- Fix a bug introduced in Synapse 1.41 where HTTP(S) forward proxy authorization would fail when using basic HTTP authentication with a long `username:password` string. ([\#16504](https://github.com/matrix-org/synapse/issues/16504)) -- Force TLS certificate verification in user registration script. ([\#16530](https://github.com/matrix-org/synapse/issues/16530)) -- Fix long-standing bug where `/sync` could tightloop after restart when using SQLite. ([\#16540](https://github.com/matrix-org/synapse/issues/16540)) -- Fix ratelimiting of message sending when using workers, where the ratelimit would only be applied after most of the work has been done. ([\#16558](https://github.com/matrix-org/synapse/issues/16558)) -- Fix a long-standing bug where invited/knocking users would not leave during a room purge. ([\#16559](https://github.com/matrix-org/synapse/issues/16559)) - -### Improved Documentation - -- Improve documentation of presence router. ([\#16529](https://github.com/matrix-org/synapse/issues/16529)) -- Add a sentence to the [opentracing docs](https://matrix-org.github.io/synapse/latest/opentracing.html) on how you can have jaeger in a different place than synapse. ([\#16531](https://github.com/matrix-org/synapse/issues/16531)) -- Correctly describe the meaning of unspecified rule lists in the [`alias_creation_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#alias_creation_rules) and [`room_list_publication_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#room_list_publication_rules) config options and improve their descriptions more generally. ([\#16541](https://github.com/matrix-org/synapse/issues/16541)) -- Pin the recommended poetry version in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16550](https://github.com/matrix-org/synapse/issues/16550)) -- Fix a broken link to the [client breakdown](https://matrix.org/ecosystem/clients/) in the README. ([\#16569](https://github.com/matrix-org/synapse/issues/16569)) - -### Internal Changes - -- Improve performance of delete device messages query, cf issue [16479](https://github.com/matrix-org/synapse/issues/16479). ([\#16492](https://github.com/matrix-org/synapse/issues/16492)) -- Reduce memory allocations. ([\#16505](https://github.com/matrix-org/synapse/issues/16505)) -- Improve replication performance when purging rooms. ([\#16510](https://github.com/matrix-org/synapse/issues/16510)) -- Run tests against Python 3.12. ([\#16511](https://github.com/matrix-org/synapse/issues/16511)) -- Run trial & integration tests in continuous integration when `.ci` directory is modified. ([\#16512](https://github.com/matrix-org/synapse/issues/16512)) -- Remove duplicate call to mark remote server 'awake' when using a federation sending worker. ([\#16515](https://github.com/matrix-org/synapse/issues/16515)) -- Enable dirty runs on Complement CI, which is significantly faster. ([\#16520](https://github.com/matrix-org/synapse/issues/16520)) -- Stop deleting from an unused table. ([\#16521](https://github.com/matrix-org/synapse/issues/16521)) -- Improve type hints. ([\#16526](https://github.com/matrix-org/synapse/issues/16526), [\#16551](https://github.com/matrix-org/synapse/issues/16551)) -- Fix running unit tests on Twisted trunk. ([\#16528](https://github.com/matrix-org/synapse/issues/16528)) -- Reduce some spurious logging in worker mode. ([\#16555](https://github.com/matrix-org/synapse/issues/16555)) -- Stop porting a table in port db that we're going to nuke and rebuild anyway. ([\#16563](https://github.com/matrix-org/synapse/issues/16563)) -- Deal with warnings from running complement in CI. ([\#16567](https://github.com/matrix-org/synapse/issues/16567)) -- Allow building with `setuptools_rust` 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574)) - -### Updates to locked dependencies - -* Bump black from 23.10.0 to 23.10.1. ([\#16575](https://github.com/matrix-org/synapse/issues/16575)) -* Bump black from 23.9.1 to 23.10.0. ([\#16538](https://github.com/matrix-org/synapse/issues/16538)) -* Bump cryptography from 41.0.4 to 41.0.5. ([\#16572](https://github.com/matrix-org/synapse/issues/16572)) -* Bump gitpython from 3.1.37 to 3.1.40. ([\#16534](https://github.com/matrix-org/synapse/issues/16534)) -* Bump phonenumbers from 8.13.22 to 8.13.23. ([\#16576](https://github.com/matrix-org/synapse/issues/16576)) -* Bump pygithub from 1.59.1 to 2.1.1. ([\#16535](https://github.com/matrix-org/synapse/issues/16535)) -- Bump matrix-synapse-ldap3 from 0.2.2 to 0.3.0. ([\#16539](https://github.com/matrix-org/synapse/issues/16539)) -* Bump serde from 1.0.189 to 1.0.190. ([\#16577](https://github.com/matrix-org/synapse/issues/16577)) -* Bump setuptools-rust from 1.7.0 to 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574)) -* Bump types-pillow from 10.0.0.3 to 10.1.0.0. ([\#16536](https://github.com/matrix-org/synapse/issues/16536)) -* Bump types-psycopg2 from 2.9.21.14 to 2.9.21.15. ([\#16573](https://github.com/matrix-org/synapse/issues/16573)) -* Bump types-requests from 2.31.0.2 to 2.31.0.10. ([\#16537](https://github.com/matrix-org/synapse/issues/16537)) -* Bump urllib3 from 1.26.17 to 1.26.18. ([\#16516](https://github.com/matrix-org/synapse/issues/16516)) - -# Synapse 1.95.1 (2023-10-31) - -## Security advisory - -The following issue is fixed in 1.95.1. - -- [GHSA-mp92-3jfm-3575](https://github.com/matrix-org/synapse/security/advisories/GHSA-mp92-3jfm-3575) / [CVE-2023-43796](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-43796) — Moderate Severity - - Cached device information of remote users can be queried from Synapse. This can be used to enumerate the remote users known to a homeserver. - -See the advisory for more details. If you have any questions, email security@matrix.org. - - - -# Synapse 1.95.0 (2023-10-24) - -### Internal Changes - -- Build Debian packages for [Ubuntu 23.10 Mantic Minotaur](https://canonical.com/blog/canonical-releases-ubuntu-23-10-mantic-minotaur). ([\#16524](https://github.com/matrix-org/synapse/issues/16524)) - - -# Synapse 1.95.0rc1 (2023-10-17) - -### Bugfixes - -- Remove legacy unspecced `knock_state_events` field returned in some responses. ([\#16403](https://github.com/matrix-org/synapse/issues/16403)) -- Fix a bug introduced in Synapse 1.81.0 where an `AttributeError` would be raised when `_matrix/client/v3/account/whoami` is called over a unix socket. Contributed by @Sir-Photch. ([\#16404](https://github.com/matrix-org/synapse/issues/16404)) -- Properly return inline media when content types have parameters. ([\#16440](https://github.com/matrix-org/synapse/issues/16440)) -- Prevent the purging of large rooms from timing out when Postgres is in use. The timeout which causes this issue was introduced in Synapse 1.88.0. ([\#16455](https://github.com/matrix-org/synapse/issues/16455)) -- Improve the performance of purging rooms, particularly encrypted rooms. ([\#16457](https://github.com/matrix-org/synapse/issues/16457)) -- Fix a bug introduced in Synapse 1.59.0 where servers could be incorrectly marked as available after an error response was received. ([\#16506](https://github.com/matrix-org/synapse/issues/16506)) - -### Improved Documentation - -- Document internal background update mechanism. ([\#16420](https://github.com/matrix-org/synapse/issues/16420)) -- Fix a typo in the sql for [useful SQL for admins document](https://matrix-org.github.io/synapse/latest/usage/administration/useful_sql_for_admins.html). ([\#16477](https://github.com/matrix-org/synapse/issues/16477)) - -### Internal Changes - -- Bump pyo3 from 0.17.1 to 0.19.2. ([\#16162](https://github.com/matrix-org/synapse/issues/16162)) -- Update registration of media repository URLs. ([\#16419](https://github.com/matrix-org/synapse/issues/16419)) -- Improve type hints. ([\#16421](https://github.com/matrix-org/synapse/issues/16421), [\#16468](https://github.com/matrix-org/synapse/issues/16468), [\#16469](https://github.com/matrix-org/synapse/issues/16469), [\#16507](https://github.com/matrix-org/synapse/issues/16507)) -- Refactor some code to simplify and better type receipts stream adjacent code. ([\#16426](https://github.com/matrix-org/synapse/issues/16426)) -- Factor out `MultiWriter` token from `RoomStreamToken`. ([\#16427](https://github.com/matrix-org/synapse/issues/16427)) -- Improve code comments. ([\#16428](https://github.com/matrix-org/synapse/issues/16428)) -- Reduce memory allocations. ([\#16429](https://github.com/matrix-org/synapse/issues/16429), [\#16431](https://github.com/matrix-org/synapse/issues/16431), [\#16433](https://github.com/matrix-org/synapse/issues/16433), [\#16434](https://github.com/matrix-org/synapse/issues/16434), [\#16438](https://github.com/matrix-org/synapse/issues/16438), [\#16444](https://github.com/matrix-org/synapse/issues/16444)) -- Remove unused method. ([\#16435](https://github.com/matrix-org/synapse/issues/16435)) -- Improve rate limiting logic. ([\#16441](https://github.com/matrix-org/synapse/issues/16441)) -- Do not block running of CI behind the check for sign-off on PRs. ([\#16454](https://github.com/matrix-org/synapse/issues/16454)) -- Update the release script to remind releaser to check for special release notes. ([\#16461](https://github.com/matrix-org/synapse/issues/16461)) -- Update complement.sh to match new public API shape. ([\#16466](https://github.com/matrix-org/synapse/issues/16466)) -- Clean up logging on event persister endpoints. ([\#16488](https://github.com/matrix-org/synapse/issues/16488)) -- Remove useless async job to delete device messages on sync, since we only deliver (and hence delete) up to 100 device messages at a time. ([\#16491](https://github.com/matrix-org/synapse/issues/16491)) - -### Updates to locked dependencies - -* Bump bleach from 6.0.0 to 6.1.0. ([\#16451](https://github.com/matrix-org/synapse/issues/16451)) -* Bump jsonschema from 4.19.0 to 4.19.1. ([\#16500](https://github.com/matrix-org/synapse/issues/16500)) -* Bump netaddr from 0.8.0 to 0.9.0. ([\#16453](https://github.com/matrix-org/synapse/issues/16453)) -* Bump packaging from 23.1 to 23.2. ([\#16497](https://github.com/matrix-org/synapse/issues/16497)) -* Bump pillow from 10.0.1 to 10.1.0. ([\#16498](https://github.com/matrix-org/synapse/issues/16498)) -* Bump psycopg2 from 2.9.8 to 2.9.9. ([\#16452](https://github.com/matrix-org/synapse/issues/16452)) -* Bump pyo3-log from 0.8.3 to 0.8.4. ([\#16495](https://github.com/matrix-org/synapse/issues/16495)) -* Bump ruff from 0.0.290 to 0.0.292. ([\#16449](https://github.com/matrix-org/synapse/issues/16449)) -* Bump sentry-sdk from 1.31.0 to 1.32.0. ([\#16496](https://github.com/matrix-org/synapse/issues/16496)) -* Bump serde from 1.0.188 to 1.0.189. ([\#16494](https://github.com/matrix-org/synapse/issues/16494)) -* Bump types-bleach from 6.0.0.4 to 6.1.0.0. ([\#16450](https://github.com/matrix-org/synapse/issues/16450)) -* Bump types-jsonschema from 4.17.0.10 to 4.19.0.3. ([\#16499](https://github.com/matrix-org/synapse/issues/16499)) - -# Synapse 1.94.0 (2023-10-10) - -No significant changes since 1.94.0rc1. -However, please take note of the security advisory that follows. - -## Security advisory - -The following issue is fixed in 1.94.0 (and RC). - -- [GHSA-5chr-wjw5-3gq4](https://github.com/matrix-org/synapse/security/advisories/GHSA-5chr-wjw5-3gq4) / [CVE-2023-45129](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-45129) — Moderate Severity - - A malicious server ACL event can impact performance temporarily or permanently leading to a persistent denial of service. - - Homeservers running on a closed federation (which presumably do not need to use server ACLs) are not affected. - -See the advisory for more details. If you have any questions, email security@matrix.org. - - -# Synapse 1.94.0rc1 (2023-10-03) - -### Features - -- Render plain, CSS, CSV, JSON and common image formats in the browser (inline) when requested through the /download endpoint. ([\#15988](https://github.com/matrix-org/synapse/issues/15988)) -- Add experimental support for [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028) to push all encrypted events to clients. ([\#16361](https://github.com/matrix-org/synapse/issues/16361)) -- Minor performance improvement when sending presence to federated servers. ([\#16385](https://github.com/matrix-org/synapse/issues/16385)) -- Minor performance improvement by caching server ACL checking. ([\#16360](https://github.com/matrix-org/synapse/issues/16360)) - -### Improved Documentation - -- Add developer documentation concerning gradual schema migrations with column alterations. ([\#15691](https://github.com/matrix-org/synapse/issues/15691)) -- Improve documentation of the user directory search algorithm. ([\#16320](https://github.com/matrix-org/synapse/issues/16320)) -- Fix rendering of user admin API documentation around deactivation. This was broken in Synapse 1.91.0. ([\#16355](https://github.com/matrix-org/synapse/issues/16355)) -- Update documentation around message retention policies. ([\#16382](https://github.com/matrix-org/synapse/issues/16382)) -- Add note to `federation_domain_whitelist` config option to clarify its usage. ([\#16416](https://github.com/matrix-org/synapse/issues/16416)) -- Improve legacy release notes. ([\#16418](https://github.com/matrix-org/synapse/issues/16418)) - -### Deprecations and Removals - -- Remove Python version from `/_synapse/admin/v1/server_version`. ([\#16380](https://github.com/matrix-org/synapse/issues/16380)) - -### Internal Changes - -- Avoid running CI steps when the files they check have not been changed. ([\#14745](https://github.com/matrix-org/synapse/issues/14745), [\#16387](https://github.com/matrix-org/synapse/issues/16387)) -- Improve type hints. ([\#14911](https://github.com/matrix-org/synapse/issues/14911), [\#16350](https://github.com/matrix-org/synapse/issues/16350), [\#16356](https://github.com/matrix-org/synapse/issues/16356), [\#16395](https://github.com/matrix-org/synapse/issues/16395)) -- Added support for pydantic v2 in addition to pydantic v1. Contributed by Maxwell G (@gotmax23). ([\#16332](https://github.com/matrix-org/synapse/issues/16332)) -- Get CI to check PRs have been signed-off. ([\#16348](https://github.com/matrix-org/synapse/issues/16348)) -- Add missing licence header. ([\#16359](https://github.com/matrix-org/synapse/issues/16359)) -- Improve type hints, and bump types-psycopg2 from 2.9.21.11 to 2.9.21.14. ([\#16381](https://github.com/matrix-org/synapse/issues/16381)) -- Improve comments in `StateGroupBackgroundUpdateStore`. ([\#16383](https://github.com/matrix-org/synapse/issues/16383)) -- Update maturin configuration. ([\#16394](https://github.com/matrix-org/synapse/issues/16394)) -- Downgrade replication stream time out error log lines to warning. ([\#16401](https://github.com/matrix-org/synapse/issues/16401)) - -### Updates to locked dependencies - -* Bump actions/checkout from 3 to 4. ([\#16250](https://github.com/matrix-org/synapse/issues/16250)) -* Bump cryptography from 41.0.3 to 41.0.4. ([\#16362](https://github.com/matrix-org/synapse/issues/16362)) -* Bump dawidd6/action-download-artifact from 2.27.0 to 2.28.0. ([\#16374](https://github.com/matrix-org/synapse/issues/16374)) -* Bump docker/setup-buildx-action from 2 to 3. ([\#16375](https://github.com/matrix-org/synapse/issues/16375)) -* Bump gitpython from 3.1.35 to 3.1.37. ([\#16376](https://github.com/matrix-org/synapse/issues/16376)) -* Bump msgpack from 1.0.5 to 1.0.6. ([\#16377](https://github.com/matrix-org/synapse/issues/16377)) -* Bump msgpack from 1.0.6 to 1.0.7. ([\#16412](https://github.com/matrix-org/synapse/issues/16412)) -* Bump phonenumbers from 8.13.19 to 8.13.22. ([\#16413](https://github.com/matrix-org/synapse/issues/16413)) -* Bump psycopg2 from 2.9.7 to 2.9.8. ([\#16409](https://github.com/matrix-org/synapse/issues/16409)) -* Bump pydantic from 2.3.0 to 2.4.2. ([\#16410](https://github.com/matrix-org/synapse/issues/16410)) -* Bump regex from 1.9.5 to 1.9.6. ([\#16408](https://github.com/matrix-org/synapse/issues/16408)) -* Bump sentry-sdk from 1.30.0 to 1.31.0. ([\#16378](https://github.com/matrix-org/synapse/issues/16378)) -* Bump types-netaddr from 0.8.0.9 to 0.9.0.1. ([\#16411](https://github.com/matrix-org/synapse/issues/16411)) -* Bump types-psycopg2 from 2.9.21.11 to 2.9.21.14. ([\#16381](https://github.com/matrix-org/synapse/issues/16381)) -* Bump urllib3 from 1.26.15 to 1.26.17. ([\#16422](https://github.com/matrix-org/synapse/issues/16422)) - -# Synapse 1.93.0 (2023-09-26) - -No significant changes since 1.93.0rc1. - - -## Security advisory - -The following issues are fixed in 1.93.0 (and RCs). - -- [GHSA-4f74-84v3-j9q5](https://github.com/matrix-org/synapse/security/advisories/GHSA-4f74-84v3-j9q5) / [CVE-2023-41335](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-41335) — Low Severity - - Temporary storage of plaintext passwords during password changes. - -- [GHSA-7565-cq32-vx2x](https://github.com/matrix-org/synapse/security/advisories/GHSA-7565-cq32-vx2x) / [CVE-2023-42453](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-42453) — Low Severity - - Improper validation of receipts allows forged read receipts. - -See the advisories for more details. If you have any questions, email security@matrix.org. - - -# Synapse 1.93.0rc1 (2023-09-19) - -### Features - -- Add automatic purge after all users have forgotten a room. ([\#15488](https://github.com/matrix-org/synapse/issues/15488)) -- Restore room purge/shutdown after a Synapse restart. ([\#15488](https://github.com/matrix-org/synapse/issues/15488)) -- Support resolving homeservers using `matrix-fed` DNS SRV records from [MSC4040](https://github.com/matrix-org/matrix-spec-proposals/pull/4040). ([\#16137](https://github.com/matrix-org/synapse/issues/16137)) -- Add the ability to use `G` (GiB) and `T` (TiB) suffixes in configuration options that refer to numbers of bytes. ([\#16219](https://github.com/matrix-org/synapse/issues/16219)) -- Add span information to requests sent to appservices. Contributed by MTRNord. ([\#16227](https://github.com/matrix-org/synapse/issues/16227)) -- Add the ability to enable/disable registrations when using CAS. Contributed by Aurélien Grimpard. ([\#16262](https://github.com/matrix-org/synapse/issues/16262)) -- Allow the `/notifications` endpoint to be routed to workers. ([\#16265](https://github.com/matrix-org/synapse/issues/16265)) -- Enable users to easily unsubscribe to notifications emails via the `List-Unsubscribe` header. ([\#16274](https://github.com/matrix-org/synapse/issues/16274)) -- Report whether a user is `locked` in the [List Accounts admin API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html#list-accounts), and exclude locked users by default. ([\#16328](https://github.com/matrix-org/synapse/issues/16328)) - -### Bugfixes - -- Fix a long-standing bug where multi-device accounts could cause high load due to presence. ([\#16066](https://github.com/matrix-org/synapse/issues/16066), [\#16170](https://github.com/matrix-org/synapse/issues/16170), [\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172), [\#16174](https://github.com/matrix-org/synapse/issues/16174)) -- Fix a long-standing bug where appservices using [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409) to receive `to_device` messages would only get messages for one user. ([\#16251](https://github.com/matrix-org/synapse/issues/16251)) -- Fix bug when using workers where Synapse could end up re-requesting the same remote device repeatedly. ([\#16252](https://github.com/matrix-org/synapse/issues/16252)) -- Fix long-standing bug where we kept re-requesting a remote server's key repeatedly, potentially causing delays in receiving events over federation. ([\#16257](https://github.com/matrix-org/synapse/issues/16257)) -- Avoid temporary storage of sensitive information. ([\#16272](https://github.com/matrix-org/synapse/issues/16272)) -- Fix bug introduced in Synapse 1.49.0 when using dehydrated devices ([MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697)) and refresh tokens. Contributed by Hanadi. ([\#16288](https://github.com/matrix-org/synapse/issues/16288)) -- Fix a long-standing bug where invalid receipts would be accepted. ([\#16327](https://github.com/matrix-org/synapse/issues/16327)) -- Use standard name for UTF-8 charset in emails. ([\#16329](https://github.com/matrix-org/synapse/issues/16329)) -- Don't try refetching device lists for users on remote hosts that are marked as "down". ([\#16298](https://github.com/matrix-org/synapse/issues/16298)) - -### Improved Documentation - -- Fix typos in the documentation. ([\#16282](https://github.com/matrix-org/synapse/issues/16282)) -- Link to the Alpine Linux community package for Synapse. ([\#16304](https://github.com/matrix-org/synapse/issues/16304)) -- Use string for `federation_client_minimum_tls_version` documentation examples. Contributed by @jcgruenhage. ([\#16353](https://github.com/matrix-org/synapse/issues/16353)) - -### Internal Changes - -- Allow modules to delete rooms. ([\#15997](https://github.com/matrix-org/synapse/issues/15997)) -- Add GCC and GNU Make to the Nix flake development environment so that `ruff` can be compiled. ([\#16090](https://github.com/matrix-org/synapse/issues/16090), [\#16263](https://github.com/matrix-org/synapse/issues/16263)) -- Fix type checking when using the new version of Twisted. ([\#16235](https://github.com/matrix-org/synapse/issues/16235)) -- Delete device messages asynchronously and in staged batches using the task scheduler. ([\#16240](https://github.com/matrix-org/synapse/issues/16240), [\#16311](https://github.com/matrix-org/synapse/issues/16311), [\#16312](https://github.com/matrix-org/synapse/issues/16312), [\#16313](https://github.com/matrix-org/synapse/issues/16313)) -- Bump minimum supported Rust version to 1.61.0. ([\#16248](https://github.com/matrix-org/synapse/issues/16248)) -- Update rust to version 1.71.1 in the nix development environment. ([\#16260](https://github.com/matrix-org/synapse/issues/16260)) -- Simplify server key storage. ([\#16261](https://github.com/matrix-org/synapse/issues/16261)) -- Reduce CPU overhead of change password endpoint. ([\#16264](https://github.com/matrix-org/synapse/issues/16264)) -- Stop purging from tables slated for removal. ([\#16273](https://github.com/matrix-org/synapse/issues/16273)) -- Improve type hints. ([\#16276](https://github.com/matrix-org/synapse/issues/16276), [\#16301](https://github.com/matrix-org/synapse/issues/16301), [\#16325](https://github.com/matrix-org/synapse/issues/16325), [\#16326](https://github.com/matrix-org/synapse/issues/16326)) -- Raise `setuptools_rust` version cap to 1.7.0. ([\#16277](https://github.com/matrix-org/synapse/issues/16277)) -- Fix using the new task scheduler causing lots of CPU to be used. ([\#16278](https://github.com/matrix-org/synapse/issues/16278)) -- Upgrade CI run of Python 3.12 from rc1 to rc2. ([\#16280](https://github.com/matrix-org/synapse/issues/16280)) -- Include values in SQL debug when using `execute_values` with Postgres. ([\#16281](https://github.com/matrix-org/synapse/issues/16281)) -- Enable additional linting checks. ([\#16283](https://github.com/matrix-org/synapse/issues/16283)) -- Refactor `receipts_graph` Postgres transactions to stop error messages. ([\#16299](https://github.com/matrix-org/synapse/issues/16299)) -- Small improvements to logging in replication code. ([\#16309](https://github.com/matrix-org/synapse/issues/16309)) -- Remove a reference cycle in background processes. ([\#16314](https://github.com/matrix-org/synapse/issues/16314)) -- Only use literal strings for background process names. ([\#16315](https://github.com/matrix-org/synapse/issues/16315)) -- Refactor `get_user_by_id`. ([\#16316](https://github.com/matrix-org/synapse/issues/16316)) -- Speed up task to delete to-device messages. ([\#16318](https://github.com/matrix-org/synapse/issues/16318)) -- Avoid patching code in tests. ([\#16349](https://github.com/matrix-org/synapse/issues/16349)) -- Test against PostgreSQL 16. ([\#16351](https://github.com/matrix-org/synapse/issues/16351)) - -### Updates to locked dependencies - -* Bump mypy from 1.4.1 to 1.5.1. ([\#16300](https://github.com/matrix-org/synapse/issues/16300)) -* Bump black from 23.7.0 to 23.9.1. ([\#16295](https://github.com/matrix-org/synapse/issues/16295)) -* Bump docker/build-push-action from 4 to 5. ([\#16336](https://github.com/matrix-org/synapse/issues/16336)) -* Bump docker/login-action from 2 to 3. ([\#16339](https://github.com/matrix-org/synapse/issues/16339)) -* Bump docker/metadata-action from 4 to 5. ([\#16337](https://github.com/matrix-org/synapse/issues/16337)) -* Bump docker/setup-qemu-action from 2 to 3. ([\#16338](https://github.com/matrix-org/synapse/issues/16338)) -* Bump furo from 2023.8.19 to 2023.9.10. ([\#16340](https://github.com/matrix-org/synapse/issues/16340)) -* Bump gitpython from 3.1.32 to 3.1.35. ([\#16267](https://github.com/matrix-org/synapse/issues/16267), [\#16279](https://github.com/matrix-org/synapse/issues/16279)) -* Bump mypy-zope from 1.0.0 to 1.0.1. ([\#16291](https://github.com/matrix-org/synapse/issues/16291)) -* Bump pillow from 10.0.0 to 10.0.1. ([\#16344](https://github.com/matrix-org/synapse/issues/16344)) -* Bump regex from 1.9.4 to 1.9.5. ([\#16233](https://github.com/matrix-org/synapse/issues/16233)) -* Bump ruff from 0.0.286 to 0.0.290. ([\#16342](https://github.com/matrix-org/synapse/issues/16342)) -* Bump serde_json from 1.0.105 to 1.0.107. ([\#16296](https://github.com/matrix-org/synapse/issues/16296), [\#16345](https://github.com/matrix-org/synapse/issues/16345)) -* Bump twisted from 22.10.0 to 23.8.0. ([\#16235](https://github.com/matrix-org/synapse/issues/16235)) -* Bump types-pillow from 10.0.0.2 to 10.0.0.3. ([\#16293](https://github.com/matrix-org/synapse/issues/16293)) -* Bump types-setuptools from 68.0.0.3 to 68.2.0.0. ([\#16292](https://github.com/matrix-org/synapse/issues/16292)) -* Bump typing-extensions from 4.7.1 to 4.8.0. ([\#16341](https://github.com/matrix-org/synapse/issues/16341)) - -# Synapse 1.92.3 (2023-09-18) - -This is again a security update targeted at mitigating [CVE-2023-4863](https://cve.org/CVERecord?id=CVE-2023-4863). -It turns out that libwebp is bundled statically in Pillow wheels so we need to update this dependency instead of -libwebp package at the OS level. - -Unlike what was advertised in 1.92.2 changelog this release also impacts PyPI wheels and Debian packages from matrix.org. - -We encourage admins to upgrade as soon as possible. - - -### Internal Changes - -- Pillow 10.0.1 is now mandatory because of libwebp CVE-2023-4863, since Pillow provides libwebp in the wheels. ([\#16347](https://github.com/matrix-org/synapse/issues/16347)) - -### Updates to locked dependencies - -* Bump pillow from 10.0.0 to 10.0.1. ([\#16344](https://github.com/matrix-org/synapse/issues/16344)) - -# Synapse 1.92.2 (2023-09-15) - -This is a Docker-only update to mitigate [CVE-2023-4863](https://cve.org/CVERecord?id=CVE-2023-4863), a critical vulnerability in `libwebp`. Server admins not using Docker should ensure that their `libwebp` is up to date (if installed). We encourage admins to upgrade as soon as possible. - +- Add index to sliding sync ([MSC4186](https://github.com/matrix-org/matrix-doc/pull/4186)) membership snapshot table, to fix a performance issue. ([\#18074](https://github.com/element-hq/synapse/issues/18074)) ### Updates to the Docker image -- Update docker image to use Debian bookworm as the base. ([\#16324](https://github.com/matrix-org/synapse/issues/16324)) - - -# Synapse 1.92.1 (2023-09-12) - -This minor release was needed only because of CI-related trouble on [v1.92.0](https://github.com/matrix-org/synapse/releases/tag/v1.92.0), which was never released. - -### Internal Changes - -- Stop building Ubuntu Kinetic since it is EOL and repos seem to be dead. - - -# Synapse 1.92.0 (2023-09-12) - -This release includes the same [bugfix](https://github.com/matrix-org/synapse/issues/16258) as Synapse 1.91.2. - -This version was never released following a CI build failure, cf [v1.92.1 changelog](https://github.com/matrix-org/synapse/releases/tag/v1.92.1). - -### Bugfixes - -- Revert [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) introspection cache, admin impersonation and account lock. ([\#16258](https://github.com/matrix-org/synapse/issues/16258)) - -### Internal Changes - -- Fix incorrect docstring for `Ratelimiter`. ([\#16255](https://github.com/matrix-org/synapse/issues/16255)) -- Update the release script to work on macOS. ([\#16266](https://github.com/matrix-org/synapse/issues/16266)) - - -# Synapse 1.91.2 (2023-09-06) - -### Bugfixes - -- Revert [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) introspection cache, admin impersonation and account lock. ([\#16258](https://github.com/matrix-org/synapse/issues/16258)) - - -# Synapse 1.92.0rc1 (2023-09-05) - -### Features - -- Add configuration setting for CAS protocol version. Contributed by Aurélien Grimpard. ([\#15816](https://github.com/matrix-org/synapse/issues/15816)) -- Suppress notifications from message edits per [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958). ([\#16113](https://github.com/matrix-org/synapse/issues/16113)) -- Experimental support for [MSC4041](https://github.com/matrix-org/matrix-spec-proposals/pull/4041): return a `Retry-After` header with `M_LIMIT_EXCEEDED` error responses. ([\#16136](https://github.com/matrix-org/synapse/issues/16136)) -- Add `last_seen_ts` to the [admin users API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html). ([\#16218](https://github.com/matrix-org/synapse/issues/16218)) -- Improve resource usage when sending data to a large number of remote hosts that are marked as "down". ([\#16223](https://github.com/matrix-org/synapse/issues/16223)) - -### Bugfixes - -- Fix IPv6-related bugs on SMTP settings, adding groundwork to fix similar issues. Contributed by @evilham and @telmich (ungleich.ch). ([\#16155](https://github.com/matrix-org/synapse/issues/16155)) -- Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `include_all_networks` as a string. ([\#16185](https://github.com/matrix-org/synapse/issues/16185)) -- Fix inaccurate error message while attempting to ban or unban a user with the same or higher PL by spliting the conditional statements. Contributed by @leviosacz. ([\#16205](https://github.com/matrix-org/synapse/issues/16205)) -- Fix a rare bug that broke looping calls, which could lead to e.g. linearly increasing memory usage. Introduced in v1.90.0. ([\#16210](https://github.com/matrix-org/synapse/issues/16210)) -- Fix a long-standing bug where uploading images would fail if we could not generate thumbnails for them. ([\#16211](https://github.com/matrix-org/synapse/issues/16211)) -- Fix a long-standing bug where we did not correctly back off from servers that had "gone" if they returned 4xx series error codes. ([\#16221](https://github.com/matrix-org/synapse/issues/16221)) +- Specify the architecture of installed packages via an APT config option, which is more reliable than appending package names with `:{arch}`. ([\#18271](https://github.com/element-hq/synapse/issues/18271)) +- Always specify base image debian versions with a build argument. ([\#18272](https://github.com/element-hq/synapse/issues/18272)) +- Allow passing arguments to `start_for_complement.sh` (to be sent to `configure_workers_and_start.py`). ([\#18273](https://github.com/element-hq/synapse/issues/18273)) +- Make some improvements to the `prefix-log` script in the workers image. ([\#18274](https://github.com/element-hq/synapse/issues/18274)) +- Use `uv pip` to install `supervisor` in the worker image. ([\#18275](https://github.com/element-hq/synapse/issues/18275)) +- Avoid needing to download & use `rsync` in a build layer. ([\#18287](https://github.com/element-hq/synapse/issues/18287)) ### Improved Documentation -- Update links to the [matrix.org blog](https://matrix.org/blog/). ([\#16008](https://github.com/matrix-org/synapse/issues/16008)) -- Document which [admin APIs](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) are disabled when experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support is enabled. ([\#16168](https://github.com/matrix-org/synapse/issues/16168)) -- Document [`exclude_rooms_from_sync`](https://matrix-org.github.io/synapse/v1.92/usage/configuration/config_documentation.html#exclude_rooms_from_sync) configuration option. ([\#16178](https://github.com/matrix-org/synapse/issues/16178)) +- Fix how to obtain access token and change naming from riot to element ([\#18225](https://github.com/element-hq/synapse/issues/18225)) +- Correct a small typo in the SSO mapping providers documentation. ([\#18276](https://github.com/element-hq/synapse/issues/18276)) +- Add docs for how to clear out the Poetry wheel cache. ([\#18283](https://github.com/element-hq/synapse/issues/18283)) ### Internal Changes -- Prepare unit tests for Python 3.12. ([\#16099](https://github.com/matrix-org/synapse/issues/16099)) -- Fix nightly CI jobs. ([\#16121](https://github.com/matrix-org/synapse/issues/16121), [\#16213](https://github.com/matrix-org/synapse/issues/16213)) -- Describe which rate limiter was hit in logs. ([\#16135](https://github.com/matrix-org/synapse/issues/16135)) -- Simplify presence code when using workers. ([\#16170](https://github.com/matrix-org/synapse/issues/16170)) -- Track per-device information in the presence code. ([\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172)) -- Stop using the `event_txn_id` table. ([\#16175](https://github.com/matrix-org/synapse/issues/16175)) -- Use `AsyncMock` instead of custom code. ([\#16179](https://github.com/matrix-org/synapse/issues/16179), [\#16180](https://github.com/matrix-org/synapse/issues/16180)) -- Improve error reporting of invalid data passed to `/_matrix/key/v2/query`. ([\#16183](https://github.com/matrix-org/synapse/issues/16183)) -- Task scheduler: add replication notify for new task to launch ASAP. ([\#16184](https://github.com/matrix-org/synapse/issues/16184)) -- Improve type hints. ([\#16186](https://github.com/matrix-org/synapse/issues/16186), [\#16188](https://github.com/matrix-org/synapse/issues/16188), [\#16201](https://github.com/matrix-org/synapse/issues/16201)) -- Bump black version to 23.7.0. ([\#16187](https://github.com/matrix-org/synapse/issues/16187)) -- Log the details of background update failures. ([\#16212](https://github.com/matrix-org/synapse/issues/16212)) -- Cache device resync requests over replication. ([\#16241](https://github.com/matrix-org/synapse/issues/16241)) +- Add a column `participant` to `room_memberships` table. ([\#18068](https://github.com/element-hq/synapse/issues/18068)) +- Update Poetry to 2.1.1, including updating the lock file version. ([\#18251](https://github.com/element-hq/synapse/issues/18251)) +- Pin GitHub Actions dependencies by commit hash. ([\#18255](https://github.com/element-hq/synapse/issues/18255)) +- Add DB delta to remove the old state group deletion job. ([\#18284](https://github.com/element-hq/synapse/issues/18284)) + + ### Updates to locked dependencies -* Bump anyhow from 1.0.72 to 1.0.75. ([\#16141](https://github.com/matrix-org/synapse/issues/16141)) -* Bump furo from 2023.7.26 to 2023.8.19. ([\#16238](https://github.com/matrix-org/synapse/issues/16238)) -* Bump phonenumbers from 8.13.18 to 8.13.19. ([\#16237](https://github.com/matrix-org/synapse/issues/16237)) -* Bump psycopg2 from 2.9.6 to 2.9.7. ([\#16196](https://github.com/matrix-org/synapse/issues/16196)) -* Bump regex from 1.9.3 to 1.9.4. ([\#16195](https://github.com/matrix-org/synapse/issues/16195)) -* Bump ruff from 0.0.277 to 0.0.286. ([\#16198](https://github.com/matrix-org/synapse/issues/16198)) -* Bump sentry-sdk from 1.29.2 to 1.30.0. ([\#16236](https://github.com/matrix-org/synapse/issues/16236)) -* Bump serde from 1.0.184 to 1.0.188. ([\#16194](https://github.com/matrix-org/synapse/issues/16194)) -* Bump serde_json from 1.0.104 to 1.0.105. ([\#16140](https://github.com/matrix-org/synapse/issues/16140)) -* Bump types-psycopg2 from 2.9.21.10 to 2.9.21.11. ([\#16200](https://github.com/matrix-org/synapse/issues/16200)) -* Bump types-pyyaml from 6.0.12.10 to 6.0.12.11. ([\#16199](https://github.com/matrix-org/synapse/issues/16199)) +* Bump actions/add-to-project from f5473ace9aeee8b97717b281e26980aa5097023f to 280af8ae1f83a494cfad2cb10f02f6d13529caa9. ([\#18303](https://github.com/element-hq/synapse/issues/18303)) +* Bump actions/cache from 4.2.2 to 4.2.3. ([\#18266](https://github.com/element-hq/synapse/issues/18266)) +* Bump actions/download-artifact from 4.2.0 to 4.2.1. ([\#18268](https://github.com/element-hq/synapse/issues/18268)) +* Bump actions/setup-python from 5.4.0 to 5.5.0. ([\#18298](https://github.com/element-hq/synapse/issues/18298)) +* Bump actions/upload-artifact from 4.6.1 to 4.6.2. ([\#18304](https://github.com/element-hq/synapse/issues/18304)) +* Bump authlib from 1.4.1 to 1.5.1. ([\#18306](https://github.com/element-hq/synapse/issues/18306)) +* Bump dawidd6/action-download-artifact from 8 to 9. ([\#18204](https://github.com/element-hq/synapse/issues/18204)) +* Bump jinja2 from 3.1.5 to 3.1.6. ([\#18223](https://github.com/element-hq/synapse/issues/18223)) +* Bump log from 0.4.26 to 0.4.27. ([\#18267](https://github.com/element-hq/synapse/issues/18267)) +* Bump phonenumbers from 8.13.50 to 9.0.2. ([\#18299](https://github.com/element-hq/synapse/issues/18299)) +* Bump pygithub from 2.5.0 to 2.6.1. ([\#18243](https://github.com/element-hq/synapse/issues/18243)) +* Bump pyo3-log from 0.12.1 to 0.12.2. ([\#18269](https://github.com/element-hq/synapse/issues/18269)) + +# Synapse 1.127.1 (2025-03-26) + +## Security +- Fix [CVE-2025-30355](https://www.cve.org/CVERecord?id=CVE-2025-30355) / [GHSA-v56r-hwv5-mxg6](https://github.com/element-hq/synapse/security/advisories/GHSA-v56r-hwv5-mxg6). **High severity vulnerability affecting federation. The vulnerability has been exploited in the wild.** -# Synapse 1.91.1 (2023-09-04) -### Bugfixes +# Synapse 1.127.0 (2025-03-25) -- Fix a performance regression introduced in Synapse 1.91.0 where event persistence would cause an excessive linear growth in CPU usage. ([\#16220](https://github.com/matrix-org/synapse/issues/16220)) +No significant changes since 1.127.0rc1. -# Synapse 1.91.0 (2023-08-30) - -No significant changes since 1.91.0rc1. -# Synapse 1.91.0rc1 (2023-08-23) +# Synapse 1.127.0rc1 (2025-03-18) ### Features -- Implements an admin API to lock an user without deactivating them. Based on [MSC3939](https://github.com/matrix-org/matrix-spec-proposals/pull/3939). ([\#15870](https://github.com/matrix-org/synapse/issues/15870)) -- Implements a task scheduler for resumable potentially long running tasks. ([\#15891](https://github.com/matrix-org/synapse/issues/15891)) -- Allow specifying `client_secret_path` as alternative to `client_secret` for OIDC providers. This avoids leaking the client secret in the homeserver config. Contributed by @Ma27. ([\#16030](https://github.com/matrix-org/synapse/issues/16030)) -- Allow customising the IdP display name, icon, and brand for SAML and CAS providers (in addition to OIDC provider). ([\#16094](https://github.com/matrix-org/synapse/issues/16094)) -- Add an `admins` query parameter to the [List Accounts](https://matrix-org.github.io/synapse/v1.91/admin_api/user_admin_api.html#list-accounts) [admin API](https://matrix-org.github.io/synapse/v1.91/usage/administration/admin_api/index.html), to include only admins or to exclude admins in user queries. ([\#16114](https://github.com/matrix-org/synapse/issues/16114)) - -### Bugfixes - -- Fix long-standing bug where concurrent requests to change a user's push rules could cause a deadlock. Contributed by Nick @ Beeper (@fizzadar). ([\#16052](https://github.com/matrix-org/synapse/issues/16052)) -- Fix a long-standing bu in `/sync` where timeout=0 does not skip caching, resulting in slow calls in cases where there are no new changes. Contributed by @PlasmaIntec. ([\#16080](https://github.com/matrix-org/synapse/issues/16080)) -- Fix performance of state resolutions for large, old rooms that did not have the full auth chain persisted. ([\#16116](https://github.com/matrix-org/synapse/issues/16116)) -- Filter out user agent references to the sliding sync proxy and rust-sdk from the user_daily_visits table to ensure that Element X can be represented fully. ([\#16124](https://github.com/matrix-org/synapse/issues/16124)) -- User constent and 3-PID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support. ([\#16127](https://github.com/matrix-org/synapse/issues/16127), [\#16134](https://github.com/matrix-org/synapse/issues/16134)) -- Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0. ([\#16133](https://github.com/matrix-org/synapse/issues/16133), [\#16169](https://github.com/matrix-org/synapse/issues/16169)) -- Fix performance degredation when there are a lot of in-flight replication requests. ([\#16148](https://github.com/matrix-org/synapse/issues/16148)) -- Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico. ([\#16156](https://github.com/matrix-org/synapse/issues/16156), [\#16164](https://github.com/matrix-org/synapse/issues/16164)) +- Update [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140) implementation to no longer cancel a user's own delayed state events with an event type & state key that match a more recent state event sent by that user. ([\#17810](https://github.com/element-hq/synapse/issues/17810)) ### Improved Documentation -- Structured logging docs: add a link to explain the ELK stack ([\#16091](https://github.com/matrix-org/synapse/issues/16091)) +- Fixed a minor typo in the Synapse documentation. Contributed by @karuto12. ([\#18224](https://github.com/element-hq/synapse/issues/18224)) ### Internal Changes -- Update dehydrated devices implementation. ([\#16010](https://github.com/matrix-org/synapse/issues/16010)) -- Fix database performance of read/write worker locks. ([\#16061](https://github.com/matrix-org/synapse/issues/16061)) -- Fix building the nix development environment on MacOS systems. ([\#16063](https://github.com/matrix-org/synapse/issues/16063)) -- Override global statement timeout when creating indexes in Postgres. ([\#16085](https://github.com/matrix-org/synapse/issues/16085)) -- Fix the type annotation on `run_db_interaction` in the Module API. ([\#16089](https://github.com/matrix-org/synapse/issues/16089)) -- Clean-up the presence code. ([\#16092](https://github.com/matrix-org/synapse/issues/16092)) -- Run `pyupgrade` for Python 3.8+. ([\#16110](https://github.com/matrix-org/synapse/issues/16110)) -- Rename pagination and purge locks and add comments to explain why they exist and how they work. ([\#16112](https://github.com/matrix-org/synapse/issues/16112)) -- Attempt to fix the twisted trunk job. ([\#16115](https://github.com/matrix-org/synapse/issues/16115)) -- Cache token introspection response from OIDC provider. ([\#16117](https://github.com/matrix-org/synapse/issues/16117)) -- Add cache to `get_server_keys_json_for_remote`. ([\#16123](https://github.com/matrix-org/synapse/issues/16123)) -- Add an admin endpoint to allow authorizing server to signal token revocations. ([\#16125](https://github.com/matrix-org/synapse/issues/16125)) -- Add response time metrics for introspection requests for delegated auth. ([\#16131](https://github.com/matrix-org/synapse/issues/16131)) -- MSC3861: allow impersonation by an admin user using `_oidc_admin_impersonate_user_id` query parameter. ([\#16132](https://github.com/matrix-org/synapse/issues/16132)) -- Increase performance of read/write locks. ([\#16149](https://github.com/matrix-org/synapse/issues/16149)) -- Improve presence tests. ([\#16150](https://github.com/matrix-org/synapse/issues/16150), [\#16151](https://github.com/matrix-org/synapse/issues/16151), [\#16158](https://github.com/matrix-org/synapse/issues/16158)) -- Raised the poetry-core version cap to 1.7.0. ([\#16152](https://github.com/matrix-org/synapse/issues/16152)) -- Fix assertion in user directory unit tests. ([\#16157](https://github.com/matrix-org/synapse/issues/16157)) -- Reduce scope of locks when paginating to alleviate DB contention. ([\#16159](https://github.com/matrix-org/synapse/issues/16159)) -- Reduce DB contention on worker locks. ([\#16160](https://github.com/matrix-org/synapse/issues/16160)) -- Task scheduler: mark task as active if we are scheduling as soon as possible. ([\#16165](https://github.com/matrix-org/synapse/issues/16165)) +- Remove undocumented `SYNAPSE_USE_FROZEN_DICTS` environment variable. ([\#18123](https://github.com/element-hq/synapse/issues/18123)) +- Fix detection of workflow failures in the release script. ([\#18211](https://github.com/element-hq/synapse/issues/18211)) +- Add caching support to media endpoints. ([\#18235](https://github.com/element-hq/synapse/issues/18235)) + + ### Updates to locked dependencies -* Bump click from 8.1.6 to 8.1.7. ([\#16145](https://github.com/matrix-org/synapse/issues/16145)) -* Bump gitpython from 3.1.31 to 3.1.32. ([\#16103](https://github.com/matrix-org/synapse/issues/16103)) -* Bump ijson from 3.2.1 to 3.2.3. ([\#16143](https://github.com/matrix-org/synapse/issues/16143)) -* Bump isort from 5.11.5 to 5.12.0. ([\#16108](https://github.com/matrix-org/synapse/issues/16108)) -* Bump log from 0.4.19 to 0.4.20. ([\#16109](https://github.com/matrix-org/synapse/issues/16109)) -* Bump pygithub from 1.59.0 to 1.59.1. ([\#16144](https://github.com/matrix-org/synapse/issues/16144)) -* Bump sentry-sdk from 1.28.1 to 1.29.2. ([\#16142](https://github.com/matrix-org/synapse/issues/16142)) -* Bump serde from 1.0.183 to 1.0.184. ([\#16139](https://github.com/matrix-org/synapse/issues/16139)) -* Bump txredisapi from 1.4.9 to 1.4.10. ([\#16107](https://github.com/matrix-org/synapse/issues/16107)) -* Bump types-bleach from 6.0.0.3 to 6.0.0.4. ([\#16106](https://github.com/matrix-org/synapse/issues/16106)) -* Bump types-pillow from 10.0.0.1 to 10.0.0.2. ([\#16105](https://github.com/matrix-org/synapse/issues/16105)) -* Bump types-pyopenssl from 23.2.0.1 to 23.2.0.2. ([\#16146](https://github.com/matrix-org/synapse/issues/16146)) +* Bump anyhow from 1.0.96 to 1.0.97. ([\#18201](https://github.com/element-hq/synapse/issues/18201)) +* Bump bcrypt from 4.2.1 to 4.3.0. ([\#18207](https://github.com/element-hq/synapse/issues/18207)) +* Bump bytes from 1.10.0 to 1.10.1. ([\#18227](https://github.com/element-hq/synapse/issues/18227)) +* Bump http from 1.2.0 to 1.3.1. ([\#18245](https://github.com/element-hq/synapse/issues/18245)) +* Bump sentry-sdk from 2.19.2 to 2.22.0. ([\#18205](https://github.com/element-hq/synapse/issues/18205)) +* Bump serde from 1.0.218 to 1.0.219. ([\#18228](https://github.com/element-hq/synapse/issues/18228)) +* Bump serde_json from 1.0.139 to 1.0.140. ([\#18202](https://github.com/element-hq/synapse/issues/18202)) +* Bump ulid from 1.2.0 to 1.2.1. ([\#18246](https://github.com/element-hq/synapse/issues/18246)) -# Synapse 1.91.0rc1 (2023-08-23) +# Synapse 1.126.0 (2025-03-11) +Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check +[the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.126/docs/upgrade.md#change-of-signing-key-expiry-date-for-the-debianubuntu-package-repository) +as we have recently updated the expiry date on the repository's GPG signing key. The old version of the key will expire on `2025-03-15`. -### Features +No significant changes since 1.126.0rc3. -- Implements an admin API to lock an user without deactivating them. Based on [MSC3939](https://github.com/matrix-org/matrix-spec-proposals/pull/3939). ([\#15870](https://github.com/matrix-org/synapse/issues/15870)) -- Allow specifying `client_secret_path` as alternative to `client_secret` for OIDC providers. This avoids leaking the client secret in the homeserver config. Contributed by @Ma27. ([\#16030](https://github.com/matrix-org/synapse/issues/16030)) -- Allow customising the IdP display name, icon, and brand for SAML and CAS providers (in addition to OIDC provider). ([\#16094](https://github.com/matrix-org/synapse/issues/16094)) -- Add an `admins` query parameter to the [List Accounts](https://matrix-org.github.io/synapse/v1.91/admin_api/user_admin_api.html#list-accounts) [admin API](https://matrix-org.github.io/synapse/v1.91/usage/administration/admin_api/index.html), to include only admins or to exclude admins in user queries. ([\#16114](https://github.com/matrix-org/synapse/issues/16114)) + + + +# Synapse 1.126.0rc3 (2025-03-07) ### Bugfixes -- Fix long-standing bug where concurrent requests to change a user's push rules could cause a deadlock. Contributed by Nick @ Beeper (@fizzadar). ([\#16052](https://github.com/matrix-org/synapse/issues/16052)) -- Fix a long-standing bug in `/sync` where timeout=0 does not skip caching, resulting in slow calls in cases where there are no new changes. Contributed by @PlasmaIntec. ([\#16080](https://github.com/matrix-org/synapse/issues/16080)) -- Fix performance of state resolutions for large, old rooms that did not have the full auth chain persisted. ([\#16116](https://github.com/matrix-org/synapse/issues/16116)) -- Filter out user agent references to the sliding sync proxy and rust-sdk from the `user_daily_visits` table to ensure that Element X can be represented fully. ([\#16124](https://github.com/matrix-org/synapse/issues/16124)) -- User constent and third-party ID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support. ([\#16127](https://github.com/matrix-org/synapse/issues/16127), [\#16134](https://github.com/matrix-org/synapse/issues/16134)) -- Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0. ([\#16133](https://github.com/matrix-org/synapse/issues/16133), [\#16169](https://github.com/matrix-org/synapse/issues/16169)) -- Fix performance degredation when there are a lot of in-flight replication requests. ([\#16148](https://github.com/matrix-org/synapse/issues/16148)) -- Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico. ([\#16156](https://github.com/matrix-org/synapse/issues/16156), [\#16164](https://github.com/matrix-org/synapse/issues/16164)) +- Revert the background job to clear unreferenced state groups (that was introduced in v1.126.0rc1), due to [a suspected issue](https://github.com/element-hq/synapse/issues/18217) that causes increased disk usage. ([\#18222](https://github.com/element-hq/synapse/issues/18222)) -### Improved Documentation -- Structured logging docs: add a link to explain the ELK stack ([\#16091](https://github.com/matrix-org/synapse/issues/16091)) + + +# Synapse 1.126.0rc2 (2025-03-05) + ### Internal Changes -- Update dehydrated devices implementation. ([\#16010](https://github.com/matrix-org/synapse/issues/16010)) -- Fix database performance of read/write worker locks. ([\#16061](https://github.com/matrix-org/synapse/issues/16061)) -- Fix building the nix development environment on MacOS systems. ([\#16063](https://github.com/matrix-org/synapse/issues/16063)) -- Override global statement timeout when creating indexes in Postgres. ([\#16085](https://github.com/matrix-org/synapse/issues/16085)) -- Fix the type annotation on `run_db_interaction` in the Module API. ([\#16089](https://github.com/matrix-org/synapse/issues/16089)) -- Clean-up the presence code. ([\#16092](https://github.com/matrix-org/synapse/issues/16092)) -- Run `pyupgrade` for Python 3.8+. ([\#16110](https://github.com/matrix-org/synapse/issues/16110)) -- Rename pagination and purge locks and add comments to explain why they exist and how they work. ([\#16112](https://github.com/matrix-org/synapse/issues/16112)) -- Attempt to fix the twisted trunk job. ([\#16115](https://github.com/matrix-org/synapse/issues/16115)) -- Cache token introspection response from OIDC provider. ([\#16117](https://github.com/matrix-org/synapse/issues/16117)) -- Add cache to `get_server_keys_json_for_remote`. ([\#16123](https://github.com/matrix-org/synapse/issues/16123)) -- Add an admin endpoint to allow authorizing server to signal token revocations. ([\#16125](https://github.com/matrix-org/synapse/issues/16125)) -- Add response time metrics for introspection requests for delegated auth. ([\#16131](https://github.com/matrix-org/synapse/issues/16131)) -- [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861): allow impersonation by an admin user using `_oidc_admin_impersonate_user_id` query parameter. ([\#16132](https://github.com/matrix-org/synapse/issues/16132)) -- Increase performance of read/write locks. ([\#16149](https://github.com/matrix-org/synapse/issues/16149)) -- Improve presence tests. ([\#16150](https://github.com/matrix-org/synapse/issues/16150), [\#16151](https://github.com/matrix-org/synapse/issues/16151), [\#16158](https://github.com/matrix-org/synapse/issues/16158)) -- Raised the poetry-core version cap to 1.7.0. ([\#16152](https://github.com/matrix-org/synapse/issues/16152)) -- Fix assertion in user directory unit tests. ([\#16157](https://github.com/matrix-org/synapse/issues/16157)) -- Reduce scope of locks when paginating to alleviate DB contention. ([\#16159](https://github.com/matrix-org/synapse/issues/16159)) -- Reduce DB contention on worker locks. ([\#16160](https://github.com/matrix-org/synapse/issues/16160)) -- Task scheduler: mark task as active if we are scheduling as soon as possible. ([\#16165](https://github.com/matrix-org/synapse/issues/16165)) -- Implements a task scheduler for resumable potentially long running tasks. ([\#15891](https://github.com/matrix-org/synapse/issues/15891)) +- Fix wheel building configuration in CI by installing libatomic1. ([\#18212](https://github.com/element-hq/synapse/issues/18212), [\#18213](https://github.com/element-hq/synapse/issues/18213)) -### Updates to locked dependencies +# Synapse 1.126.0rc1 (2025-03-04) -* Bump click from 8.1.6 to 8.1.7. ([\#16145](https://github.com/matrix-org/synapse/issues/16145)) -* Bump gitpython from 3.1.31 to 3.1.32. ([\#16103](https://github.com/matrix-org/synapse/issues/16103)) -* Bump ijson from 3.2.1 to 3.2.3. ([\#16143](https://github.com/matrix-org/synapse/issues/16143)) -* Bump isort from 5.11.5 to 5.12.0. ([\#16108](https://github.com/matrix-org/synapse/issues/16108)) -* Bump log from 0.4.19 to 0.4.20. ([\#16109](https://github.com/matrix-org/synapse/issues/16109)) -* Bump pygithub from 1.59.0 to 1.59.1. ([\#16144](https://github.com/matrix-org/synapse/issues/16144)) -* Bump sentry-sdk from 1.28.1 to 1.29.2. ([\#16142](https://github.com/matrix-org/synapse/issues/16142)) -* Bump serde from 1.0.183 to 1.0.184. ([\#16139](https://github.com/matrix-org/synapse/issues/16139)) -* Bump txredisapi from 1.4.9 to 1.4.10. ([\#16107](https://github.com/matrix-org/synapse/issues/16107)) -* Bump types-bleach from 6.0.0.3 to 6.0.0.4. ([\#16106](https://github.com/matrix-org/synapse/issues/16106)) -* Bump types-pillow from 10.0.0.1 to 10.0.0.2. ([\#16105](https://github.com/matrix-org/synapse/issues/16105)) -* Bump types-pyopenssl from 23.2.0.1 to 23.2.0.2. ([\#16146](https://github.com/matrix-org/synapse/issues/16146)) - -# Synapse 1.90.0 (2023-08-15) - -No significant changes since 1.90.0rc1. - - -# Synapse 1.90.0rc1 (2023-08-08) +Synapse 1.126.0rc1 was not fully released due to an error in CI. ### Features -- Scope transaction IDs to devices (implement [MSC3970](https://github.com/matrix-org/matrix-spec-proposals/pull/3970)). ([\#15629](https://github.com/matrix-org/synapse/issues/15629)) -- Remove old rows from the `cache_invalidation_stream_by_instance` table automatically (this table is unused in SQLite). ([\#15868](https://github.com/matrix-org/synapse/issues/15868)) +- Define ratelimit configuration for delayed event management. ([\#18019](https://github.com/element-hq/synapse/issues/18019)) +- Add `form_secret_path` config option. ([\#18090](https://github.com/element-hq/synapse/issues/18090)) +- Add the `--no-secrets-in-config` command line option. ([\#18092](https://github.com/element-hq/synapse/issues/18092)) +- Add background job to clear unreferenced state groups. ([\#18154](https://github.com/element-hq/synapse/issues/18154)) +- Add support for specifying/overriding `id_token_signing_alg_values_supported` for an OpenID identity provider. ([\#18177](https://github.com/element-hq/synapse/issues/18177)) +- Add `worker_replication_secret_path` config option. ([\#18191](https://github.com/element-hq/synapse/issues/18191)) +- Add support for specifying/overriding `redirect_uri` in the authorization and token requests against an OpenID identity provider. ([\#18197](https://github.com/element-hq/synapse/issues/18197)) ### Bugfixes -- Fix a long-standing bug where purging history and paginating simultaneously could lead to database corruption when using workers. ([\#15791](https://github.com/matrix-org/synapse/issues/15791)) -- Fix a long-standing bug where profile endpoint returned a 404 when the user's display name was empty. ([\#16012](https://github.com/matrix-org/synapse/issues/16012)) -- Fix a long-standing bug where the `synapse_port_db` failed to configure sequences for application services and partial stated rooms. ([\#16043](https://github.com/matrix-org/synapse/issues/16043)) -- Fix long-standing bug with deletion in dehydrated devices v2. ([\#16046](https://github.com/matrix-org/synapse/issues/16046)) +- Make sure we advertise registration as disabled when [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) is enabled. ([\#17661](https://github.com/element-hq/synapse/issues/17661)) +- Prevent suspended users from sending encrypted messages. ([\#18157](https://github.com/element-hq/synapse/issues/18157)) +- Cleanup deleted state group references. ([\#18165](https://github.com/element-hq/synapse/issues/18165)) +- Fix [MSC4108 QR-code login](https://github.com/matrix-org/matrix-spec-proposals/pull/4108) not working with some reverse-proxy setups. ([\#18178](https://github.com/element-hq/synapse/issues/18178)) +- Support device IDs that can't be represented in a scope when delegating auth to Matrix Authentication Service 0.15.0+. ([\#18174](https://github.com/element-hq/synapse/issues/18174)) ### Updates to the Docker image -- Add `org.opencontainers.image.version` labels to Docker containers [published by Matrix.org](https://hub.docker.com/r/matrixdotorg/synapse). Contributed by Mo Balaa. ([\#15972](https://github.com/matrix-org/synapse/issues/15972), [\#16009](https://github.com/matrix-org/synapse/issues/16009)) +- Speed up the building of the Docker image. ([\#18038](https://github.com/element-hq/synapse/issues/18038)) ### Improved Documentation -- Add a internal documentation page describing the ["streams" used within Synapse](https://matrix-org.github.io/synapse/v1.90/development/synapse_architecture/streams.html). ([\#16015](https://github.com/matrix-org/synapse/issues/16015)) -- Clarify comment on the keys/upload over replication enpoint. ([\#16016](https://github.com/matrix-org/synapse/issues/16016)) -- Do not expose Admin API in caddy reverse proxy example. Contributed by @NilsIrl. ([\#16027](https://github.com/matrix-org/synapse/issues/16027)) +- Move incorrectly placed version indicator in User Event Redaction Admin API docs. ([\#18152](https://github.com/element-hq/synapse/issues/18152)) +- Document suspension Admin API. ([\#18162](https://github.com/element-hq/synapse/issues/18162)) ### Deprecations and Removals -- Remove support for legacy application service paths. ([\#15964](https://github.com/matrix-org/synapse/issues/15964)) -- Move support for application service query parameter authorization behind a configuration option. ([\#16017](https://github.com/matrix-org/synapse/issues/16017)) - -### Internal Changes - -- Update SQL queries to inline boolean parameters as supported in SQLite 3.27. ([\#15525](https://github.com/matrix-org/synapse/issues/15525)) -- Allow for the configuration of the backoff algorithm for federation destinations. ([\#15754](https://github.com/matrix-org/synapse/issues/15754)) -- Allow modules to check whether the current worker is configured to run background tasks. ([\#15991](https://github.com/matrix-org/synapse/issues/15991)) -- Update support for [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958) to match the latest revision of the MSC. ([\#15992](https://github.com/matrix-org/synapse/issues/15992)) -- Allow modules to schedule delayed background calls. ([\#15993](https://github.com/matrix-org/synapse/issues/15993)) -- Properly overwrite the `redacts` content-property for forwards-compatibility with room versions 1 through 10. ([\#16013](https://github.com/matrix-org/synapse/issues/16013)) -- Fix building the nix development environment on MacOS systems. ([\#16019](https://github.com/matrix-org/synapse/issues/16019)) -- Remove leading and trailing spaces when setting a display name. ([\#16031](https://github.com/matrix-org/synapse/issues/16031)) -- Combine duplicated code. ([\#16023](https://github.com/matrix-org/synapse/issues/16023)) -- Collect additional metrics from `ResponseCache` for eviction. ([\#16028](https://github.com/matrix-org/synapse/issues/16028)) -- Fix endpoint improperly declaring support for MSC3814. ([\#16068](https://github.com/matrix-org/synapse/issues/16068)) -- Drop backwards compat hack for event serialization. ([\#16069](https://github.com/matrix-org/synapse/issues/16069)) +- Disable room list publication by default. ([\#18175](https://github.com/element-hq/synapse/issues/18175)) ### Updates to locked dependencies -* Update PyYAML to 6.0.1. ([\#16011](https://github.com/matrix-org/synapse/issues/16011)) -* Bump cryptography from 41.0.2 to 41.0.3. ([\#16048](https://github.com/matrix-org/synapse/issues/16048)) -* Bump furo from 2023.5.20 to 2023.7.26. ([\#16077](https://github.com/matrix-org/synapse/issues/16077)) -* Bump immutabledict from 2.2.4 to 3.0.0. ([\#16034](https://github.com/matrix-org/synapse/issues/16034)) -* Update certifi to 2023.7.22 and pygments to 2.15.1. ([\#16044](https://github.com/matrix-org/synapse/issues/16044)) -* Bump jsonschema from 4.18.3 to 4.19.0. ([\#16081](https://github.com/matrix-org/synapse/issues/16081)) -* Bump phonenumbers from 8.13.14 to 8.13.18. ([\#16076](https://github.com/matrix-org/synapse/issues/16076)) -* Bump regex from 1.9.1 to 1.9.3. ([\#16073](https://github.com/matrix-org/synapse/issues/16073)) -* Bump serde from 1.0.171 to 1.0.175. ([\#15982](https://github.com/matrix-org/synapse/issues/15982)) -* Bump serde from 1.0.175 to 1.0.179. ([\#16033](https://github.com/matrix-org/synapse/issues/16033)) -* Bump serde from 1.0.179 to 1.0.183. ([\#16074](https://github.com/matrix-org/synapse/issues/16074)) -* Bump serde_json from 1.0.103 to 1.0.104. ([\#16032](https://github.com/matrix-org/synapse/issues/16032)) -* Bump service-identity from 21.1.0 to 23.1.0. ([\#16038](https://github.com/matrix-org/synapse/issues/16038)) -* Bump types-commonmark from 0.9.2.3 to 0.9.2.4. ([\#16037](https://github.com/matrix-org/synapse/issues/16037)) -* Bump types-jsonschema from 4.17.0.8 to 4.17.0.10. ([\#16036](https://github.com/matrix-org/synapse/issues/16036)) -* Bump types-netaddr from 0.8.0.8 to 0.8.0.9. ([\#16035](https://github.com/matrix-org/synapse/issues/16035)) -* Bump types-opentracing from 2.4.10.5 to 2.4.10.6. ([\#16078](https://github.com/matrix-org/synapse/issues/16078)) -* Bump types-setuptools from 68.0.0.0 to 68.0.0.3. ([\#16079](https://github.com/matrix-org/synapse/issues/16079)) - -# Synapse 1.89.0 (2023-08-01) - -No significant changes since 1.89.0rc1. +* Bump anyhow from 1.0.95 to 1.0.96. ([\#18187](https://github.com/element-hq/synapse/issues/18187)) +* Bump authlib from 1.4.0 to 1.4.1. ([\#18190](https://github.com/element-hq/synapse/issues/18190)) +* Bump click from 8.1.7 to 8.1.8. ([\#18189](https://github.com/element-hq/synapse/issues/18189)) +* Bump log from 0.4.25 to 0.4.26. ([\#18184](https://github.com/element-hq/synapse/issues/18184)) +* Bump pyo3-log from 0.12.0 to 0.12.1. ([\#18046](https://github.com/element-hq/synapse/issues/18046)) +* Bump serde from 1.0.217 to 1.0.218. ([\#18183](https://github.com/element-hq/synapse/issues/18183)) +* Bump serde_json from 1.0.138 to 1.0.139. ([\#18186](https://github.com/element-hq/synapse/issues/18186)) +* Bump sigstore/cosign-installer from 3.8.0 to 3.8.1. ([\#18185](https://github.com/element-hq/synapse/issues/18185)) +* Bump types-psycopg2 from 2.9.21.20241019 to 2.9.21.20250121. ([\#18188](https://github.com/element-hq/synapse/issues/18188)) -# Synapse 1.89.0rc1 (2023-07-25) +# Synapse 1.125.0 (2025-02-25) + +No significant changes since 1.125.0rc1. + + +# Synapse 1.125.0rc1 (2025-02-18) ### Features -- Add Unix Socket support for HTTP Replication Listeners. [Document and provide usage instructions](https://matrix-org.github.io/synapse/v1.89/usage/configuration/config_documentation.html#listeners) for utilizing Unix sockets in Synapse. Contributed by Jason Little. ([\#15708](https://github.com/matrix-org/synapse/issues/15708), [\#15924](https://github.com/matrix-org/synapse/issues/15924)) -- Allow `+` in Matrix IDs, per [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009). ([\#15911](https://github.com/matrix-org/synapse/issues/15911)) -- Support room version 11 from [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820). ([\#15912](https://github.com/matrix-org/synapse/issues/15912)) -- Allow configuring the set of workers to proxy outbound federation traffic through via `outbound_federation_restricted_to`. ([\#15913](https://github.com/matrix-org/synapse/issues/15913), [\#15969](https://github.com/matrix-org/synapse/issues/15969)) -- Implement [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814), dehydrated devices v2/shrivelled sessions and move [MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697) behind a config flag. Contributed by Nico from Famedly, H-Shay and poljar. ([\#15929](https://github.com/matrix-org/synapse/issues/15929)) +- Add functionality to be able to use multiple values in SSO feature `attribute_requirements`. ([\#17949](https://github.com/element-hq/synapse/issues/17949)) +- Add experimental config options `admin_token_path` and `client_secret_path` for [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861). ([\#18004](https://github.com/element-hq/synapse/issues/18004)) +- Add `get_current_time_msec()` method to the [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) for sound time comparisons with Synapse. ([\#18144](https://github.com/element-hq/synapse/issues/18144)) ### Bugfixes -- Fix a long-standing bug where remote invites weren't correctly pushed. ([\#15820](https://github.com/matrix-org/synapse/issues/15820)) -- Fix background schema updates failing over a large upgrade gap. ([\#15887](https://github.com/matrix-org/synapse/issues/15887)) -- Fix a bug introduced in 1.86.0 where Synapse starting with an empty `experimental_features` configuration setting. ([\#15925](https://github.com/matrix-org/synapse/issues/15925)) -- Fixed deploy annotations in the provided Grafana dashboard config, so that it shows for any homeserver and not just matrix.org. Contributed by @wrjlewis. ([\#15957](https://github.com/matrix-org/synapse/issues/15957)) -- Ensure a long state res does not starve CPU by occasionally yielding to the reactor. ([\#15960](https://github.com/matrix-org/synapse/issues/15960)) -- Properly handle redactions of creation events. ([\#15973](https://github.com/matrix-org/synapse/issues/15973)) -- Fix a bug where resyncing stale device lists could block responding to federation transactions, and thus delay receiving new data from the remote server. ([\#15975](https://github.com/matrix-org/synapse/issues/15975)) +- Update the response when a client attempts to add an invalid email address to the user's account from a 500, to a 400 with error text. ([\#18125](https://github.com/element-hq/synapse/issues/18125)) +- Fix user directory search when using a legacy module with a `check_username_for_spam` callback. Broke in v1.122.0. ([\#18135](https://github.com/element-hq/synapse/issues/18135)) + +### Updates to the Docker image + +- Add `SYNAPSE_HTTP_PROXY`/`SYNAPSE_HTTPS_PROXY`/`SYNAPSE_NO_PROXY` environment variables to pass through specifically to the Synapse process (instead of needing to apply [`http_proxy`/`https_proxy`/`no_proxy`](https://element-hq.github.io/synapse/latest/setup/forward_proxy.html) globally). ([\#18158](https://github.com/element-hq/synapse/issues/18158)) ### Improved Documentation -- Better clarify how to run a worker instance (pass both configs). ([\#15921](https://github.com/matrix-org/synapse/issues/15921)) -- Improve [the documentation](https://matrix-org.github.io/synapse/v1.89/admin_api/user_admin_api.html#login-as-a-user) for the login as a user admin API. ([\#15938](https://github.com/matrix-org/synapse/issues/15938)) -- Fix broken Arch Linux package link. Contributed by @SnipeXandrej. ([\#15981](https://github.com/matrix-org/synapse/issues/15981)) - -### Deprecations and Removals - -- Remove support for calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15928](https://github.com/matrix-org/synapse/issues/15928)) +- Add Oracle Linux 8 and 9 installation instructions. ([\#17436](https://github.com/element-hq/synapse/issues/17436)) +- Document missing server config options (`daemonize`, `print_pidfile`, `user_agent_suffix`, `use_frozen_dicts`, `manhole`). ([\#18122](https://github.com/element-hq/synapse/issues/18122)) +- Document consequences of replacing secrets. ([\#18138](https://github.com/element-hq/synapse/issues/18138)) +- Make `burst_count` field an integer in `rc_presence` config documentation example. ([\#18159](https://github.com/element-hq/synapse/issues/18159)) ### Internal Changes -- Mark `get_user_in_directory` private since it is only used in tests. Also remove the cache from it. ([\#15884](https://github.com/matrix-org/synapse/issues/15884)) -- Document which Python version runs on a given Linux distribution so we can more easily clean up later. ([\#15909](https://github.com/matrix-org/synapse/issues/15909)) -- Add details to warning in log when we fail to fetch an alias. ([\#15922](https://github.com/matrix-org/synapse/issues/15922)) -- Remove unneeded `__init__`. ([\#15926](https://github.com/matrix-org/synapse/issues/15926)) -- Fix bug with read/write lock implementation. This is currently unused so has no observable effects. ([\#15933](https://github.com/matrix-org/synapse/issues/15933), [\#15958](https://github.com/matrix-org/synapse/issues/15958)) -- Unbreak the nix development environment by pinning the Rust version to 1.70.0. ([\#15940](https://github.com/matrix-org/synapse/issues/15940)) -- Update presence metrics to differentiate remote vs local users. ([\#15952](https://github.com/matrix-org/synapse/issues/15952)) -- Stop reading from column `user_id` of table `profiles`. ([\#15955](https://github.com/matrix-org/synapse/issues/15955)) -- Build packages for Debian Trixie. ([\#15961](https://github.com/matrix-org/synapse/issues/15961)) -- Reduce the amount of state we pull out. ([\#15968](https://github.com/matrix-org/synapse/issues/15968)) -- Speed up updating state in large rooms. ([\#15971](https://github.com/matrix-org/synapse/issues/15971)) +- Overload `DatabasePool.simple_select_one_txn` to return non-`None` when the `allow_none` parameter is `False`. ([\#17616](https://github.com/element-hq/synapse/issues/17616)) +- Python 3.8 EOL: compile native extensions with the 3.9 ABI and use typing hints from the standard library. ([\#17967](https://github.com/element-hq/synapse/issues/17967)) +- Add log message when worker lock timeouts get large. ([\#18124](https://github.com/element-hq/synapse/issues/18124)) +- Make it explicit that you can buy an AGPL-alternative commercial license from Element. ([\#18134](https://github.com/element-hq/synapse/issues/18134)) +- Fix the 'Fix linting' GitHub Actions workflow. ([\#18136](https://github.com/element-hq/synapse/issues/18136)) +- Do not log at the exception-level when clients provide empty `since` token to `/sync` API. ([\#18139](https://github.com/element-hq/synapse/issues/18139)) +- Reduce database load of user search when using large search terms. ([\#18172](https://github.com/element-hq/synapse/issues/18172)) + + ### Updates to locked dependencies -* Bump anyhow from 1.0.71 to 1.0.72. ([\#15949](https://github.com/matrix-org/synapse/issues/15949)) -* Bump click from 8.1.3 to 8.1.6. ([\#15984](https://github.com/matrix-org/synapse/issues/15984)) -* Bump cryptography from 41.0.1 to 41.0.2. ([\#15943](https://github.com/matrix-org/synapse/issues/15943)) -* Bump jsonschema from 4.17.3 to 4.18.3. ([\#15948](https://github.com/matrix-org/synapse/issues/15948)) -* Bump pillow from 9.4.0 to 10.0.0. ([\#15986](https://github.com/matrix-org/synapse/issues/15986)) -* Bump prometheus-client from 0.17.0 to 0.17.1. ([\#15945](https://github.com/matrix-org/synapse/issues/15945)) -* Bump pydantic from 1.10.10 to 1.10.11. ([\#15946](https://github.com/matrix-org/synapse/issues/15946)) -* Bump pygithub from 1.58.2 to 1.59.0. ([\#15834](https://github.com/matrix-org/synapse/issues/15834)) -* Bump pyo3-log from 0.8.2 to 0.8.3. ([\#15951](https://github.com/matrix-org/synapse/issues/15951)) -* Bump sentry-sdk from 1.26.0 to 1.28.1. ([\#15985](https://github.com/matrix-org/synapse/issues/15985)) -* Bump serde_json from 1.0.100 to 1.0.103. ([\#15950](https://github.com/matrix-org/synapse/issues/15950)) -* Bump types-pillow from 9.5.0.4 to 10.0.0.1. ([\#15932](https://github.com/matrix-org/synapse/issues/15932)) -* Bump types-requests from 2.31.0.1 to 2.31.0.2. ([\#15983](https://github.com/matrix-org/synapse/issues/15983)) -* Bump typing-extensions from 4.5.0 to 4.7.1. ([\#15947](https://github.com/matrix-org/synapse/issues/15947)) +* Bump bcrypt from 4.2.0 to 4.2.1. ([\#18127](https://github.com/element-hq/synapse/issues/18127)) +* Bump bytes from 1.9.0 to 1.10.0. ([\#18149](https://github.com/element-hq/synapse/issues/18149)) +* Bump gitpython from 3.1.43 to 3.1.44. ([\#18128](https://github.com/element-hq/synapse/issues/18128)) +* Bump hiredis from 3.0.0 to 3.1.0. ([\#18169](https://github.com/element-hq/synapse/issues/18169)) +* Bump serde_json from 1.0.137 to 1.0.138. ([\#18129](https://github.com/element-hq/synapse/issues/18129)) +* Bump service-identity from 24.1.0 to 24.2.0. ([\#18171](https://github.com/element-hq/synapse/issues/18171)) +* Bump sigstore/cosign-installer from 3.7.0 to 3.8.0. ([\#18147](https://github.com/element-hq/synapse/issues/18147)) +* Bump twine from 6.0.1 to 6.1.0. ([\#18170](https://github.com/element-hq/synapse/issues/18170)) +* Bump types-pyyaml from 6.0.12.20240917 to 6.0.12.20241230. ([\#18097](https://github.com/element-hq/synapse/issues/18097)) +* Bump ulid from 1.1.4 to 1.2.0. ([\#18148](https://github.com/element-hq/synapse/issues/18148)) -# Synapse 1.88.0 (2023-07-18) +# Synapse 1.124.0 (2025-02-11) -This release - - raises the minimum supported version of Python to 3.8, as Python 3.7 is now [end-of-life](https://devguide.python.org/versions/), and - - removes deprecated config options related to worker deployment. +No significant changes since 1.124.0rc3. -See [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.88/docs/upgrade.md#upgrading-to-v1880) for more information. + +# Synapse 1.124.0rc3 (2025-02-07) + ### Bugfixes -- Revert "Stop writing to column `user_id` of tables `profiles` and `user_filters`", which was introduced in Synapse 1.88.0rc1. ([\#15953](https://github.com/matrix-org/synapse/issues/15953)) +- Fix regression in performance of sending events due to superfluous reads and locks. Introduced in v1.124.0rc1. ([\#18141](https://github.com/element-hq/synapse/issues/18141)) -# Synapse 1.88.0rc1 (2023-07-11) + + +# Synapse 1.124.0rc2 (2025-02-05) + +### Bugfixes + +- Fix regression where persisting events in some rooms could fail after a previous unclean shutdown. Introduced in v1.124.0rc1. ([\#18137](https://github.com/element-hq/synapse/issues/18137)) + + + + +# Synapse 1.124.0rc1 (2025-02-04) + +### Bugfixes + +- Add rate limit `rc_presence.per_user`. This prevents load from excessive presence updates sent by clients via sync api. Also rate limit `/_matrix/client/v3/presence` as per the spec. Contributed by @rda0. ([\#18000](https://github.com/element-hq/synapse/issues/18000)) +- Deactivated users will no longer automatically accept an invite when `auto_accept_invites` is enabled. ([\#18073](https://github.com/element-hq/synapse/issues/18073)) +- Fix join being denied after being invited over federation. Also fixes other out-of-band membership transitions. ([\#18075](https://github.com/element-hq/synapse/issues/18075)) +- Updates contributed `docker-compose.yml` file to PostgreSQL v15, as v12 is no longer supported by Synapse. + Contributed by @maxkratz. ([\#18089](https://github.com/element-hq/synapse/issues/18089)) +- Fix rare edge case where state groups could be deleted while we are persisting new events that reference them. ([\#18107](https://github.com/element-hq/synapse/issues/18107), [\#18130](https://github.com/element-hq/synapse/issues/18130), [\#18131](https://github.com/element-hq/synapse/issues/18131)) +- Raise an error if someone is using an incorrect suffix in a config duration string. ([\#18112](https://github.com/element-hq/synapse/issues/18112)) +- Fix a bug where the [Delete Room Admin API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#version-2-new-version) would fail if the `block` parameter was set to `true` and a worker other than the main process was configured to handle background tasks. ([\#18119](https://github.com/element-hq/synapse/issues/18119)) + +### Internal Changes + +- Increase the length of the generated `nonce` parameter when perfoming OIDC logins to comply with the TI-Messenger spec. ([\#18109](https://github.com/element-hq/synapse/issues/18109)) + + + +### Updates to locked dependencies + +* Bump dawidd6/action-download-artifact from 7 to 8. ([\#18108](https://github.com/element-hq/synapse/issues/18108)) +* Bump log from 0.4.22 to 0.4.25. ([\#18098](https://github.com/element-hq/synapse/issues/18098)) +* Bump python-multipart from 0.0.18 to 0.0.20. ([\#18096](https://github.com/element-hq/synapse/issues/18096)) +* Bump serde_json from 1.0.135 to 1.0.137. ([\#18099](https://github.com/element-hq/synapse/issues/18099)) +* Bump types-bleach from 6.1.0.20240331 to 6.2.0.20241123. ([\#18082](https://github.com/element-hq/synapse/issues/18082)) + +# Synapse 1.123.0 (2025-01-28) + +No significant changes since 1.123.0rc1. + + + + +# Synapse 1.123.0rc1 (2025-01-21) ### Features -- Add `not_user_type` param to the [list accounts admin API](https://matrix-org.github.io/synapse/v1.88/admin_api/user_admin_api.html#list-accounts). ([\#15844](https://github.com/matrix-org/synapse/issues/15844)) +- Implement [MSC4133](https://github.com/matrix-org/matrix-spec-proposals/pull/4133) for custom profile fields. Contributed by @clokep. ([\#17488](https://github.com/element-hq/synapse/issues/17488)) +- Add a query parameter `type` to the [Room State Admin API](https://element-hq.github.io/synapse/develop/admin_api/rooms.html#room-state-api) that filters the state event. ([\#18035](https://github.com/element-hq/synapse/issues/18035)) +- Support the new `/auth_metadata` endpoint defined in [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965). ([\#18093](https://github.com/element-hq/synapse/issues/18093)) ### Bugfixes -- Pin `pydantic` to `^=1.7.4` to avoid backwards-incompatible API changes from the 2.0.0 release. - Contributed by @PaarthShah. ([\#15862](https://github.com/matrix-org/synapse/issues/15862)) -- Correctly resize thumbnails with pillow version >=10. ([\#15876](https://github.com/matrix-org/synapse/issues/15876)) +- Fix membership caches not updating in state reset scenarios. ([\#17732](https://github.com/element-hq/synapse/issues/17732)) +- Fix rare race where on upgrade to v1.122.0 a long running database upgrade could lock out new events from being received or sent. ([\#18091](https://github.com/element-hq/synapse/issues/18091)) ### Improved Documentation -- Fixed header levels on the [Admin API "Users"](https://matrix-org.github.io/synapse/v1.87/admin_api/user_admin_api.html) documentation page. Contributed by @sumnerevans at @beeper. ([\#15852](https://github.com/matrix-org/synapse/issues/15852)) -- Remove deprecated `worker_replication_host`, `worker_replication_http_port` and `worker_replication_http_tls` configuration options. ([\#15872](https://github.com/matrix-org/synapse/issues/15872)) +- Document `tls` option for a worker instance in `instance_map`. ([\#18064](https://github.com/element-hq/synapse/issues/18064)) ### Deprecations and Removals -- **Remove deprecated `worker_replication_host`, `worker_replication_http_port` and `worker_replication_http_tls` configuration options.** See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.88/docs/upgrade.md#removal-of-worker_replication_-settings) for more details. ([\#15860](https://github.com/matrix-org/synapse/issues/15860)) -- Remove support for Python 3.7 and hence for Debian Buster. ([\#15851](https://github.com/matrix-org/synapse/issues/15851), [\#15892](https://github.com/matrix-org/synapse/issues/15892), [\#15893](https://github.com/matrix-org/synapse/issues/15893), [\#15917](https://github.com/matrix-org/synapse/pull/15917)) +- Remove the unstable [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151) implementation. The stable support remains, per [Matrix 1.13](https://spec.matrix.org/v1.13/client-server-api/#post_matrixclientv3roomsroomidreport). ([\#18052](https://github.com/element-hq/synapse/issues/18052)) ### Internal Changes -- Add foreign key constraint to `event_forward_extremities`. ([\#15751](https://github.com/matrix-org/synapse/issues/15751), [\#15907](https://github.com/matrix-org/synapse/issues/15907)) -- Add read/write style cross-worker locks. ([\#15782](https://github.com/matrix-org/synapse/issues/15782)) -- Stop writing to column `user_id` of tables `profiles` and `user_filters`. ([\#15787](https://github.com/matrix-org/synapse/issues/15787)) -- Use lower isolation level when cleaning old presence stream data to avoid serialization errors. ([\#15826](https://github.com/matrix-org/synapse/issues/15826)) -- Add tracing to media `/upload` code paths. ([\#15850](https://github.com/matrix-org/synapse/issues/15850), [\#15888](https://github.com/matrix-org/synapse/issues/15888)) -- Add a timeout that aborts any Postgres statement taking more than 1 hour. ([\#15853](https://github.com/matrix-org/synapse/issues/15853)) -- Fix the `devenv up` configuration which was ignoring the config overrides. ([\#15854](https://github.com/matrix-org/synapse/issues/15854)) -- Optimised cleanup of old entries in `device_lists_stream`. ([\#15861](https://github.com/matrix-org/synapse/issues/15861)) -- Update the Matrix clients link in the _It works! Synapse is running_ landing page. ([\#15874](https://github.com/matrix-org/synapse/issues/15874)) -- Fix building Synapse with the nightly Rust compiler. ([\#15906](https://github.com/matrix-org/synapse/issues/15906)) -- Add `Server` to Access-Control-Expose-Headers header. ([\#15908](https://github.com/matrix-org/synapse/issues/15908)) +- Increase invite rate limits (`rc_invites.per_issuer`) for Complement. ([\#18072](https://github.com/element-hq/synapse/issues/18072)) + + ### Updates to locked dependencies -* Bump authlib from 1.2.0 to 1.2.1. ([\#15864](https://github.com/matrix-org/synapse/issues/15864)) -* Bump importlib-metadata from 6.6.0 to 6.7.0. ([\#15865](https://github.com/matrix-org/synapse/issues/15865)) -* Bump lxml from 4.9.2 to 4.9.3. ([\#15897](https://github.com/matrix-org/synapse/issues/15897)) -* Bump regex from 1.8.4 to 1.9.1. ([\#15902](https://github.com/matrix-org/synapse/issues/15902)) -* Bump ruff from 0.0.275 to 0.0.277. ([\#15900](https://github.com/matrix-org/synapse/issues/15900)) -* Bump sentry-sdk from 1.25.1 to 1.26.0. ([\#15867](https://github.com/matrix-org/synapse/issues/15867)) -* Bump serde_json from 1.0.99 to 1.0.100. ([\#15901](https://github.com/matrix-org/synapse/issues/15901)) -* Bump types-pyopenssl from 23.2.0.0 to 23.2.0.1. ([\#15866](https://github.com/matrix-org/synapse/issues/15866)) +* Bump jinja2 from 3.1.4 to 3.1.5. ([\#18067](https://github.com/element-hq/synapse/issues/18067)) +* Bump mypy from 1.12.1 to 1.13.0. ([\#18083](https://github.com/element-hq/synapse/issues/18083)) +* Bump pillow from 11.0.0 to 11.1.0. ([\#18084](https://github.com/element-hq/synapse/issues/18084)) +* Bump pyo3 from 0.23.3 to 0.23.4. ([\#18079](https://github.com/element-hq/synapse/issues/18079)) +* Bump pyopenssl from 24.2.1 to 24.3.0. ([\#18062](https://github.com/element-hq/synapse/issues/18062)) +* Bump serde_json from 1.0.134 to 1.0.135. ([\#18081](https://github.com/element-hq/synapse/issues/18081)) +* Bump ulid from 1.1.3 to 1.1.4. ([\#18080](https://github.com/element-hq/synapse/issues/18080)) -# Synapse 1.87.0 (2023-07-04) +# Synapse 1.122.0 (2025-01-14) -Please note that this will be the last release of Synapse that is compatible with -Python 3.7 and earlier. -This is due to Python 3.7 now having reached End of Life; see our [deprecation policy](https://matrix-org.github.io/synapse/v1.87/deprecation_policy.html) -for more details. +Please note that this version of Synapse drops support for PostgreSQL 11 and 12. The minimum version of PostgreSQL supported is now version 13. -### Bugfixes - -- Pin `pydantic` to `^1.7.4` to avoid backwards-incompatible API changes from the 2.0.0 release. - Resolves https://github.com/matrix-org/synapse/issues/15858. - Contributed by @PaarthShah. ([\#15862](https://github.com/matrix-org/synapse/issues/15862)) - -### Internal Changes - -- Split out 2022 changes from the changelog so the rendered version in GitHub doesn't timeout as much. ([\#15846](https://github.com/matrix-org/synapse/issues/15846)) +No significant changes since 1.122.0rc1. -# Synapse 1.87.0rc1 (2023-06-27) +# Synapse 1.122.0rc1 (2025-01-07) + +### Deprecations and Removals + +- Remove support for PostgreSQL 11 and 12. Contributed by @clokep. ([\#18034](https://github.com/element-hq/synapse/issues/18034)) ### Features -- Improve `/messages` response time by avoiding backfill when we already have messages to return. ([\#15737](https://github.com/matrix-org/synapse/issues/15737)) -- Add spam checker module API for logins. ([\#15838](https://github.com/matrix-org/synapse/issues/15838)) +- Added the `email.tlsname` config option. This allows specifying the domain name used to validate the SMTP server's TLS certificate separately from the `email.smtp_host` to connect to. ([\#17849](https://github.com/element-hq/synapse/issues/17849)) +- Module developers will have access to the user ID of the requester when adding `check_username_for_spam` callbacks to `spam_checker_module_callbacks`. Contributed by Wilson@Pangea.chat. ([\#17916](https://github.com/element-hq/synapse/issues/17916)) +- Add endpoints to the Admin API to fetch the number of invites the provided user has sent after a given timestamp, + fetch the number of rooms the provided user has joined after a given timestamp, and get report IDs of event + reports against a provided user (i.e. where the user was the sender of the reported event). ([\#17948](https://github.com/element-hq/synapse/issues/17948)) +- Support stable account suspension from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823). ([\#17964](https://github.com/element-hq/synapse/issues/17964)) +- Add `macaroon_secret_key_path` config option. ([\#17983](https://github.com/element-hq/synapse/issues/17983)) ### Bugfixes -- Fix a long-standing bug where media files were served in an unsafe manner. Contributed by @joshqou. ([\#15680](https://github.com/matrix-org/synapse/issues/15680)) -- Avoid invalidating a cache that was just prefilled. ([\#15758](https://github.com/matrix-org/synapse/issues/15758)) -- Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983). ([\#15770](https://github.com/matrix-org/synapse/issues/15770)) -- Fix joining rooms through aliases where the alias server isn't a real homeserver. Contributed by @tulir @ Beeper. ([\#15776](https://github.com/matrix-org/synapse/issues/15776)) -- Fix a bug in push rules handling leading to an invalid (per spec) `is_user_mention` rule sent to clients. Also fix wrong rule names for `is_user_mention` and `is_room_mention`. ([\#15781](https://github.com/matrix-org/synapse/issues/15781)) -- Fix a bug introduced in 1.57.0 where the wrong table would be locked on updating database rows when using SQLite as the database backend. ([\#15788](https://github.com/matrix-org/synapse/issues/15788)) -- Fix Sytest environmental variable evaluation in CI. ([\#15804](https://github.com/matrix-org/synapse/issues/15804)) -- Fix forgotten rooms missing from initial sync after rejoining them. Contributed by Nico from Famedly. ([\#15815](https://github.com/matrix-org/synapse/issues/15815)) -- Fix sqlite `user_filters` upgrade introduced in v1.86.0. ([\#15817](https://github.com/matrix-org/synapse/issues/15817)) +- Fix bug when rejecting withdrew invite with a `third_party_rules` module, where the invite would be stuck for the client. ([\#17930](https://github.com/element-hq/synapse/issues/17930)) +- Properly purge state groups tables when purging a room with the Admin API. ([\#18024](https://github.com/element-hq/synapse/issues/18024)) +- Fix a bug preventing the admin redaction endpoint from working on messages from remote users. ([\#18029](https://github.com/element-hq/synapse/issues/18029), [\#18043](https://github.com/element-hq/synapse/issues/18043)) ### Improved Documentation -- Document `looping_call()` functionality that will wait for the given function to finish before scheduling another. ([\#15772](https://github.com/matrix-org/synapse/issues/15772)) -- Fix a typo in the [Admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html). ([\#15805](https://github.com/matrix-org/synapse/issues/15805)) -- Fix typo in MSC number in faster remote room join architecture doc. ([\#15812](https://github.com/matrix-org/synapse/issues/15812)) - -### Deprecations and Removals - -- Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms. ([\#15748](https://github.com/matrix-org/synapse/issues/15748)) +- Update `synapse.app.generic_worker` documentation to only recommend `GET` requests for stream writer routes by default, unless the worker is also configured as a stream writer. Contributed by @evoL. ([\#17954](https://github.com/element-hq/synapse/issues/17954)) +- Add documentation for the previously-undocumented `last_seen_ts` query parameter to the query user Admin API. ([\#17976](https://github.com/element-hq/synapse/issues/17976)) +- Improve documentation for the `TaskScheduler` class. ([\#17992](https://github.com/element-hq/synapse/issues/17992)) +- Fix example in reverse proxy docs to include server port. ([\#17994](https://github.com/element-hq/synapse/issues/17994)) +- Update Alpine Linux Synapse Package Maintainer within the installation instructions. ([\#17846](https://github.com/element-hq/synapse/issues/17846)) ### Internal Changes -- Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`. ([\#15233](https://github.com/matrix-org/synapse/issues/15233)) -- Regularly try to send transactions to other servers after they failed instead of waiting for a new event to be available before trying. ([\#15743](https://github.com/matrix-org/synapse/issues/15743)) -- Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983). ([\#15755](https://github.com/matrix-org/synapse/issues/15755)) -- Allow for the configuration of max request retries and min/max retry delays in the matrix federation client. ([\#15783](https://github.com/matrix-org/synapse/issues/15783)) -- Switch from `matrix://` to `matrix-federation://` scheme for internal Synapse routing of outbound federation traffic. ([\#15806](https://github.com/matrix-org/synapse/issues/15806)) -- Fix harmless exceptions being printed when running the port DB script. ([\#15814](https://github.com/matrix-org/synapse/issues/15814)) +- Add `RoomID` & `EventID` rust types. ([\#17996](https://github.com/element-hq/synapse/issues/17996)) +- Fix various type errors across the codebase. ([\#17998](https://github.com/element-hq/synapse/issues/17998)) +- Disable DB statement timeout when doing a room purge since it can be quite long. ([\#18017](https://github.com/element-hq/synapse/issues/18017)) +- Remove some remaining uses of `twisted.internet.defer.returnValue`. Contributed by Colin Watson. ([\#18020](https://github.com/element-hq/synapse/issues/18020)) +- Refactor `get_profile` to no longer include fields with a value of `None`. ([\#18063](https://github.com/element-hq/synapse/issues/18063)) ### Updates to locked dependencies -* Bump attrs from 22.2.0 to 23.1.0. ([\#15801](https://github.com/matrix-org/synapse/issues/15801)) -* Bump cryptography from 40.0.2 to 41.0.1. ([\#15800](https://github.com/matrix-org/synapse/issues/15800)) -* Bump ijson from 3.2.0.post0 to 3.2.1. ([\#15802](https://github.com/matrix-org/synapse/issues/15802)) -* Bump phonenumbers from 8.13.13 to 8.13.14. ([\#15798](https://github.com/matrix-org/synapse/issues/15798)) -* Bump ruff from 0.0.265 to 0.0.272. ([\#15799](https://github.com/matrix-org/synapse/issues/15799)) -* Bump ruff from 0.0.272 to 0.0.275. ([\#15833](https://github.com/matrix-org/synapse/issues/15833)) -* Bump serde_json from 1.0.96 to 1.0.97. ([\#15797](https://github.com/matrix-org/synapse/issues/15797)) -* Bump serde_json from 1.0.97 to 1.0.99. ([\#15832](https://github.com/matrix-org/synapse/issues/15832)) -* Bump towncrier from 22.12.0 to 23.6.0. ([\#15831](https://github.com/matrix-org/synapse/issues/15831)) -* Bump types-opentracing from 2.4.10.4 to 2.4.10.5. ([\#15830](https://github.com/matrix-org/synapse/issues/15830)) -* Bump types-setuptools from 67.8.0.0 to 68.0.0.0. ([\#15835](https://github.com/matrix-org/synapse/issues/15835)) - -Synapse 1.86.0 (2023-06-20) -=========================== - -No significant changes since 1.86.0rc2. - - -Synapse 1.86.0rc2 (2023-06-14) -============================== - -Bugfixes --------- - -- Fix an error when having workers of different versions running. ([\#15774](https://github.com/matrix-org/synapse/issues/15774)) - - -Synapse 1.86.0rc1 (2023-06-13) -============================== - -This version was tagged but never released. - -Features --------- - -- Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session. ([\#15388](https://github.com/matrix-org/synapse/issues/15388)) -- Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API. ([\#15450](https://github.com/matrix-org/synapse/issues/15450)) -- Enable support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#15520](https://github.com/matrix-org/synapse/issues/15520)) -- Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider. ([\#15582](https://github.com/matrix-org/synapse/issues/15582)) -- Add Synapse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions. ([\#15674](https://github.com/matrix-org/synapse/issues/15674)) -- Add a catch-all * to the supported relation types when redacting an event and its related events. This is an update to [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation. ([\#15705](https://github.com/matrix-org/synapse/issues/15705)) -- Speed up `/messages` by backfilling in the background when there are no backward extremities where we are directly paginating. ([\#15710](https://github.com/matrix-org/synapse/issues/15710)) -- Expose a metric reporting the database background update status. ([\#15740](https://github.com/matrix-org/synapse/issues/15740)) - - -Bugfixes --------- - -- Correctly clear caches when we delete a room. ([\#15609](https://github.com/matrix-org/synapse/issues/15609)) -- Check permissions for enabling encryption earlier during room creation to avoid creating broken rooms. ([\#15695](https://github.com/matrix-org/synapse/issues/15695)) - - -Improved Documentation ----------------------- - -- Simplify query to find participating servers in a room. ([\#15732](https://github.com/matrix-org/synapse/issues/15732)) - - -Internal Changes ----------------- - -- Log when events are (maybe unexpectedly) filtered out of responses in tests. ([\#14213](https://github.com/matrix-org/synapse/issues/14213)) -- Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`. ([\#15649](https://github.com/matrix-org/synapse/issues/15649)) -- Add support for tracing functions which return `Awaitable`s. ([\#15650](https://github.com/matrix-org/synapse/issues/15650)) -- Cache requests for user's devices over federation. ([\#15675](https://github.com/matrix-org/synapse/issues/15675)) -- Add fully qualified docker image names to Dockerfiles. ([\#15689](https://github.com/matrix-org/synapse/issues/15689)) -- Remove some unused code. ([\#15690](https://github.com/matrix-org/synapse/issues/15690)) -- Improve type hints. ([\#15694](https://github.com/matrix-org/synapse/issues/15694), [\#15697](https://github.com/matrix-org/synapse/issues/15697)) -- Update docstring and traces on `maybe_backfill()` functions. ([\#15709](https://github.com/matrix-org/synapse/issues/15709)) -- Add context for when/why to use the `long_retries` option when sending Federation requests. ([\#15721](https://github.com/matrix-org/synapse/issues/15721)) -- Removed some unused fields. ([\#15723](https://github.com/matrix-org/synapse/issues/15723)) -- Update federation error to more plainly explain we can only authorize our own membership events. ([\#15725](https://github.com/matrix-org/synapse/issues/15725)) -- Prevent the `latest_deps` and `twisted_trunk` daily GitHub Actions workflows from running on forks of the codebase. ([\#15726](https://github.com/matrix-org/synapse/issues/15726)) -- Improve performance of user directory search. ([\#15729](https://github.com/matrix-org/synapse/issues/15729)) -- Remove redundant table join with `room_memberships` when doing a `is_host_joined()`/`is_host_invited()` call (`membership` is already part of the `current_state_events`). ([\#15731](https://github.com/matrix-org/synapse/issues/15731)) -- Remove superfluous `room_memberships` join from background update. ([\#15733](https://github.com/matrix-org/synapse/issues/15733)) -- Speed up typechecking CI. ([\#15752](https://github.com/matrix-org/synapse/issues/15752)) -- Bump minimum supported Rust version to 1.60.0. ([\#15768](https://github.com/matrix-org/synapse/issues/15768)) - -### Updates to locked dependencies - -* Bump importlib-metadata from 6.1.0 to 6.6.0. ([\#15711](https://github.com/matrix-org/synapse/issues/15711)) -* Bump library/redis from 6-bullseye to 7-bullseye in /docker. ([\#15712](https://github.com/matrix-org/synapse/issues/15712)) -* Bump log from 0.4.18 to 0.4.19. ([\#15761](https://github.com/matrix-org/synapse/issues/15761)) -* Bump phonenumbers from 8.13.11 to 8.13.13. ([\#15763](https://github.com/matrix-org/synapse/issues/15763)) -* Bump pyasn1 from 0.4.8 to 0.5.0. ([\#15713](https://github.com/matrix-org/synapse/issues/15713)) -* Bump pydantic from 1.10.8 to 1.10.9. ([\#15762](https://github.com/matrix-org/synapse/issues/15762)) -* Bump pyo3-log from 0.8.1 to 0.8.2. ([\#15759](https://github.com/matrix-org/synapse/issues/15759)) -* Bump pyopenssl from 23.1.1 to 23.2.0. ([\#15765](https://github.com/matrix-org/synapse/issues/15765)) -* Bump regex from 1.7.3 to 1.8.4. ([\#15769](https://github.com/matrix-org/synapse/issues/15769)) -* Bump sentry-sdk from 1.22.1 to 1.25.0. ([\#15714](https://github.com/matrix-org/synapse/issues/15714)) -* Bump sentry-sdk from 1.25.0 to 1.25.1. ([\#15764](https://github.com/matrix-org/synapse/issues/15764)) -* Bump serde from 1.0.163 to 1.0.164. ([\#15760](https://github.com/matrix-org/synapse/issues/15760)) -* Bump types-jsonschema from 4.17.0.7 to 4.17.0.8. ([\#15716](https://github.com/matrix-org/synapse/issues/15716)) -* Bump types-pyopenssl from 23.1.0.2 to 23.2.0.0. ([\#15766](https://github.com/matrix-org/synapse/issues/15766)) -* Bump types-requests from 2.31.0.0 to 2.31.0.1. ([\#15715](https://github.com/matrix-org/synapse/issues/15715)) - -Synapse 1.85.2 (2023-06-08) -=========================== - -Bugfixes --------- - -- Fix regression where using TLS for HTTP replication between workers did not work. Introduced in v1.85.0. ([\#15746](https://github.com/matrix-org/synapse/issues/15746)) - - -Synapse 1.85.1 (2023-06-07) -=========================== - -Note: this release only fixes a bug that stopped some deployments from upgrading to v1.85.0. There is no need to upgrade to v1.85.1 if successfully running v1.85.0. - -Bugfixes --------- - -- Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. ([\#15738](https://github.com/matrix-org/synapse/issues/15738), [\#15739](https://github.com/matrix-org/synapse/issues/15739)) - - -Synapse 1.85.0 (2023-06-06) -=========================== - -No significant changes since 1.85.0rc2. - - -## Security advisory - -The following issues are fixed in 1.85.0 (and RCs). - -- [GHSA-26c5-ppr8-f33p](https://github.com/matrix-org/synapse/security/advisories/GHSA-26c5-ppr8-f33p) / [CVE-2023-32682](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32682) — Low Severity - - It may be possible for a deactivated user to login when using uncommon configurations. - -- [GHSA-98px-6486-j7qc](https://github.com/matrix-org/synapse/security/advisories/GHSA-98px-6486-j7qc) / [CVE-2023-32683](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32683) — Low Severity - - A discovered oEmbed or image URL can bypass the `url_preview_url_blacklist` setting potentially allowing server side request forgery or bypassing network policies. Impact is limited to IP addresses allowed by the `url_preview_ip_range_blacklist` setting (by default this only allows public IPs). - -See the advisories for more details. If you have any questions, email security@matrix.org. - - -Synapse 1.85.0rc2 (2023-06-01) -============================== - -Bugfixes --------- - -- Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. ([\#15693](https://github.com/matrix-org/synapse/issues/15693)) - - -Deprecations and Removals -------------------------- - -- Deprecate calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15703](https://github.com/matrix-org/synapse/issues/15703)) - - -Internal Changes ----------------- - -- Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. ([\#15700](https://github.com/matrix-org/synapse/issues/15700)) - - -Synapse 1.85.0rc1 (2023-05-30) -============================== - -Features --------- - -- Improve performance of backfill requests by performing backfill of previously failed requests in the background. ([\#15585](https://github.com/matrix-org/synapse/issues/15585)) -- Add a new [admin API](https://matrix-org.github.io/synapse/v1.85/usage/administration/admin_api/index.html) to [create a new device for a user](https://matrix-org.github.io/synapse/v1.85/admin_api/user_admin_api.html#create-a-device). ([\#15611](https://github.com/matrix-org/synapse/issues/15611)) -- Add Unix socket support for Redis connections. Contributed by Jason Little. ([\#15644](https://github.com/matrix-org/synapse/issues/15644)) - - -Bugfixes --------- - -- Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar). ([\#15464](https://github.com/matrix-org/synapse/issues/15464)) -- Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL. ([\#15601](https://github.com/matrix-org/synapse/issues/15601)) -- Fix a long-standing bug where filters with multiple backslashes were rejected. ([\#15607](https://github.com/matrix-org/synapse/issues/15607)) -- Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted. ([\#15614](https://github.com/matrix-org/synapse/issues/15614)) -- Fix a long-standing bug where deactivated users were still able to login using the custom `org.matrix.login.jwt` login type (if enabled). ([\#15624](https://github.com/matrix-org/synapse/issues/15624)) -- Fix a long-standing bug where deactivated users were able to login in uncommon situations. ([\#15634](https://github.com/matrix-org/synapse/issues/15634)) - - -Improved Documentation ----------------------- - -- Warn users that at least 3.75GB of space is needed for the nix Synapse development environment. ([\#15613](https://github.com/matrix-org/synapse/issues/15613)) -- Remove outdated comment from the generated and sample homeserver log configs. ([\#15648](https://github.com/matrix-org/synapse/issues/15648)) -- Improve contributor docs to make it more clear that Rust is a necessary prerequisite. Contributed by @grantm. ([\#15668](https://github.com/matrix-org/synapse/issues/15668)) - - -Deprecations and Removals -------------------------- - -- Remove the old version of the R30 (30-day retained users) phone-home metric. ([\#10428](https://github.com/matrix-org/synapse/issues/10428)) - - -Internal Changes ----------------- - -- Create dependabot changelogs at release time. ([\#15481](https://github.com/matrix-org/synapse/issues/15481)) -- Add not null constraint to column `full_user_id` of tables `profiles` and `user_filters`. ([\#15537](https://github.com/matrix-org/synapse/issues/15537)) -- Allow connecting to HTTP Replication Endpoints by using `worker_name` when constructing the request. ([\#15578](https://github.com/matrix-org/synapse/issues/15578)) -- Make the `thread_id` column on `event_push_actions`, `event_push_actions_staging`, and `event_push_summary` non-null. ([\#15597](https://github.com/matrix-org/synapse/issues/15597)) -- Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible. ([\#15602](https://github.com/matrix-org/synapse/issues/15602)) -- Fix subscriptable type usage in Python <3.9. ([\#15604](https://github.com/matrix-org/synapse/issues/15604)) -- Update internal terminology. ([\#15606](https://github.com/matrix-org/synapse/issues/15606), [\#15620](https://github.com/matrix-org/synapse/issues/15620)) -- Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing. ([\#15610](https://github.com/matrix-org/synapse/issues/15610), [\#15647](https://github.com/matrix-org/synapse/issues/15647)) -- Trace how many new events from the backfill response we need to process. ([\#15633](https://github.com/matrix-org/synapse/issues/15633)) -- Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s. ([\#15615](https://github.com/matrix-org/synapse/issues/15615)) -- Update Mutual Rooms ([MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666)) implementation to match new proposal text. ([\#15621](https://github.com/matrix-org/synapse/issues/15621)) -- Remove the unstable identifiers from faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706)). ([\#15625](https://github.com/matrix-org/synapse/issues/15625)) -- Fix the olddeps CI. ([\#15626](https://github.com/matrix-org/synapse/issues/15626)) -- Remove duplicate timestamp from test logs (`_trial_temp/test.log`). ([\#15636](https://github.com/matrix-org/synapse/issues/15636)) -- Fix two memory leaks in `trial` test runs. ([\#15630](https://github.com/matrix-org/synapse/issues/15630)) -- Limit the size of the `HomeServerConfig` cache in trial test runs. ([\#15646](https://github.com/matrix-org/synapse/issues/15646)) -- Improve type hints. ([\#15658](https://github.com/matrix-org/synapse/issues/15658), [\#15659](https://github.com/matrix-org/synapse/issues/15659)) -- Add requesting user id parameter to key claim methods in `TransportLayerClient`. ([\#15663](https://github.com/matrix-org/synapse/issues/15663)) -- Speed up rebuilding of the user directory for local users. ([\#15665](https://github.com/matrix-org/synapse/issues/15665)) -- Implement "option 2" for [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820): Room version 11. ([\#15666](https://github.com/matrix-org/synapse/issues/15666), [\#15678](https://github.com/matrix-org/synapse/issues/15678)) - -### Updates to locked dependencies - -* Bump furo from 2023.3.27 to 2023.5.20. ([\#15642](https://github.com/matrix-org/synapse/issues/15642)) -* Bump log from 0.4.17 to 0.4.18. ([\#15681](https://github.com/matrix-org/synapse/issues/15681)) -* Bump prometheus-client from 0.16.0 to 0.17.0. ([\#15682](https://github.com/matrix-org/synapse/issues/15682)) -* Bump pydantic from 1.10.7 to 1.10.8. ([\#15685](https://github.com/matrix-org/synapse/issues/15685)) -* Bump pygithub from 1.58.1 to 1.58.2. ([\#15643](https://github.com/matrix-org/synapse/issues/15643)) -* Bump requests from 2.28.2 to 2.31.0. ([\#15651](https://github.com/matrix-org/synapse/issues/15651)) -* Bump sphinx from 6.1.3 to 6.2.1. ([\#15641](https://github.com/matrix-org/synapse/issues/15641)) -* Bump types-bleach from 6.0.0.1 to 6.0.0.3. ([\#15686](https://github.com/matrix-org/synapse/issues/15686)) -* Bump types-pillow from 9.5.0.2 to 9.5.0.4. ([\#15640](https://github.com/matrix-org/synapse/issues/15640)) -* Bump types-pyyaml from 6.0.12.9 to 6.0.12.10. ([\#15683](https://github.com/matrix-org/synapse/issues/15683)) -* Bump types-requests from 2.30.0.0 to 2.31.0.0. ([\#15684](https://github.com/matrix-org/synapse/issues/15684)) -* Bump types-setuptools from 67.7.0.2 to 67.8.0.0. ([\#15639](https://github.com/matrix-org/synapse/issues/15639)) - -Synapse 1.84.1 (2023-05-26) -=========================== - -This patch release fixes a major issue with homeservers that do not have an `instance_map` defined but which do use workers. -If you have already upgraded to Synapse 1.84.0 and your homeserver is working normally, then there is no need to update to this patch release. - - -Bugfixes --------- - -- Fix a bug introduced in Synapse v1.84.0 where workers do not start up when no `instance_map` was provided. ([\#15672](https://github.com/matrix-org/synapse/issues/15672)) - - -Internal Changes ----------------- - -- Add `dch` and `notify-send` to the development Nix flake so that the release script can be used. ([\#15673](https://github.com/matrix-org/synapse/issues/15673)) - - -Synapse 1.84.0 (2023-05-23) -=========================== - -The `worker_replication_*` configuration settings have been deprecated in favour of configuring the main process consistently with other instances in the `instance_map`. The deprecated settings will be removed in Synapse v1.88.0, but changing your configuration in advance is recommended. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.84/docs/upgrade.md#upgrading-to-v1840) for more information. - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.84.0rc1 where errors during startup were not reported correctly on Python < 3.10. ([\#15599](https://github.com/matrix-org/synapse/issues/15599)) - - -Synapse 1.84.0rc1 (2023-05-16) -============================== - -Features --------- - -- Add an option to prevent media downloads from configured domains. ([\#15197](https://github.com/matrix-org/synapse/issues/15197)) -- Add `forget_rooms_on_leave` config option to automatically forget rooms when users leave them or are removed from them. ([\#15224](https://github.com/matrix-org/synapse/issues/15224)) -- Add redis TLS configuration options. ([\#15312](https://github.com/matrix-org/synapse/issues/15312)) -- Add a config option to delay push notifications by a random amount, to discourage time-based profiling. ([\#15516](https://github.com/matrix-org/synapse/issues/15516)) -- Stabilize support for [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15528](https://github.com/matrix-org/synapse/issues/15528)) -- Implement [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009) to expand the supported characters in Matrix IDs. ([\#15536](https://github.com/matrix-org/synapse/issues/15536)) -- Advertise support for Matrix 1.6 on `/_matrix/client/versions`. ([\#15559](https://github.com/matrix-org/synapse/issues/15559)) -- Print full error and stack-trace of any exception that occurs during startup/initialization. ([\#15569](https://github.com/matrix-org/synapse/issues/15569)) - - -Bugfixes --------- - -- Don't fail on federation over TOR where SRV queries are not supported. Contributed by Zdzichu. ([\#15523](https://github.com/matrix-org/synapse/issues/15523)) -- Experimental support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which rejects setting the `"m.push_rules"` via account data. ([\#15554](https://github.com/matrix-org/synapse/issues/15554), [\#15555](https://github.com/matrix-org/synapse/issues/15555)) -- Fix a long-standing bug where an invalid membership event could cause an internal server error. ([\#15564](https://github.com/matrix-org/synapse/issues/15564)) -- Require at least poetry-core v1.1.0. ([\#15566](https://github.com/matrix-org/synapse/issues/15566), [\#15571](https://github.com/matrix-org/synapse/issues/15571)) - - -Deprecations and Removals -------------------------- - -- Remove need for `worker_replication_*` based settings in worker configuration yaml by placing this data directly on the `instance_map` instead. ([\#15491](https://github.com/matrix-org/synapse/issues/15491)) - - -Updates to the Docker image ---------------------------- - -- Add pkg-config package to Stage 0 to be able to build Dockerfile on ppc64le architecture. ([\#15567](https://github.com/matrix-org/synapse/issues/15567)) - - -Improved Documentation ----------------------- - -- Clarify documentation of the "Create or modify account" Admin API. ([\#15544](https://github.com/matrix-org/synapse/issues/15544)) -- Fix path to the `statistics/database/rooms` admin API in documentation. ([\#15560](https://github.com/matrix-org/synapse/issues/15560)) -- Update and improve Mastodon Single Sign-On documentation. ([\#15587](https://github.com/matrix-org/synapse/issues/15587)) - - -Internal Changes ----------------- - -- Use oEmbed to generate URL previews for YouTube Shorts. ([\#15025](https://github.com/matrix-org/synapse/issues/15025)) -- Create new `Client` for use with HTTP Replication between workers. Contributed by Jason Little. ([\#15470](https://github.com/matrix-org/synapse/issues/15470)) -- Bump pyicu from 2.10.2 to 2.11. ([\#15509](https://github.com/matrix-org/synapse/issues/15509)) -- Remove references to supporting per-user flag for [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654). ([\#15522](https://github.com/matrix-org/synapse/issues/15522)) -- Don't use a trusted key server when running the demo scripts. ([\#15527](https://github.com/matrix-org/synapse/issues/15527)) -- Speed up rebuilding of the user directory for local users. ([\#15529](https://github.com/matrix-org/synapse/issues/15529)) -- Speed up deleting of old rows in `event_push_actions`. ([\#15531](https://github.com/matrix-org/synapse/issues/15531)) -- Install the `xmlsec` and `mdbook` packages and switch back to the upstream [cachix/devenv](https://github.com/cachix/devenv) repo in the nix development environment. ([\#15532](https://github.com/matrix-org/synapse/issues/15532), [\#15533](https://github.com/matrix-org/synapse/issues/15533), [\#15545](https://github.com/matrix-org/synapse/issues/15545)) -- Implement [MSC3987](https://github.com/matrix-org/matrix-spec-proposals/pull/3987) by removing `"dont_notify"` from the list of actions in default push rules. ([\#15534](https://github.com/matrix-org/synapse/issues/15534)) -- Move various module API callback registration methods to a dedicated class. ([\#15535](https://github.com/matrix-org/synapse/issues/15535)) -- Proxy `/user/devices` federation queries to application services for [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984). ([\#15539](https://github.com/matrix-org/synapse/issues/15539)) -- Factor out an `is_mine_server_name` method. ([\#15542](https://github.com/matrix-org/synapse/issues/15542)) -- Allow running Complement tests using [podman](https://podman.io/) by adding a `PODMAN` environment variable to `scripts-dev/complement.sh`. ([\#15543](https://github.com/matrix-org/synapse/issues/15543)) -- Bump serde from 1.0.160 to 1.0.162. ([\#15548](https://github.com/matrix-org/synapse/issues/15548)) -- Bump types-setuptools from 67.6.0.5 to 67.7.0.1. ([\#15549](https://github.com/matrix-org/synapse/issues/15549)) -- Bump sentry-sdk from 1.19.1 to 1.22.1. ([\#15550](https://github.com/matrix-org/synapse/issues/15550)) -- Bump ruff from 0.0.259 to 0.0.265. ([\#15551](https://github.com/matrix-org/synapse/issues/15551)) -- Bump hiredis from 2.2.2 to 2.2.3. ([\#15552](https://github.com/matrix-org/synapse/issues/15552)) -- Bump types-requests from 2.29.0.0 to 2.30.0.0. ([\#15553](https://github.com/matrix-org/synapse/issues/15553)) -- Add `org.matrix.msc3981` info to `/_matrix/client/versions`. ([\#15558](https://github.com/matrix-org/synapse/issues/15558)) -- Declare unstable support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) under `/_matrix/client/versions` if the experimental implementation is enabled. ([\#15562](https://github.com/matrix-org/synapse/issues/15562)) -- Implement [MSC3821](https://github.com/matrix-org/matrix-spec-proposals/pull/3821) to update the redaction rules. ([\#15563](https://github.com/matrix-org/synapse/issues/15563)) -- Implement updated redaction rules from [MSC3389](https://github.com/matrix-org/matrix-spec-proposals/pull/3389). ([\#15565](https://github.com/matrix-org/synapse/issues/15565)) -- Allow `pip install` to use setuptools_rust 1.6.0 when building Synapse. ([\#15570](https://github.com/matrix-org/synapse/issues/15570)) -- Deal with upcoming Github Actions deprecations. ([\#15576](https://github.com/matrix-org/synapse/issues/15576)) -- Export `run_as_background_process` from the module API. ([\#15577](https://github.com/matrix-org/synapse/issues/15577)) -- Update build system requirements to allow building with poetry-core==1.6.0. ([\#15588](https://github.com/matrix-org/synapse/issues/15588)) -- Bump serde from 1.0.162 to 1.0.163. ([\#15589](https://github.com/matrix-org/synapse/issues/15589)) -- Bump phonenumbers from 8.13.7 to 8.13.11. ([\#15590](https://github.com/matrix-org/synapse/issues/15590)) -- Bump types-psycopg2 from 2.9.21.9 to 2.9.21.10. ([\#15591](https://github.com/matrix-org/synapse/issues/15591)) -- Bump types-commonmark from 0.9.2.2 to 0.9.2.3. ([\#15592](https://github.com/matrix-org/synapse/issues/15592)) -- Bump types-setuptools from 67.7.0.1 to 67.7.0.2. ([\#15594](https://github.com/matrix-org/synapse/issues/15594)) - - -Synapse 1.83.0 (2023-05-09) -=========================== - -No significant changes since 1.83.0rc1. - - -Synapse 1.83.0rc1 (2023-05-02) -============================== - -Features --------- - -- Experimental support to recursively provide relations per [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981). ([\#15315](https://github.com/matrix-org/synapse/issues/15315)) -- Experimental support for [MSC3970](https://github.com/matrix-org/matrix-spec-proposals/pull/3970): Scope transaction IDs to devices. ([\#15318](https://github.com/matrix-org/synapse/issues/15318)) -- Add an [admin API endpoint](https://matrix-org.github.io/synapse/v1.83/admin_api/experimental_features.html) to support per-user feature flags. ([\#15344](https://github.com/matrix-org/synapse/issues/15344)) -- Add a module API to send an HTTP push notification. ([\#15387](https://github.com/matrix-org/synapse/issues/15387)) -- Add an [admin API endpoint](https://matrix-org.github.io/synapse/v1.83/admin_api/statistics.html#get-largest-rooms-by-size-in-database) to query the largest rooms by disk space used in the database. ([\#15482](https://github.com/matrix-org/synapse/issues/15482)) - - -Bugfixes --------- - -- Disable push rule evaluation for rooms excluded from sync. ([\#15361](https://github.com/matrix-org/synapse/issues/15361)) -- Fix a long-standing bug where cached server key results which were directly fetched would not be properly re-used. ([\#15417](https://github.com/matrix-org/synapse/issues/15417)) -- Fix a bug introduced in Synapse 1.73.0 where some experimental push rules were returned by default. ([\#15494](https://github.com/matrix-org/synapse/issues/15494)) - - -Improved Documentation ----------------------- - -- Add Nginx loadbalancing example with sticky mxid for workers. ([\#15411](https://github.com/matrix-org/synapse/issues/15411)) -- Update outdated development docs that mention restrictions in versions of SQLite that we no longer support. ([\#15498](https://github.com/matrix-org/synapse/issues/15498)) - - -Internal Changes ----------------- - -- Speedup tests by caching HomeServerConfig instances. ([\#15284](https://github.com/matrix-org/synapse/issues/15284)) -- Add denormalised event stream ordering column to membership state tables for future use. Contributed by Nick @ Beeper (@fizzadar). ([\#15356](https://github.com/matrix-org/synapse/issues/15356)) -- Always use multi-user device resync replication endpoints. ([\#15418](https://github.com/matrix-org/synapse/issues/15418)) -- Add column `full_user_id` to tables `profiles` and `user_filters`. ([\#15458](https://github.com/matrix-org/synapse/issues/15458)) -- Update support for [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) to allow always returning fallback-keys in a `/keys/claim` request. ([\#15462](https://github.com/matrix-org/synapse/issues/15462)) -- Improve type hints. ([\#15465](https://github.com/matrix-org/synapse/issues/15465), [\#15496](https://github.com/matrix-org/synapse/issues/15496), [\#15497](https://github.com/matrix-org/synapse/issues/15497)) -- Support claiming more than one OTK at a time. ([\#15468](https://github.com/matrix-org/synapse/issues/15468)) -- Bump types-pyyaml from 6.0.12.8 to 6.0.12.9. ([\#15471](https://github.com/matrix-org/synapse/issues/15471)) -- Bump pyasn1-modules from 0.2.8 to 0.3.0. ([\#15473](https://github.com/matrix-org/synapse/issues/15473)) -- Bump cryptography from 40.0.1 to 40.0.2. ([\#15474](https://github.com/matrix-org/synapse/issues/15474)) -- Bump types-netaddr from 0.8.0.7 to 0.8.0.8. ([\#15475](https://github.com/matrix-org/synapse/issues/15475)) -- Bump types-jsonschema from 4.17.0.6 to 4.17.0.7. ([\#15476](https://github.com/matrix-org/synapse/issues/15476)) -- Ask bug reporters to provide logs as text. ([\#15479](https://github.com/matrix-org/synapse/issues/15479)) -- Add a Nix flake for use as a development environment. ([\#15495](https://github.com/matrix-org/synapse/issues/15495)) -- Bump anyhow from 1.0.70 to 1.0.71. ([\#15507](https://github.com/matrix-org/synapse/issues/15507)) -- Bump types-pillow from 9.4.0.19 to 9.5.0.2. ([\#15508](https://github.com/matrix-org/synapse/issues/15508)) -- Bump packaging from 23.0 to 23.1. ([\#15510](https://github.com/matrix-org/synapse/issues/15510)) -- Bump types-requests from 2.28.11.16 to 2.29.0.0. ([\#15511](https://github.com/matrix-org/synapse/issues/15511)) -- Bump setuptools-rust from 1.5.2 to 1.6.0. ([\#15512](https://github.com/matrix-org/synapse/issues/15512)) -- Update the check_schema_delta script to account for when the schema version has been bumped locally. ([\#15466](https://github.com/matrix-org/synapse/issues/15466)) - - -Synapse 1.82.0 (2023-04-25) -=========================== - -No significant changes since 1.82.0rc1. - - -Synapse 1.82.0rc1 (2023-04-18) -============================== - -Features --------- - -- Allow loading the `/directory/room/{roomAlias}` endpoint on workers. ([\#15333](https://github.com/matrix-org/synapse/issues/15333)) -- Add some validation to `instance_map` configuration loading. ([\#15431](https://github.com/matrix-org/synapse/issues/15431)) -- Allow loading the `/capabilities` endpoint on workers. ([\#15436](https://github.com/matrix-org/synapse/issues/15436)) - - -Bugfixes --------- - -- Delete server-side backup keys when deactivating an account. ([\#15181](https://github.com/matrix-org/synapse/issues/15181)) -- Fix and document untold assumption that `on_logged_out` module hooks will be called before the deletion of pushers. ([\#15410](https://github.com/matrix-org/synapse/issues/15410)) -- Improve robustness when handling a perspective key response by deduplicating received server keys. ([\#15423](https://github.com/matrix-org/synapse/issues/15423)) -- Synapse now correctly fails to start if the config option `app_service_config_files` is not a list. ([\#15425](https://github.com/matrix-org/synapse/issues/15425)) -- Disable loading `RefreshTokenServlet` (`/_matrix/client/(r0|v3|unstable)/refresh`) on workers. ([\#15428](https://github.com/matrix-org/synapse/issues/15428)) - - -Improved Documentation ----------------------- - -- Note that the `delete_stale_devices_after` background job always runs on the main process. ([\#15452](https://github.com/matrix-org/synapse/issues/15452)) - - -Deprecations and Removals -------------------------- - -- Remove the broken, unspecced registration fallback. Note that the *login* fallback is unaffected by this change. ([\#15405](https://github.com/matrix-org/synapse/issues/15405)) - - -Internal Changes ----------------- - -- Bump black from 23.1.0 to 23.3.0. ([\#15372](https://github.com/matrix-org/synapse/issues/15372)) -- Bump pyopenssl from 23.1.0 to 23.1.1. ([\#15373](https://github.com/matrix-org/synapse/issues/15373)) -- Bump types-psycopg2 from 2.9.21.8 to 2.9.21.9. ([\#15374](https://github.com/matrix-org/synapse/issues/15374)) -- Bump types-netaddr from 0.8.0.6 to 0.8.0.7. ([\#15375](https://github.com/matrix-org/synapse/issues/15375)) -- Bump types-opentracing from 2.4.10.3 to 2.4.10.4. ([\#15376](https://github.com/matrix-org/synapse/issues/15376)) -- Bump dawidd6/action-download-artifact from 2.26.0 to 2.26.1. ([\#15404](https://github.com/matrix-org/synapse/issues/15404)) -- Bump parameterized from 0.8.1 to 0.9.0. ([\#15412](https://github.com/matrix-org/synapse/issues/15412)) -- Bump types-pillow from 9.4.0.17 to 9.4.0.19. ([\#15413](https://github.com/matrix-org/synapse/issues/15413)) -- Bump sentry-sdk from 1.17.0 to 1.19.1. ([\#15414](https://github.com/matrix-org/synapse/issues/15414)) -- Bump immutabledict from 2.2.3 to 2.2.4. ([\#15415](https://github.com/matrix-org/synapse/issues/15415)) -- Bump dawidd6/action-download-artifact from 2.26.1 to 2.27.0. ([\#15441](https://github.com/matrix-org/synapse/issues/15441)) -- Bump serde_json from 1.0.95 to 1.0.96. ([\#15442](https://github.com/matrix-org/synapse/issues/15442)) -- Bump serde from 1.0.159 to 1.0.160. ([\#15443](https://github.com/matrix-org/synapse/issues/15443)) -- Bump pillow from 9.4.0 to 9.5.0. ([\#15444](https://github.com/matrix-org/synapse/issues/15444)) -- Bump furo from 2023.3.23 to 2023.3.27. ([\#15445](https://github.com/matrix-org/synapse/issues/15445)) -- Bump types-pyopenssl from 23.1.0.0 to 23.1.0.2. ([\#15446](https://github.com/matrix-org/synapse/issues/15446)) -- Bump mypy from 1.0.0 to 1.0.1. ([\#15447](https://github.com/matrix-org/synapse/issues/15447)) -- Bump psycopg2 from 2.9.5 to 2.9.6. ([\#15448](https://github.com/matrix-org/synapse/issues/15448)) -- Improve DB performance of clearing out old data from `stream_ordering_to_exterm`. ([\#15382](https://github.com/matrix-org/synapse/issues/15382), [\#15429](https://github.com/matrix-org/synapse/issues/15429)) -- Implement [MSC3989](https://github.com/matrix-org/matrix-spec-proposals/pull/3989) redaction algorithm. ([\#15393](https://github.com/matrix-org/synapse/issues/15393)) -- Implement [MSC2175](https://github.com/matrix-org/matrix-doc/pull/2175) to stop adding `creator` to create events. ([\#15394](https://github.com/matrix-org/synapse/issues/15394)) -- Implement [MSC2174](https://github.com/matrix-org/matrix-spec-proposals/pull/2174) to move the `redacts` key to a `content` property. ([\#15395](https://github.com/matrix-org/synapse/issues/15395)) -- Trust dtonlay/rust-toolchain in CI. ([\#15406](https://github.com/matrix-org/synapse/issues/15406)) -- Explicitly install Synapse during typechecking in CI. ([\#15409](https://github.com/matrix-org/synapse/issues/15409)) -- Only load the SSO redirect servlet if SSO is enabled. ([\#15421](https://github.com/matrix-org/synapse/issues/15421)) -- Refactor `SimpleHttpClient` to pull out a base class. ([\#15427](https://github.com/matrix-org/synapse/issues/15427)) -- Improve type hints. ([\#15432](https://github.com/matrix-org/synapse/issues/15432)) -- Convert async to normal tests in `TestSSOHandler`. ([\#15433](https://github.com/matrix-org/synapse/issues/15433)) -- Speed up the user directory background update. ([\#15435](https://github.com/matrix-org/synapse/issues/15435)) -- Disable directory listing for static resources in `/_matrix/static/`. ([\#15438](https://github.com/matrix-org/synapse/issues/15438)) -- Move various module API callback registration methods to a dedicated class. ([\#15453](https://github.com/matrix-org/synapse/issues/15453)) - - -Synapse 1.81.0 (2023-04-11) -=========================== - -Synapse now attempts the versioned appservice paths before falling back to the -[legacy paths](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). -Usage of the legacy routes should be considered deprecated. - -Additionally, Synapse has supported sending the application service access token -via [the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization) -since v1.70.0. For backwards compatibility it is *also* sent as the `access_token` -query parameter. This is insecure and should be considered deprecated. - -A future version of Synapse (v1.88.0 or later) will remove support for legacy -application service routes and query parameter authorization. - - -No significant changes since 1.81.0rc2. - - -Synapse 1.81.0rc2 (2023-04-06) -============================== - -Bugfixes --------- - -- Fix the `set_device_id_for_pushers_txn` background update crash. ([\#15391](https://github.com/matrix-org/synapse/issues/15391)) - - -Internal Changes ----------------- - -- Update CI to run complement under the latest stable go version. ([\#15403](https://github.com/matrix-org/synapse/issues/15403)) - - -Synapse 1.81.0rc1 (2023-04-04) -============================== - -Features --------- - -- Add the ability to enable/disable registrations when in the OIDC flow. ([\#14978](https://github.com/matrix-org/synapse/issues/14978)) -- Add a primitive helper script for listing worker endpoints. ([\#15243](https://github.com/matrix-org/synapse/issues/15243)) -- Experimental support for passing One Time Key and device key requests to application services ([MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) and [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984)). ([\#15314](https://github.com/matrix-org/synapse/issues/15314), [\#15321](https://github.com/matrix-org/synapse/issues/15321)) -- Allow loading `/password_policy` endpoint on workers. ([\#15331](https://github.com/matrix-org/synapse/issues/15331)) -- Add experimental support for Unix sockets. Contributed by Jason Little. ([\#15353](https://github.com/matrix-org/synapse/issues/15353)) -- Build Debian packages for Ubuntu 23.04 (Lunar Lobster). ([\#15381](https://github.com/matrix-org/synapse/issues/15381)) - - -Bugfixes --------- - -- Fix a long-standing bug where edits of non-`m.room.message` events would not be correctly bundled. ([\#15295](https://github.com/matrix-org/synapse/issues/15295)) -- Fix a bug introduced in Synapse v1.55.0 which could delay remote homeservers being able to decrypt encrypted messages sent by local users. ([\#15297](https://github.com/matrix-org/synapse/issues/15297)) -- Add a check to [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) - to ensure that the sqlite database passed to the script exists before trying to port from it. ([\#15306](https://github.com/matrix-org/synapse/issues/15306)) -- Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key. ([\#15309](https://github.com/matrix-org/synapse/issues/15309)) -- Fix a long-standing bug that Synpase only used the [legacy appservice routes](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). ([\#15317](https://github.com/matrix-org/synapse/issues/15317)) -- Fix a long-standing bug preventing users from rejoining rooms after being banned and unbanned over federation. Contributed by Nico. ([\#15323](https://github.com/matrix-org/synapse/issues/15323)) -- Fix bug in worker mode where on a rolling restart of workers the "typing" worker would consume 100% CPU until it got restarted. ([\#15332](https://github.com/matrix-org/synapse/issues/15332)) -- Fix a long-standing bug where some to_device messages could be dropped when using workers. ([\#15349](https://github.com/matrix-org/synapse/issues/15349)) -- Fix a bug introduced in Synapse 1.70.0 where the background sync from a faster join could spin for hours when one of the events involved had been marked for backoff. ([\#15351](https://github.com/matrix-org/synapse/issues/15351)) -- Fix missing app variable in mail subject for password resets. Contributed by Cyberes. ([\#15352](https://github.com/matrix-org/synapse/issues/15352)) -- Fix a rare bug introduced in Synapse 1.66.0 where initial syncs would fail when the user had been kicked from a faster joined room that had not finished syncing. ([\#15383](https://github.com/matrix-org/synapse/issues/15383)) - - -Improved Documentation ----------------------- - -- Fix a typo in login requests ratelimit defaults. ([\#15341](https://github.com/matrix-org/synapse/issues/15341)) -- Add some clarification to the doc/comments regarding TCP replication. ([\#15354](https://github.com/matrix-org/synapse/issues/15354)) -- Note that Synapse 1.74 queued a rebuild of the user directory tables. ([\#15386](https://github.com/matrix-org/synapse/issues/15386)) - - -Internal Changes ----------------- - -- Use `immutabledict` instead of `frozendict`. ([\#15113](https://github.com/matrix-org/synapse/issues/15113)) -- Add developer documentation for the Federation Sender and add a documentation mechanism using Sphinx. ([\#15265](https://github.com/matrix-org/synapse/issues/15265), [\#15336](https://github.com/matrix-org/synapse/issues/15336)) -- Make the pushers rely on the `device_id` instead of the `access_token_id` for various operations. ([\#15280](https://github.com/matrix-org/synapse/issues/15280)) -- Bump sentry-sdk from 1.15.0 to 1.17.0. ([\#15285](https://github.com/matrix-org/synapse/issues/15285)) -- Allow running the Twisted trunk job against other branches. ([\#15302](https://github.com/matrix-org/synapse/issues/15302)) -- Remind the releaser to ask for changelog feedback in [#synapse-dev](https://matrix.to/#/#synapse-dev:matrix.org). ([\#15303](https://github.com/matrix-org/synapse/issues/15303)) -- Bump dtolnay/rust-toolchain from e12eda571dc9a5ee5d58eecf4738ec291c66f295 to fc3253060d0c959bea12a59f10f8391454a0b02d. ([\#15304](https://github.com/matrix-org/synapse/issues/15304)) -- Reject events with an invalid "mentions" property per [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15311](https://github.com/matrix-org/synapse/issues/15311)) -- As an optimisation, use `TRUNCATE` on Postgres when clearing the user directory tables. ([\#15316](https://github.com/matrix-org/synapse/issues/15316)) -- Fix `.gitignore` rule for the Complement source tarball downloaded automatically by `complement.sh`. ([\#15319](https://github.com/matrix-org/synapse/issues/15319)) -- Bump serde from 1.0.157 to 1.0.158. ([\#15324](https://github.com/matrix-org/synapse/issues/15324)) -- Bump regex from 1.7.1 to 1.7.3. ([\#15325](https://github.com/matrix-org/synapse/issues/15325)) -- Bump types-pyopenssl from 23.0.0.4 to 23.1.0.0. ([\#15326](https://github.com/matrix-org/synapse/issues/15326)) -- Bump furo from 2022.12.7 to 2023.3.23. ([\#15327](https://github.com/matrix-org/synapse/issues/15327)) -- Bump ruff from 0.0.252 to 0.0.259. ([\#15328](https://github.com/matrix-org/synapse/issues/15328)) -- Bump cryptography from 40.0.0 to 40.0.1. ([\#15329](https://github.com/matrix-org/synapse/issues/15329)) -- Bump mypy-zope from 0.9.0 to 0.9.1. ([\#15330](https://github.com/matrix-org/synapse/issues/15330)) -- Speed up unit tests when using SQLite3. ([\#15334](https://github.com/matrix-org/synapse/issues/15334)) -- Speed up pydantic CI job. ([\#15339](https://github.com/matrix-org/synapse/issues/15339)) -- Speed up sample config CI job. ([\#15340](https://github.com/matrix-org/synapse/issues/15340)) -- Fix copyright year in SSO footer template. ([\#15358](https://github.com/matrix-org/synapse/issues/15358)) -- Bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3. ([\#15369](https://github.com/matrix-org/synapse/issues/15369)) -- Bump serde from 1.0.158 to 1.0.159. ([\#15370](https://github.com/matrix-org/synapse/issues/15370)) -- Bump serde_json from 1.0.94 to 1.0.95. ([\#15371](https://github.com/matrix-org/synapse/issues/15371)) -- Speed up membership queries for users with forgotten rooms. ([\#15385](https://github.com/matrix-org/synapse/issues/15385)) - - -Synapse 1.80.0 (2023-03-28) -=========================== - -No significant changes since 1.80.0rc2. - - -Synapse 1.80.0rc2 (2023-03-22) -============================== - -Bugfixes --------- - -- Fix a bug in which the [`POST /_matrix/client/v3/rooms/{roomId}/report/{eventId}`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3roomsroomidreporteventid) endpoint would return the wrong error if the user did not have permission to view the event. This aligns Synapse's implementation with [MSC2249](https://github.com/matrix-org/matrix-spec-proposals/pull/2249). ([\#15298](https://github.com/matrix-org/synapse/issues/15298), [\#15300](https://github.com/matrix-org/synapse/issues/15300)) -- Fix a bug introduced in Synapse 1.75.0rc1 where the [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) - would fail to open the SQLite database. ([\#15301](https://github.com/matrix-org/synapse/issues/15301)) - - -Synapse 1.80.0rc1 (2023-03-21) -============================== - -Features --------- - -- Stabilise support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): `event_property_contains` push condition. ([\#15187](https://github.com/matrix-org/synapse/issues/15187)) -- Implement [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15249](https://github.com/matrix-org/synapse/issues/15249)) -- Allow loading `/register/available` endpoint on workers. ([\#15268](https://github.com/matrix-org/synapse/issues/15268)) -- Improve performance of creating and authenticating events. ([\#15195](https://github.com/matrix-org/synapse/issues/15195)) -- Add topic and name events to group of events that are batch persisted when creating a room. ([\#15229](https://github.com/matrix-org/synapse/issues/15229)) - - -Bugfixes --------- - -- Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change. ([\#14755](https://github.com/matrix-org/synapse/issues/14755), [\#14756](https://github.com/matrix-org/synapse/issues/14756)) -- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to fix a long-standing bug where properties with dots were handled ambiguously in push rules. ([\#15190](https://github.com/matrix-org/synapse/issues/15190)) -- Faster joins: Fix a bug introduced in Synapse 1.66 where spurious "Failed to find memberships ..." errors would be logged. ([\#15232](https://github.com/matrix-org/synapse/issues/15232)) -- Fix a long-standing error when sending message into deleted room. ([\#15235](https://github.com/matrix-org/synapse/issues/15235)) - - -Updates to the Docker image ---------------------------- - -- Ensure the Dockerfile builds on platforms that don't have a `cryptography` wheel. ([\#15239](https://github.com/matrix-org/synapse/issues/15239)) -- Mirror images to the GitHub Container Registry (`ghcr.io/matrix-org/synapse`). ([\#15281](https://github.com/matrix-org/synapse/issues/15281), [\#15282](https://github.com/matrix-org/synapse/issues/15282)) - - -Improved Documentation ----------------------- - -- Add a missing endpoint to the workers documentation. ([\#15223](https://github.com/matrix-org/synapse/issues/15223)) - - -Internal Changes ----------------- - -- Add additional functionality to declaring worker types when starting Complement in worker mode. ([\#14921](https://github.com/matrix-org/synapse/issues/14921)) -- Add `Synapse-Trace-Id` to `access-control-expose-headers` header. ([\#14974](https://github.com/matrix-org/synapse/issues/14974)) -- Make the `HttpTransactionCache` use the `Requester` in addition of the just the `Request` to build the transaction key. ([\#15200](https://github.com/matrix-org/synapse/issues/15200)) -- Improve log lines when purging rooms. ([\#15222](https://github.com/matrix-org/synapse/issues/15222)) -- Improve type hints. ([\#15230](https://github.com/matrix-org/synapse/issues/15230), [\#15231](https://github.com/matrix-org/synapse/issues/15231), [\#15238](https://github.com/matrix-org/synapse/issues/15238)) -- Move various module API callback registration methods to a dedicated class. ([\#15237](https://github.com/matrix-org/synapse/issues/15237)) -- Configure GitHub Actions for merge queues. ([\#15244](https://github.com/matrix-org/synapse/issues/15244)) -- Add schema comments about the `destinations` and `destination_rooms` tables. ([\#15247](https://github.com/matrix-org/synapse/issues/15247)) -- Skip processing of auto-join room behaviour if there are no auto-join rooms configured. ([\#15262](https://github.com/matrix-org/synapse/issues/15262)) -- Remove unused store method `_set_destination_retry_timings_emulated`. ([\#15266](https://github.com/matrix-org/synapse/issues/15266)) -- Reorganize URL preview code. ([\#15269](https://github.com/matrix-org/synapse/issues/15269)) -- Clean-up direct TCP replication code. ([\#15272](https://github.com/matrix-org/synapse/issues/15272), [\#15274](https://github.com/matrix-org/synapse/issues/15274)) -- Make `configure_workers_and_start` script used in Complement tests compatible with older versions of Python. ([\#15275](https://github.com/matrix-org/synapse/issues/15275)) -- Add a `/versions` flag for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15293](https://github.com/matrix-org/synapse/issues/15293)) -- Bump hiredis from 2.2.1 to 2.2.2. ([\#15252](https://github.com/matrix-org/synapse/issues/15252)) -- Bump serde from 1.0.152 to 1.0.155. ([\#15253](https://github.com/matrix-org/synapse/issues/15253)) -- Bump pysaml2 from 7.2.1 to 7.3.1. ([\#15254](https://github.com/matrix-org/synapse/issues/15254)) -- Bump msgpack from 1.0.4 to 1.0.5. ([\#15255](https://github.com/matrix-org/synapse/issues/15255)) -- Bump gitpython from 3.1.30 to 3.1.31. ([\#15256](https://github.com/matrix-org/synapse/issues/15256)) -- Bump cryptography from 39.0.1 to 39.0.2. ([\#15257](https://github.com/matrix-org/synapse/issues/15257)) -- Bump pydantic from 1.10.4 to 1.10.6. ([\#15286](https://github.com/matrix-org/synapse/issues/15286)) -- Bump serde from 1.0.155 to 1.0.157. ([\#15287](https://github.com/matrix-org/synapse/issues/15287)) -- Bump anyhow from 1.0.69 to 1.0.70. ([\#15288](https://github.com/matrix-org/synapse/issues/15288)) -- Bump txredisapi from 1.4.7 to 1.4.9. ([\#15289](https://github.com/matrix-org/synapse/issues/15289)) -- Bump pygithub from 1.57 to 1.58.1. ([\#15290](https://github.com/matrix-org/synapse/issues/15290)) -- Bump types-requests from 2.28.11.12 to 2.28.11.15. ([\#15291](https://github.com/matrix-org/synapse/issues/15291)) - - - -Synapse 1.79.0 (2023-03-14) -=========================== - -No significant changes since 1.79.0rc2. - - -Synapse 1.79.0rc2 (2023-03-13) -============================== - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.79.0rc1 where attempting to register a `on_remove_user_third_party_identifier` module API callback would be a no-op. ([\#15227](https://github.com/matrix-org/synapse/issues/15227)) -- Fix a rare bug introduced in Synapse 1.73 where events could remain unsent to other homeservers after a faster-join to a room. ([\#15248](https://github.com/matrix-org/synapse/issues/15248)) - - -Internal Changes ----------------- - -- Refactor `filter_events_for_server`. ([\#15240](https://github.com/matrix-org/synapse/issues/15240)) - - -Synapse 1.79.0rc1 (2023-03-07) -============================== - -Features --------- - -- Add two new Third Party Rules module API callbacks: [`on_add_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_add_user_third_party_identifier) and [`on_remove_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_remove_user_third_party_identifier). ([\#15044](https://github.com/matrix-org/synapse/issues/15044)) -- Experimental support for [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967) to not require UIA for setting up cross-signing on first use. ([\#15077](https://github.com/matrix-org/synapse/issues/15077)) -- Add media information to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.79/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#15107](https://github.com/matrix-org/synapse/issues/15107)) -- Add an [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) to delete a [specific event report](https://spec.matrix.org/v1.6/client-server-api/#reporting-content). ([\#15116](https://github.com/matrix-org/synapse/issues/15116)) -- Add support for knocking to workers. ([\#15133](https://github.com/matrix-org/synapse/issues/15133)) -- Allow use of the `/filter` Client-Server APIs on workers. ([\#15134](https://github.com/matrix-org/synapse/issues/15134)) -- Update support for [MSC2677](https://github.com/matrix-org/matrix-spec-proposals/pull/2677): remove support for server-side aggregation of reactions. ([\#15172](https://github.com/matrix-org/synapse/issues/15172)) -- Stabilise support for [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758): `event_property_is` push condition. ([\#15185](https://github.com/matrix-org/synapse/issues/15185)) - - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.75 that caused experimental support for deleting account data to raise an internal server error while using an account data writer worker. ([\#14869](https://github.com/matrix-org/synapse/issues/14869)) -- Fix a long-standing bug where Synapse handled an unspecced field on push rules. ([\#15088](https://github.com/matrix-org/synapse/issues/15088)) -- Fix a long-standing bug where a URL preview would break if the discovered oEmbed failed to download. ([\#15092](https://github.com/matrix-org/synapse/issues/15092)) -- Fix a long-standing bug where an initial sync would not respond to changes to the list of ignored users if there was an initial sync cached. ([\#15163](https://github.com/matrix-org/synapse/issues/15163)) -- Add the `transaction_id` in the events included in many endpoints' responses. ([\#15174](https://github.com/matrix-org/synapse/issues/15174)) -- Fix a bug introduced in Synapse 1.78.0 where requests to claim dehydrated devices would fail with a `405` error. ([\#15180](https://github.com/matrix-org/synapse/issues/15180)) -- Stop applying edits when bundling aggregations, per [MSC3925](https://github.com/matrix-org/matrix-spec-proposals/pull/3925). ([\#15193](https://github.com/matrix-org/synapse/issues/15193)) -- Fix a long-standing bug where the user directory search was not case-insensitive for accented characters. ([\#15143](https://github.com/matrix-org/synapse/issues/15143)) - - -Updates to the Docker image ---------------------------- - -- Improve startup logging in the with-workers Docker image. ([\#15186](https://github.com/matrix-org/synapse/issues/15186)) - - -Improved Documentation ----------------------- - -- Document how to use caches in a module. ([\#14026](https://github.com/matrix-org/synapse/issues/14026)) -- Clarify which worker processes the ThirdPartyRules' [`on_new_event`](https://matrix-org.github.io/synapse/v1.78/modules/third_party_rules_callbacks.html#on_new_event) module API callback runs on. ([\#15071](https://github.com/matrix-org/synapse/issues/15071)) -- Document using [Shibboleth](https://www.shibboleth.net/) as an OpenID Provider. ([\#15112](https://github.com/matrix-org/synapse/issues/15112)) -- Correct reference to `federation_verify_certificates` in configuration documentation. ([\#15139](https://github.com/matrix-org/synapse/issues/15139)) -- Correct small documentation errors in some `MatrixFederationHttpClient` methods. ([\#15148](https://github.com/matrix-org/synapse/issues/15148)) -- Correct the description of the behavior of `registration_shared_secret_path` on startup. ([\#15168](https://github.com/matrix-org/synapse/issues/15168)) - - -Deprecations and Removals -------------------------- - -- Deprecate the `on_threepid_bind` module callback, to be replaced by [`on_add_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_add_user_third_party_identifier). See [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.79/docs/upgrade.md#upgrading-to-v1790). ([\#15044](https://github.com/matrix-org/synapse/issues/15044)) -- Remove the unspecced `room_alias` field from the [`/createRoom`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3createroom) response. ([\#15093](https://github.com/matrix-org/synapse/issues/15093)) -- Remove the unspecced `PUT` on the `/knock/{roomIdOrAlias}` endpoint. ([\#15189](https://github.com/matrix-org/synapse/issues/15189)) -- Remove the undocumented and unspecced `type` parameter to the `/thumbnail` endpoint. ([\#15137](https://github.com/matrix-org/synapse/issues/15137)) -- Remove unspecced and buggy `PUT` method on the unstable `/rooms//batch_send` endpoint. ([\#15199](https://github.com/matrix-org/synapse/issues/15199)) - - -Internal Changes ----------------- - -- Run the integration test suites with the asyncio reactor enabled in CI. ([\#14101](https://github.com/matrix-org/synapse/issues/14101)) -- Batch up storing state groups when creating a new room. ([\#14918](https://github.com/matrix-org/synapse/issues/14918)) -- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15051](https://github.com/matrix-org/synapse/issues/15051)) -- Refactor writing json data in `FileExfiltrationWriter`. ([\#15095](https://github.com/matrix-org/synapse/issues/15095)) -- Tighten the login ratelimit defaults. ([\#15135](https://github.com/matrix-org/synapse/issues/15135)) -- Fix a typo in an experimental config setting. ([\#15138](https://github.com/matrix-org/synapse/issues/15138)) -- Refactor the media modules. ([\#15146](https://github.com/matrix-org/synapse/issues/15146), [\#15175](https://github.com/matrix-org/synapse/issues/15175)) -- Improve type hints. ([\#15164](https://github.com/matrix-org/synapse/issues/15164)) -- Move `get_event_report` and `get_event_reports_paginate` from `RoomStore` to `RoomWorkerStore`. ([\#15165](https://github.com/matrix-org/synapse/issues/15165)) -- Remove dangling reference to being a reference implementation in docstring. ([\#15167](https://github.com/matrix-org/synapse/issues/15167)) -- Add an option to force a rebuild of the "editable" complement image. ([\#15184](https://github.com/matrix-org/synapse/issues/15184)) -- Use nightly rustfmt in CI. ([\#15188](https://github.com/matrix-org/synapse/issues/15188)) -- Add a `get_next_txn` method to `StreamIdGenerator` to match `MultiWriterIdGenerator`. ([\#15191](https://github.com/matrix-org/synapse/issues/15191)) -- Combine `AbstractStreamIdTracker` and `AbstractStreamIdGenerator`. ([\#15192](https://github.com/matrix-org/synapse/issues/15192)) -- Automatically fix errors with `ruff`. ([\#15194](https://github.com/matrix-org/synapse/issues/15194)) -- Refactor database transaction for query users' devices to reduce database pool contention. ([\#15215](https://github.com/matrix-org/synapse/issues/15215)) -- Correct `test_icu_word_boundary_punctuation` so that it passes with the ICU versions available in Alpine and macOS. ([\#15177](https://github.com/matrix-org/synapse/issues/15177)) - -
Locked dependency updates - - - Bump actions/checkout from 2 to 3. ([\#15155](https://github.com/matrix-org/synapse/issues/15155)) - - Bump black from 22.12.0 to 23.1.0. ([\#15103](https://github.com/matrix-org/synapse/issues/15103)) - - Bump dawidd6/action-download-artifact from 2.25.0 to 2.26.0. ([\#15152](https://github.com/matrix-org/synapse/issues/15152)) - - Bump docker/login-action from 1 to 2. ([\#15154](https://github.com/matrix-org/synapse/issues/15154)) - - Bump matrix-org/backend-meta from 1 to 2. ([\#15156](https://github.com/matrix-org/synapse/issues/15156)) - - Bump ruff from 0.0.237 to 0.0.252. ([\#15159](https://github.com/matrix-org/synapse/issues/15159)) - - Bump serde_json from 1.0.93 to 1.0.94. ([\#15214](https://github.com/matrix-org/synapse/issues/15214)) - - Bump types-commonmark from 0.9.2.1 to 0.9.2.2. ([\#15209](https://github.com/matrix-org/synapse/issues/15209)) - - Bump types-opentracing from 2.4.10.1 to 2.4.10.3. ([\#15158](https://github.com/matrix-org/synapse/issues/15158)) - - Bump types-pillow from 9.4.0.13 to 9.4.0.17. ([\#15211](https://github.com/matrix-org/synapse/issues/15211)) - - Bump types-psycopg2 from 2.9.21.4 to 2.9.21.8. ([\#15210](https://github.com/matrix-org/synapse/issues/15210)) - - Bump types-pyopenssl from 22.1.0.2 to 23.0.0.4. ([\#15213](https://github.com/matrix-org/synapse/issues/15213)) - - Bump types-setuptools from 67.3.0.1 to 67.4.0.3. ([\#15160](https://github.com/matrix-org/synapse/issues/15160)) - - Bump types-setuptools from 67.4.0.3 to 67.5.0.0. ([\#15212](https://github.com/matrix-org/synapse/issues/15212)) - - Bump typing-extensions from 4.4.0 to 4.5.0. ([\#15157](https://github.com/matrix-org/synapse/issues/15157)) -
- - -Synapse 1.78.0 (2023-02-28) -=========================== - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.76 where 5s delays would occasionally occur in deployments using workers. ([\#15150](https://github.com/matrix-org/synapse/issues/15150)) - - -Synapse 1.78.0rc1 (2023-02-21) -============================== - -Features --------- - -- Implement the experimental `exact_event_match` push rule condition from [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758). ([\#14964](https://github.com/matrix-org/synapse/issues/14964)) -- Add account data to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.78/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14969](https://github.com/matrix-org/synapse/issues/14969)) -- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to disambiguate push rule keys with dots in them. ([\#15004](https://github.com/matrix-org/synapse/issues/15004)) -- Allow Synapse to use a specific Redis [logical database](https://redis.io/commands/select/) in worker-mode deployments. ([\#15034](https://github.com/matrix-org/synapse/issues/15034)) -- Tag opentracing spans for federation requests with the name of the worker serving the request. ([\#15042](https://github.com/matrix-org/synapse/issues/15042)) -- Implement the experimental `exact_event_property_contains` push rule condition from [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966). ([\#15045](https://github.com/matrix-org/synapse/issues/15045)) -- Remove spurious `dont_notify` action from the defaults for the `.m.rule.reaction` pushrule. ([\#15073](https://github.com/matrix-org/synapse/issues/15073)) -- Update the error code returned when user sends a duplicate annotation. ([\#15075](https://github.com/matrix-org/synapse/issues/15075)) - - -Bugfixes --------- - -- Prevent clients from reporting nonexistent events. ([\#13779](https://github.com/matrix-org/synapse/issues/13779)) -- Return spec-compliant JSON errors when unknown endpoints are requested. ([\#14605](https://github.com/matrix-org/synapse/issues/14605)) -- Fix a long-standing bug where the room aliases returned could be corrupted. ([\#15038](https://github.com/matrix-org/synapse/issues/15038)) -- Fix a bug introduced in Synapse 1.76.0 where partially-joined rooms could not be deleted using the [purge room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api). ([\#15068](https://github.com/matrix-org/synapse/issues/15068)) -- Fix a long-standing bug where federated joins would fail if the first server in the list of servers to try is not in the room. ([\#15074](https://github.com/matrix-org/synapse/issues/15074)) -- Fix a bug introduced in Synapse v1.74.0 where searching with colons when using ICU for search term tokenisation would fail with an error. ([\#15079](https://github.com/matrix-org/synapse/issues/15079)) -- Reduce the likelihood of a rare race condition where rejoining a restricted room over federation would fail. ([\#15080](https://github.com/matrix-org/synapse/issues/15080)) -- Fix a bug introduced in Synapse 1.76 where workers would fail to start if the `health` listener was configured. ([\#15096](https://github.com/matrix-org/synapse/issues/15096)) -- Fix a bug introduced in Synapse 1.75 where the [portdb script](https://matrix-org.github.io/synapse/release-v1.78/postgres.html#porting-from-sqlite) would fail to run after a room had been faster-joined. ([\#15108](https://github.com/matrix-org/synapse/issues/15108)) - - -Improved Documentation ----------------------- - -- Document how to start Synapse with Poetry. Contributed by @thezaidbintariq. ([\#14892](https://github.com/matrix-org/synapse/issues/14892), [\#15022](https://github.com/matrix-org/synapse/issues/15022)) -- Update delegation documentation to clarify that SRV DNS delegation does not eliminate all needs to serve files from .well-known locations. Contributed by @williamkray. ([\#14959](https://github.com/matrix-org/synapse/issues/14959)) -- Fix a mistake in registration_shared_secret_path docs. ([\#15078](https://github.com/matrix-org/synapse/issues/15078)) -- Refer to a more recent blog post on the [Database Maintenance Tools](https://matrix-org.github.io/synapse/latest/usage/administration/database_maintenance_tools.html) page. Contributed by @jahway603. ([\#15083](https://github.com/matrix-org/synapse/issues/15083)) - - -Internal Changes ----------------- - -- Re-type hint some collections as read-only. ([\#13755](https://github.com/matrix-org/synapse/issues/13755)) -- Faster joins: don't stall when another user joins during a partial-state room resync. ([\#14606](https://github.com/matrix-org/synapse/issues/14606)) -- Add a class `UnpersistedEventContext` to allow for the batching up of storing state groups. ([\#14675](https://github.com/matrix-org/synapse/issues/14675)) -- Add a check to ensure that locked dependencies have source distributions available. ([\#14742](https://github.com/matrix-org/synapse/issues/14742)) -- Tweak comment on `_is_local_room_accessible` as part of room visibility in `/hierarchy` to clarify the condition for a room being visible. ([\#14834](https://github.com/matrix-org/synapse/issues/14834)) -- Prevent `WARNING: there is already a transaction in progress` lines appearing in PostgreSQL's logs on some occasions. ([\#14840](https://github.com/matrix-org/synapse/issues/14840)) -- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14929](https://github.com/matrix-org/synapse/issues/14929)) -- Improve performance of `/sync` in a few situations. ([\#14973](https://github.com/matrix-org/synapse/issues/14973)) -- Limit concurrent event creation for a room to avoid state resolution when sending bursts of events to a local room. ([\#14977](https://github.com/matrix-org/synapse/issues/14977)) -- Skip calculating unread push actions in /sync when enable_push is false. ([\#14980](https://github.com/matrix-org/synapse/issues/14980)) -- Add a schema dump symlinks inside `contrib`, to make it easier for IDEs to interrogate Synapse's database schema. ([\#14982](https://github.com/matrix-org/synapse/issues/14982)) -- Improve type hints. ([\#15008](https://github.com/matrix-org/synapse/issues/15008), [\#15026](https://github.com/matrix-org/synapse/issues/15026), [\#15027](https://github.com/matrix-org/synapse/issues/15027), [\#15028](https://github.com/matrix-org/synapse/issues/15028), [\#15031](https://github.com/matrix-org/synapse/issues/15031), [\#15035](https://github.com/matrix-org/synapse/issues/15035), [\#15052](https://github.com/matrix-org/synapse/issues/15052), [\#15072](https://github.com/matrix-org/synapse/issues/15072), [\#15084](https://github.com/matrix-org/synapse/issues/15084)) -- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15037](https://github.com/matrix-org/synapse/issues/15037)) -- Avoid mutating a cached value in `get_user_devices_from_cache`. ([\#15040](https://github.com/matrix-org/synapse/issues/15040)) -- Fix a rare exception in logs on start up. ([\#15041](https://github.com/matrix-org/synapse/issues/15041)) -- Update pyo3-log to v0.8.1. ([\#15043](https://github.com/matrix-org/synapse/issues/15043)) -- Avoid mutating cached values in `_generate_sync_entry_for_account_data`. ([\#15047](https://github.com/matrix-org/synapse/issues/15047)) -- Refactor arguments of `try_unbind_threepid` and `_try_unbind_threepid_with_id_server` to not use dictionaries. ([\#15053](https://github.com/matrix-org/synapse/issues/15053)) -- Merge debug logging from the hotfixes branch. ([\#15054](https://github.com/matrix-org/synapse/issues/15054)) -- Faster joins: omit device list updates originating from partial state rooms in /sync responses without lazy loading of members enabled. ([\#15069](https://github.com/matrix-org/synapse/issues/15069)) -- Fix clashing database transaction name. ([\#15070](https://github.com/matrix-org/synapse/issues/15070)) -- Upper-bound frozendict dependency. This works around us being unable to test installing our wheels against Python 3.11 in CI. ([\#15114](https://github.com/matrix-org/synapse/issues/15114)) -- Tweak logging for when a worker waits for its view of a replication stream to catch up. ([\#15120](https://github.com/matrix-org/synapse/issues/15120)) - -
Locked dependency updates - -- Bump bleach from 5.0.1 to 6.0.0. ([\#15059](https://github.com/matrix-org/synapse/issues/15059)) -- Bump cryptography from 38.0.4 to 39.0.1. ([\#15020](https://github.com/matrix-org/synapse/issues/15020)) -- Bump ruff version from 0.0.230 to 0.0.237. ([\#15033](https://github.com/matrix-org/synapse/issues/15033)) -- Bump dtolnay/rust-toolchain from 9cd00a88a73addc8617065438eff914dd08d0955 to 25dc93b901a87e864900a8aec6c12e9aa794c0c3. ([\#15060](https://github.com/matrix-org/synapse/issues/15060)) -- Bump systemd-python from 234 to 235. ([\#15061](https://github.com/matrix-org/synapse/issues/15061)) -- Bump serde_json from 1.0.92 to 1.0.93. ([\#15062](https://github.com/matrix-org/synapse/issues/15062)) -- Bump types-requests from 2.28.11.8 to 2.28.11.12. ([\#15063](https://github.com/matrix-org/synapse/issues/15063)) -- Bump types-pillow from 9.4.0.5 to 9.4.0.10. ([\#15064](https://github.com/matrix-org/synapse/issues/15064)) -- Bump sentry-sdk from 1.13.0 to 1.15.0. ([\#15065](https://github.com/matrix-org/synapse/issues/15065)) -- Bump types-jsonschema from 4.17.0.3 to 4.17.0.5. ([\#15099](https://github.com/matrix-org/synapse/issues/15099)) -- Bump types-bleach from 5.0.3.1 to 6.0.0.0. ([\#15100](https://github.com/matrix-org/synapse/issues/15100)) -- Bump dtolnay/rust-toolchain from 25dc93b901a87e864900a8aec6c12e9aa794c0c3 to e12eda571dc9a5ee5d58eecf4738ec291c66f295. ([\#15101](https://github.com/matrix-org/synapse/issues/15101)) -- Bump dawidd6/action-download-artifact from 2.24.3 to 2.25.0. ([\#15102](https://github.com/matrix-org/synapse/issues/15102)) -- Bump types-pillow from 9.4.0.10 to 9.4.0.13. ([\#15104](https://github.com/matrix-org/synapse/issues/15104)) -- Bump types-setuptools from 67.1.0.0 to 67.3.0.1. ([\#15105](https://github.com/matrix-org/synapse/issues/15105)) - - -
- - -Synapse 1.77.0 (2023-02-14) -=========================== - -No significant changes since 1.77.0rc2. - - -Synapse 1.77.0rc2 (2023-02-10) -============================== - -Bugfixes --------- - -- Fix bug where retried replication requests would return a failure. Introduced in v1.76.0. ([\#15024](https://github.com/matrix-org/synapse/issues/15024)) - - -Internal Changes ----------------- - -- Prepare for future database schema changes. ([\#15036](https://github.com/matrix-org/synapse/issues/15036)) - - -Synapse 1.77.0rc1 (2023-02-07) -============================== - -Features --------- - -- Experimental support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#14823](https://github.com/matrix-org/synapse/issues/14823), [\#14943](https://github.com/matrix-org/synapse/issues/14943), [\#14957](https://github.com/matrix-org/synapse/issues/14957), [\#14958](https://github.com/matrix-org/synapse/issues/14958)) -- Experimental support to suppress notifications from message edits ([MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958)). ([\#14960](https://github.com/matrix-org/synapse/issues/14960), [\#15016](https://github.com/matrix-org/synapse/issues/15016)) -- Add profile information, devices and connections to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.77/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14894](https://github.com/matrix-org/synapse/issues/14894)) -- Improve performance when joining or sending an event in large rooms. ([\#14962](https://github.com/matrix-org/synapse/issues/14962)) -- Improve performance of joining and leaving large rooms with many local users. ([\#14971](https://github.com/matrix-org/synapse/issues/14971)) - - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.53.0 where `next_batch` tokens from `/sync` could not be used with the `/relations` endpoint. ([\#14866](https://github.com/matrix-org/synapse/issues/14866)) -- Fix a bug introduced in Synapse 1.35.0 where the module API's `send_local_online_presence_to` would fail to send presence updates over federation. ([\#14880](https://github.com/matrix-org/synapse/issues/14880)) -- Fix a bug introduced in Synapse 1.70.0 where the background updates to add non-thread unique indexes on receipts could fail when upgrading from 1.67.0 or earlier. ([\#14915](https://github.com/matrix-org/synapse/issues/14915)) -- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14926](https://github.com/matrix-org/synapse/issues/14926)) -- Fix a bug introduced in Synapse 1.68.0 where we were unable to service remote joins in rooms with `@room` notification levels set to `null` in their (malformed) power levels. ([\#14942](https://github.com/matrix-org/synapse/issues/14942)) -- Fix a bug introduced in Synapse 1.64.0 where boolean power levels were erroneously permitted in [v10 rooms](https://spec.matrix.org/v1.5/rooms/v10/). ([\#14944](https://github.com/matrix-org/synapse/issues/14944)) -- Fix a long-standing bug where sending messages on servers with presence enabled would spam "Re-starting finished log context" log lines. ([\#14947](https://github.com/matrix-org/synapse/issues/14947)) -- Fix a bug introduced in Synapse 1.68.0 where logging from the Rust module was not properly logged. ([\#14976](https://github.com/matrix-org/synapse/issues/14976)) -- Fix various long-standing bugs in Synapse's config, event and request handling where booleans were unintentionally accepted where an integer was expected. ([\#14945](https://github.com/matrix-org/synapse/issues/14945)) - - -Internal Changes ----------------- - -- Add missing type hints. ([\#14879](https://github.com/matrix-org/synapse/issues/14879), [\#14886](https://github.com/matrix-org/synapse/issues/14886), [\#14887](https://github.com/matrix-org/synapse/issues/14887), [\#14904](https://github.com/matrix-org/synapse/issues/14904), [\#14927](https://github.com/matrix-org/synapse/issues/14927), [\#14956](https://github.com/matrix-org/synapse/issues/14956), [\#14983](https://github.com/matrix-org/synapse/issues/14983), [\#14984](https://github.com/matrix-org/synapse/issues/14984), [\#14985](https://github.com/matrix-org/synapse/issues/14985), [\#14987](https://github.com/matrix-org/synapse/issues/14987), [\#14988](https://github.com/matrix-org/synapse/issues/14988), [\#14990](https://github.com/matrix-org/synapse/issues/14990), [\#14991](https://github.com/matrix-org/synapse/issues/14991), [\#14992](https://github.com/matrix-org/synapse/issues/14992), [\#15007](https://github.com/matrix-org/synapse/issues/15007)) -- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14922](https://github.com/matrix-org/synapse/issues/14922)) -- Allow running the complement tests suites with the asyncio reactor enabled. ([\#14858](https://github.com/matrix-org/synapse/issues/14858)) -- Improve performance of `/sync` in a few situations. ([\#14908](https://github.com/matrix-org/synapse/issues/14908), [\#14970](https://github.com/matrix-org/synapse/issues/14970)) -- Document how to handle Dependabot pull requests. ([\#14916](https://github.com/matrix-org/synapse/issues/14916)) -- Fix typo in release script. ([\#14920](https://github.com/matrix-org/synapse/issues/14920)) -- Update build system requirements to allow building with poetry-core 1.5.0. ([\#14949](https://github.com/matrix-org/synapse/issues/14949), [\#15019](https://github.com/matrix-org/synapse/issues/15019)) -- Add an [lnav](https://lnav.org) config file for Synapse logs to `/contrib/lnav`. ([\#14953](https://github.com/matrix-org/synapse/issues/14953)) -- Faster joins: Refactor internal handling of servers in room to never store an empty list. ([\#14954](https://github.com/matrix-org/synapse/issues/14954)) -- Faster joins: tag `v2/send_join/` requests to indicate if they served a partial join response. ([\#14950](https://github.com/matrix-org/synapse/issues/14950)) -- Allow running `cargo` without the `extension-module` option. ([\#14965](https://github.com/matrix-org/synapse/issues/14965)) -- Preparatory work for adding a denormalised event stream ordering column in the future. Contributed by Nick @ Beeper (@fizzadar). ([\#14979](https://github.com/matrix-org/synapse/issues/14979), [9cd7610](https://github.com/matrix-org/synapse/commit/9cd7610f86ab5051c9365dd38d1eec405a5f8ca6), [f10caa7](https://github.com/matrix-org/synapse/commit/f10caa73eee0caa91cf373966104d1ededae2aee); see [\#15014](https://github.com/matrix-org/synapse/issues/15014)) -- Add tests for `_flatten_dict`. ([\#14981](https://github.com/matrix-org/synapse/issues/14981), [\#15002](https://github.com/matrix-org/synapse/issues/15002)) - -
Locked dependency updates - -- Bump dtolnay/rust-toolchain from e645b0cf01249a964ec099494d38d2da0f0b349f to 9cd00a88a73addc8617065438eff914dd08d0955. ([\#14968](https://github.com/matrix-org/synapse/issues/14968)) -- Bump docker/build-push-action from 3 to 4. ([\#14952](https://github.com/matrix-org/synapse/issues/14952)) -- Bump ijson from 3.1.4 to 3.2.0.post0. ([\#14935](https://github.com/matrix-org/synapse/issues/14935)) -- Bump types-pyyaml from 6.0.12.2 to 6.0.12.3. ([\#14936](https://github.com/matrix-org/synapse/issues/14936)) -- Bump types-jsonschema from 4.17.0.2 to 4.17.0.3. ([\#14937](https://github.com/matrix-org/synapse/issues/14937)) -- Bump types-pillow from 9.4.0.3 to 9.4.0.5. ([\#14938](https://github.com/matrix-org/synapse/issues/14938)) -- Bump hiredis from 2.0.0 to 2.1.1. ([\#14939](https://github.com/matrix-org/synapse/issues/14939)) -- Bump hiredis from 2.1.1 to 2.2.1. ([\#14993](https://github.com/matrix-org/synapse/issues/14993)) -- Bump types-setuptools from 65.6.0.3 to 67.1.0.0. ([\#14994](https://github.com/matrix-org/synapse/issues/14994)) -- Bump prometheus-client from 0.15.0 to 0.16.0. ([\#14995](https://github.com/matrix-org/synapse/issues/14995)) -- Bump anyhow from 1.0.68 to 1.0.69. ([\#14996](https://github.com/matrix-org/synapse/issues/14996)) -- Bump serde_json from 1.0.91 to 1.0.92. ([\#14997](https://github.com/matrix-org/synapse/issues/14997)) -- Bump isort from 5.11.4 to 5.11.5. ([\#14998](https://github.com/matrix-org/synapse/issues/14998)) -- Bump phonenumbers from 8.13.4 to 8.13.5. ([\#14999](https://github.com/matrix-org/synapse/issues/14999)) -
- -Synapse 1.76.0 (2023-01-31) -=========================== - -The 1.76 release is the first to enable faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706) and [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902)) by default. Admins can opt-out: see [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#faster-joins-are-enabled-by-default) for more details. - -The upgrade from 1.75 to 1.76 changes the account data replication streams in a backwards-incompatible manner. Server operators running a multi-worker deployment should consult [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#changes-to-the-account-data-replication-streams). - -Those who are `poetry install`ing from source using our lockfile should ensure their poetry version is 1.3.2 or higher; [see upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#minimum-version-of-poetry-is-now-132). - - -Notes on faster joins ---------------------- - -The faster joins project sees the most benefit when joining a room with a large number of members (joined or historical). We expect it to be particularly useful for joining large public rooms like the [Matrix HQ](https://matrix.to/#/#matrix:matrix.org) or [Synapse Admins](https://matrix.to/#/#synapse:matrix.org) rooms. - -After a faster join, Synapse considers that room "partially joined". In this state, you should be able to - -- read incoming messages; -- see incoming state changes, e.g. room topic changes; and -- send messages, if the room is unencrypted. - -Synapse has to spend more effort to complete the join in the background. Once this finishes, you will be able to - -- send messages, if the room is in encrypted; -- retrieve room history from before your join, if permitted by the room settings; and -- access the full list of room members. - - -Improved Documentation ----------------------- - -- Describe the ideas and the internal machinery behind faster joins. ([\#14677](https://github.com/matrix-org/synapse/issues/14677)) - - -Synapse 1.76.0rc2 (2023-01-27) -============================== - -Bugfixes --------- - -- Faster joins: Fix a bug introduced in Synapse 1.69 where device list EDUs could fail to be handled after a restart when a faster join sync is in progress. ([\#14914](https://github.com/matrix-org/synapse/issues/14914)) - - -Internal Changes ----------------- - -- Faster joins: Improve performance of looking up partial-state status of rooms. ([\#14917](https://github.com/matrix-org/synapse/issues/14917)) - - -Synapse 1.76.0rc1 (2023-01-25) -============================== - -Features --------- - -- Update the default room version to [v10](https://spec.matrix.org/v1.5/rooms/v10/) ([MSC 3904](https://github.com/matrix-org/matrix-spec-proposals/pull/3904)). Contributed by @FSG-Cat. ([\#14111](https://github.com/matrix-org/synapse/issues/14111)) -- Add a `set_displayname()` method to the module API for setting a user's display name. ([\#14629](https://github.com/matrix-org/synapse/issues/14629)) -- Add a dedicated listener configuration for `health` endpoint. ([\#14747](https://github.com/matrix-org/synapse/issues/14747)) -- Implement support for [MSC3890](https://github.com/matrix-org/matrix-spec-proposals/pull/3890): Remotely silence local notifications. ([\#14775](https://github.com/matrix-org/synapse/issues/14775)) -- Implement experimental support for [MSC3930](https://github.com/matrix-org/matrix-spec-proposals/pull/3930): Push rules for ([MSC3381](https://github.com/matrix-org/matrix-spec-proposals/pull/3381)) Polls. ([\#14787](https://github.com/matrix-org/synapse/issues/14787)) -- Per [MSC3925](https://github.com/matrix-org/matrix-spec-proposals/pull/3925), bundle the whole of the replacement with any edited events, and optionally inhibit server-side replacement. ([\#14811](https://github.com/matrix-org/synapse/issues/14811)) -- Faster joins: always serve a partial join response to servers that request it with the stable query param. ([\#14839](https://github.com/matrix-org/synapse/issues/14839)) -- Faster joins: allow non-lazy-loading ("eager") syncs to complete after a partial join by omitting partial state rooms until they become fully stated. ([\#14870](https://github.com/matrix-org/synapse/issues/14870)) -- Faster joins: request partial joins by default. Admins can opt-out of this for the time being---see the upgrade notes. ([\#14905](https://github.com/matrix-org/synapse/issues/14905)) - - -Bugfixes --------- - -- Add index to improve performance of the `/timestamp_to_event` endpoint used for jumping to a specific date in the timeline of a room. ([\#14799](https://github.com/matrix-org/synapse/issues/14799)) -- Fix a long-standing bug where Synapse would exhaust the stack when processing many federation requests where the remote homeserver has disconencted early. ([\#14812](https://github.com/matrix-org/synapse/issues/14812), [\#14842](https://github.com/matrix-org/synapse/issues/14842)) -- Fix rare races when using workers. ([\#14820](https://github.com/matrix-org/synapse/issues/14820)) -- Fix a bug introduced in Synapse 1.64.0 when using room version 10 with frozen events enabled. ([\#14864](https://github.com/matrix-org/synapse/issues/14864)) -- Fix a long-standing bug where the `populate_room_stats` background job could fail on broken rooms. ([\#14873](https://github.com/matrix-org/synapse/issues/14873)) -- Faster joins: Fix a bug in worker deployments where the room stats and user directory would not get updated when finishing a fast join until another event is sent or received. ([\#14874](https://github.com/matrix-org/synapse/issues/14874)) -- Faster joins: Fix incompatibility with joins into restricted rooms where no local users have the ability to invite. ([\#14882](https://github.com/matrix-org/synapse/issues/14882)) -- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14910](https://github.com/matrix-org/synapse/issues/14910)) - - -Updates to the Docker image ---------------------------- - -- Bump default Python version in the Dockerfile from 3.9 to 3.11. ([\#14875](https://github.com/matrix-org/synapse/issues/14875)) - - -Improved Documentation ----------------------- - -- Include `x_forwarded` entry in the HTTP listener example configs and remove the remaining `worker_main_http_uri` entries. ([\#14667](https://github.com/matrix-org/synapse/issues/14667)) -- Remove duplicate commands from the Code Style documentation page; point to the Contributing Guide instead. ([\#14773](https://github.com/matrix-org/synapse/issues/14773)) -- Add missing documentation for `tag` to `listeners` section. ([\#14803](https://github.com/matrix-org/synapse/issues/14803)) -- Updated documentation in configuration manual for `user_directory.search_all_users`. ([\#14818](https://github.com/matrix-org/synapse/issues/14818)) -- Add `worker_manhole` to configuration manual. ([\#14824](https://github.com/matrix-org/synapse/issues/14824)) -- Fix the example config missing the `id` field in [application service documentation](https://matrix-org.github.io/synapse/latest/application_services.html). ([\#14845](https://github.com/matrix-org/synapse/issues/14845)) -- Minor corrections to the logging configuration documentation. ([\#14868](https://github.com/matrix-org/synapse/issues/14868)) -- Document the export user data command. Contributed by @thezaidbintariq. ([\#14883](https://github.com/matrix-org/synapse/issues/14883)) - - -Deprecations and Removals -------------------------- - -- Poetry 1.3.2 or higher is now required when `poetry install`ing from source. ([\#14860](https://github.com/matrix-org/synapse/issues/14860)) - - -Internal Changes ----------------- - -- Faster remote room joins (worker mode): do not populate external hosts-in-room cache when sending events as this requires blocking for full state. ([\#14749](https://github.com/matrix-org/synapse/issues/14749)) -- Enable Complement tests for Faster Remote Room Joins against worker-mode Synapse. ([\#14752](https://github.com/matrix-org/synapse/issues/14752)) -- Add some clarifying comments and refactor a portion of the `Keyring` class for readability. ([\#14804](https://github.com/matrix-org/synapse/issues/14804)) -- Add local poetry config files (`poetry.toml`) to `.gitignore`. ([\#14807](https://github.com/matrix-org/synapse/issues/14807)) -- Add missing type hints. ([\#14816](https://github.com/matrix-org/synapse/issues/14816), [\#14885](https://github.com/matrix-org/synapse/issues/14885), [\#14889](https://github.com/matrix-org/synapse/issues/14889)) -- Refactor push tests. ([\#14819](https://github.com/matrix-org/synapse/issues/14819)) -- Re-enable some linting that was disabled when we switched to ruff. ([\#14821](https://github.com/matrix-org/synapse/issues/14821)) -- Add `cargo fmt` and `cargo clippy` to the lint script. ([\#14822](https://github.com/matrix-org/synapse/issues/14822)) -- Drop unused table `presence`. ([\#14825](https://github.com/matrix-org/synapse/issues/14825)) -- Merge the two account data and the two device list replication streams. ([\#14826](https://github.com/matrix-org/synapse/issues/14826), [\#14833](https://github.com/matrix-org/synapse/issues/14833)) -- Faster joins: use stable identifiers from [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). ([\#14832](https://github.com/matrix-org/synapse/issues/14832), [\#14841](https://github.com/matrix-org/synapse/issues/14841)) -- Add a parameter to control whether the federation client performs a partial state join. ([\#14843](https://github.com/matrix-org/synapse/issues/14843)) -- Add check to avoid starting duplicate partial state syncs. ([\#14844](https://github.com/matrix-org/synapse/issues/14844)) -- Add an early return when handling no-op presence updates. ([\#14855](https://github.com/matrix-org/synapse/issues/14855)) -- Fix `wait_for_stream_position` to correctly wait for the right instance to advance its token. ([\#14856](https://github.com/matrix-org/synapse/issues/14856), [\#14872](https://github.com/matrix-org/synapse/issues/14872)) -- Always notify replication when a stream advances automatically. ([\#14877](https://github.com/matrix-org/synapse/issues/14877)) -- Reduce max time we wait for stream positions. ([\#14881](https://github.com/matrix-org/synapse/issues/14881)) -- Faster joins: allow the resync process more time to fetch `/state` ids. ([\#14912](https://github.com/matrix-org/synapse/issues/14912)) -- Bump regex from 1.7.0 to 1.7.1. ([\#14848](https://github.com/matrix-org/synapse/issues/14848)) -- Bump peaceiris/actions-gh-pages from 3.9.1 to 3.9.2. ([\#14861](https://github.com/matrix-org/synapse/issues/14861)) -- Bump ruff from 0.0.215 to 0.0.224. ([\#14862](https://github.com/matrix-org/synapse/issues/14862)) -- Bump types-pillow from 9.4.0.0 to 9.4.0.3. ([\#14863](https://github.com/matrix-org/synapse/issues/14863)) -- Bump types-opentracing from 2.4.10 to 2.4.10.1. ([\#14896](https://github.com/matrix-org/synapse/issues/14896)) -- Bump ruff from 0.0.224 to 0.0.230. ([\#14897](https://github.com/matrix-org/synapse/issues/14897)) -- Bump types-requests from 2.28.11.7 to 2.28.11.8. ([\#14899](https://github.com/matrix-org/synapse/issues/14899)) -- Bump types-psycopg2 from 2.9.21.2 to 2.9.21.4. ([\#14900](https://github.com/matrix-org/synapse/issues/14900)) -- Bump types-commonmark from 0.9.2 to 0.9.2.1. ([\#14901](https://github.com/matrix-org/synapse/issues/14901)) - - -Synapse 1.75.0 (2023-01-17) -=========================== - -No significant changes since 1.75.0rc2. - - -Synapse 1.75.0rc2 (2023-01-12) -============================== - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.75.0rc1 where device lists could be miscalculated with some sync filters. ([\#14810](https://github.com/matrix-org/synapse/issues/14810)) -- Fix race where calling `/members` or `/state` with an `at` parameter could fail for newly created rooms, when using multiple workers. ([\#14817](https://github.com/matrix-org/synapse/issues/14817)) - - -Synapse 1.75.0rc1 (2023-01-10) -============================== - -Features --------- - -- Add a `cached` function to `synapse.module_api` that returns a decorator to cache return values of functions. ([\#14663](https://github.com/matrix-org/synapse/issues/14663)) -- Add experimental support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) (removing account data). ([\#14714](https://github.com/matrix-org/synapse/issues/14714)) -- Support [RFC7636](https://datatracker.ietf.org/doc/html/rfc7636) Proof Key for Code Exchange for OAuth single sign-on. ([\#14750](https://github.com/matrix-org/synapse/issues/14750)) -- Support non-OpenID compliant userinfo claims for subject and picture. ([\#14753](https://github.com/matrix-org/synapse/issues/14753)) -- Improve performance of `/sync` when filtering all rooms, message types, or senders. ([\#14786](https://github.com/matrix-org/synapse/issues/14786)) -- Improve performance of the `/hierarchy` endpoint. ([\#14263](https://github.com/matrix-org/synapse/issues/14263)) - - -Bugfixes --------- - -- Fix the *MAU Limits* section of the Grafana dashboard relying on a specific `job` name for the workers of a Synapse deployment. ([\#14644](https://github.com/matrix-org/synapse/issues/14644)) -- Fix a bug introduced in Synapse 1.70.0 which could cause spurious `UNIQUE constraint failed` errors in the `rotate_notifs` background job. ([\#14669](https://github.com/matrix-org/synapse/issues/14669)) -- Ensure stream IDs are always updated after caches get invalidated with workers. Contributed by Nick @ Beeper (@fizzadar). ([\#14723](https://github.com/matrix-org/synapse/issues/14723)) -- Remove the unspecced `device` field from `/pushrules` responses. ([\#14727](https://github.com/matrix-org/synapse/issues/14727)) -- Fix a bug introduced in Synapse 1.73.0 where the `picture_claim` configured under `oidc_providers` was unused (the default value of `"picture"` was used instead). ([\#14751](https://github.com/matrix-org/synapse/issues/14751)) -- Unescape HTML entities in URL preview titles making use of oEmbed responses. ([\#14781](https://github.com/matrix-org/synapse/issues/14781)) -- Disable sending confirmation email when 3pid is disabled. ([\#14725](https://github.com/matrix-org/synapse/issues/14725)) - - -Improved Documentation ----------------------- - -- Declare support for Python 3.11. ([\#14673](https://github.com/matrix-org/synapse/issues/14673)) -- Fix `target_memory_usage` being used in the description for the actual `cache_autotune` sub-option `target_cache_memory_usage`. ([\#14674](https://github.com/matrix-org/synapse/issues/14674)) -- Move `email` to Server section in config file documentation. ([\#14730](https://github.com/matrix-org/synapse/issues/14730)) -- Fix broken links in the Synapse documentation. ([\#14744](https://github.com/matrix-org/synapse/issues/14744)) -- Add missing worker settings to shared configuration documentation. ([\#14748](https://github.com/matrix-org/synapse/issues/14748)) -- Document using Twitter as a OAuth 2.0 authentication provider. ([\#14778](https://github.com/matrix-org/synapse/issues/14778)) -- Fix Synapse 1.74 upgrade notes to correctly explain how to install pyICU when installing Synapse from PyPI. ([\#14797](https://github.com/matrix-org/synapse/issues/14797)) -- Update link to towncrier in contribution guide. ([\#14801](https://github.com/matrix-org/synapse/issues/14801)) -- Use `htmltest` to check links in the Synapse documentation. ([\#14743](https://github.com/matrix-org/synapse/issues/14743)) - - -Internal Changes ----------------- - -- Faster remote room joins: stream the un-partial-stating of events over replication. ([\#14545](https://github.com/matrix-org/synapse/issues/14545), [\#14546](https://github.com/matrix-org/synapse/issues/14546)) -- Use [ruff](https://github.com/charliermarsh/ruff/) instead of flake8. ([\#14633](https://github.com/matrix-org/synapse/issues/14633), [\#14741](https://github.com/matrix-org/synapse/issues/14741)) -- Change `handle_new_client_event` signature so that a 429 does not reach clients on `PartialStateConflictError`, and internally retry when needed instead. ([\#14665](https://github.com/matrix-org/synapse/issues/14665)) -- Remove dependency on jQuery on reCAPTCHA page. ([\#14672](https://github.com/matrix-org/synapse/issues/14672)) -- Faster joins: make `compute_state_after_events` consistent with other state-fetching functions that take a `StateFilter`. ([\#14676](https://github.com/matrix-org/synapse/issues/14676)) -- Add missing type hints. ([\#14680](https://github.com/matrix-org/synapse/issues/14680), [\#14681](https://github.com/matrix-org/synapse/issues/14681), [\#14687](https://github.com/matrix-org/synapse/issues/14687)) -- Improve type annotations for the helper methods on a `CachedFunction`. ([\#14685](https://github.com/matrix-org/synapse/issues/14685)) -- Check that the SQLite database file exists before porting to PostgreSQL. ([\#14692](https://github.com/matrix-org/synapse/issues/14692)) -- Add `.direnv/` directory to .gitignore to prevent local state generated by the [direnv](https://direnv.net/) development tool from being committed. ([\#14707](https://github.com/matrix-org/synapse/issues/14707)) -- Batch up replication requests to request the resyncing of remote users's devices. ([\#14716](https://github.com/matrix-org/synapse/issues/14716)) -- If debug logging is enabled, log the `msgid`s of any to-device messages that are returned over `/sync`. ([\#14724](https://github.com/matrix-org/synapse/issues/14724)) -- Change GHA CI job to follow best practices. ([\#14772](https://github.com/matrix-org/synapse/issues/14772)) -- Switch to our fork of `dh-virtualenv` to work around an upstream Python 3.11 incompatibility. ([\#14774](https://github.com/matrix-org/synapse/issues/14774)) -- Skip testing built wheels for PyPy 3.7 on Linux x86_64 as we lack new required dependencies in the build environment. ([\#14802](https://github.com/matrix-org/synapse/issues/14802)) - -### Dependabot updates - -
- -- Bump JasonEtco/create-an-issue from 2.8.1 to 2.8.2. ([\#14693](https://github.com/matrix-org/synapse/issues/14693)) -- Bump anyhow from 1.0.66 to 1.0.68. ([\#14694](https://github.com/matrix-org/synapse/issues/14694)) -- Bump blake2 from 0.10.5 to 0.10.6. ([\#14695](https://github.com/matrix-org/synapse/issues/14695)) -- Bump serde_json from 1.0.89 to 1.0.91. ([\#14696](https://github.com/matrix-org/synapse/issues/14696)) -- Bump serde from 1.0.150 to 1.0.151. ([\#14697](https://github.com/matrix-org/synapse/issues/14697)) -- Bump lxml from 4.9.1 to 4.9.2. ([\#14698](https://github.com/matrix-org/synapse/issues/14698)) -- Bump types-jsonschema from 4.17.0.1 to 4.17.0.2. ([\#14700](https://github.com/matrix-org/synapse/issues/14700)) -- Bump sentry-sdk from 1.11.1 to 1.12.0. ([\#14701](https://github.com/matrix-org/synapse/issues/14701)) -- Bump types-setuptools from 65.6.0.1 to 65.6.0.2. ([\#14702](https://github.com/matrix-org/synapse/issues/14702)) -- Bump minimum PyYAML to 3.13. ([\#14720](https://github.com/matrix-org/synapse/issues/14720)) -- Bump JasonEtco/create-an-issue from 2.8.2 to 2.9.1. ([\#14731](https://github.com/matrix-org/synapse/issues/14731)) -- Bump towncrier from 22.8.0 to 22.12.0. ([\#14732](https://github.com/matrix-org/synapse/issues/14732)) -- Bump isort from 5.10.1 to 5.11.4. ([\#14733](https://github.com/matrix-org/synapse/issues/14733)) -- Bump attrs from 22.1.0 to 22.2.0. ([\#14734](https://github.com/matrix-org/synapse/issues/14734)) -- Bump black from 22.10.0 to 22.12.0. ([\#14735](https://github.com/matrix-org/synapse/issues/14735)) -- Bump sentry-sdk from 1.12.0 to 1.12.1. ([\#14736](https://github.com/matrix-org/synapse/issues/14736)) -- Bump setuptools from 65.3.0 to 65.5.1. ([\#14738](https://github.com/matrix-org/synapse/issues/14738)) -- Bump serde from 1.0.151 to 1.0.152. ([\#14758](https://github.com/matrix-org/synapse/issues/14758)) -- Bump ruff from 0.0.189 to 0.0.206. ([\#14759](https://github.com/matrix-org/synapse/issues/14759)) -- Bump pydantic from 1.10.2 to 1.10.4. ([\#14760](https://github.com/matrix-org/synapse/issues/14760)) -- Bump gitpython from 3.1.29 to 3.1.30. ([\#14761](https://github.com/matrix-org/synapse/issues/14761)) -- Bump pillow from 9.3.0 to 9.4.0. ([\#14762](https://github.com/matrix-org/synapse/issues/14762)) -- Bump types-requests from 2.28.11.5 to 2.28.11.7. ([\#14763](https://github.com/matrix-org/synapse/issues/14763)) -- Bump dawidd6/action-download-artifact from 2.24.2 to 2.24.3. ([\#14779](https://github.com/matrix-org/synapse/issues/14779)) -- Bump peaceiris/actions-gh-pages from 3.9.0 to 3.9.1. ([\#14791](https://github.com/matrix-org/synapse/issues/14791)) -- Bump types-pillow from 9.3.0.4 to 9.4.0.0. ([\#14792](https://github.com/matrix-org/synapse/issues/14792)) -- Bump pyopenssl from 22.1.0 to 23.0.0. ([\#14793](https://github.com/matrix-org/synapse/issues/14793)) -- Bump types-setuptools from 65.6.0.2 to 65.6.0.3. ([\#14794](https://github.com/matrix-org/synapse/issues/14794)) -- Bump importlib-metadata from 4.2.0 to 6.0.0. ([\#14795](https://github.com/matrix-org/synapse/issues/14795)) -- Bump ruff from 0.0.206 to 0.0.215. ([\#14796](https://github.com/matrix-org/synapse/issues/14796)) -
+* Bump anyhow from 1.0.93 to 1.0.95. ([\#18012](https://github.com/element-hq/synapse/issues/18012), [\#18045](https://github.com/element-hq/synapse/issues/18045)) +* Bump authlib from 1.3.2 to 1.4.0. ([\#18048](https://github.com/element-hq/synapse/issues/18048)) +* Bump dawidd6/action-download-artifact from 6 to 7. ([\#17981](https://github.com/element-hq/synapse/issues/17981)) +* Bump http from 1.1.0 to 1.2.0. ([\#18013](https://github.com/element-hq/synapse/issues/18013)) +- Bump mypy from 1.11.2 to 1.12.1. ([\#17999](https://github.com/element-hq/synapse/issues/17999)) +* Bump mypy-zope from 1.0.8 to 1.0.9. ([\#18047](https://github.com/element-hq/synapse/issues/18047)) +* Bump pillow from 10.4.0 to 11.0.0. ([\#18015](https://github.com/element-hq/synapse/issues/18015)) +* Bump pydantic from 2.9.2 to 2.10.3. ([\#18014](https://github.com/element-hq/synapse/issues/18014)) +* Bump pyicu from 2.13.1 to 2.14. ([\#18060](https://github.com/element-hq/synapse/issues/18060)) +* Bump pyo3 from 0.23.2 to 0.23.3. ([\#18001](https://github.com/element-hq/synapse/issues/18001)) +* Bump python-multipart from 0.0.16 to 0.0.18. ([\#17985](https://github.com/element-hq/synapse/issues/17985)) +* Bump sentry-sdk from 2.17.0 to 2.19.2. ([\#18061](https://github.com/element-hq/synapse/issues/18061)) +* Bump serde from 1.0.215 to 1.0.217. ([\#18031](https://github.com/element-hq/synapse/issues/18031), [\#18059](https://github.com/element-hq/synapse/issues/18059)) +* Bump serde_json from 1.0.133 to 1.0.134. ([\#18044](https://github.com/element-hq/synapse/issues/18044)) +* Bump twine from 5.1.1 to 6.0.1. ([\#18049](https://github.com/element-hq/synapse/issues/18049)) **Changelogs for older versions can be found [here](docs/changelogs/).** diff --git a/Cargo.lock b/Cargo.lock index 5c8f627fd7..678b888e13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -13,9 +28,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "arc-swap" @@ -24,22 +39,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] -name = "autocfg" -version = "1.3.0" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "blake2" @@ -61,27 +97,67 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytes" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -107,12 +183,127 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -125,22 +316,67 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] [[package]] -name = "headers" -version = "0.4.0" +name = "getrandom" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" + +[[package]] +name = "headers" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ "base64", "bytes", @@ -162,9 +398,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hex" @@ -174,15 +410,44 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.1.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", "itoa", ] +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "httpdate" version = "1.0.3" @@ -190,23 +455,275 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] -name = "indoc" -version = "2.0.5" +name = "hyper" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.0", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ae5921528335e91da1b6c695dbf1ec37df5ac13faa3f91e5640be93aa2fbefd" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_locale_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locale_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fdef0c124749d06a743c69e938350816554eb63ac979166590e2b4ee4252765" + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_segmenter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e185fc13b6401c138cf40db12b863b35f5edf31b88192a545857b41aeaf7d3d3" +dependencies = [ + "core_maths", + "displaydoc", + "icu_collections", + "icu_locale", + "icu_locale_core", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5360a2fbe97f617c4f8b944356dedb36d423f7da7f13c070995cf89e59f01220" + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" + +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -218,31 +735,39 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.154" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] -name = "lock_api" -version = "0.4.12" +name = "libm" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "log" -version = "0.4.22" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memoffset" @@ -260,67 +785,109 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] -name = "once_cell" -version = "1.19.0" +name = "miniz_oxide" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "parking_lot" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ - "lock_api", - "parking_lot_core", + "adler2", ] [[package]] -name = "parking_lot_core" -version = "0.9.10" +name = "mio" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ - "cfg-if", "libc", - "redox_syscall", - "smallvec", - "windows-targets", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", ] +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "serde", + "zerovec", +] [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.21.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" +checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" dependencies = [ "anyhow", - "cfg-if", "indoc", "libc", "memoffset", - "parking_lot", + "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -330,9 +897,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.21.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" +checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" dependencies = [ "once_cell", "target-lexicon", @@ -340,9 +907,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.21.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" +checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" dependencies = [ "libc", "pyo3-build-config", @@ -350,9 +917,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.10.0" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af49834b8d2ecd555177e63b273b708dea75150abc6f5341d0a6e1a9623976c" +checksum = "45192e5e4a4d2505587e27806c7b710c231c40c56f3bfc19535d0bb25df52264" dependencies = [ "arc-swap", "log", @@ -361,9 +928,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.21.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" +checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -373,9 +940,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.21.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" +checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" dependencies = [ "heck", "proc-macro2", @@ -386,39 +953,99 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.21.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0664248812c38cc55a4ed07f88e4df516ce82604b93b1ffdc041aa77a6cb3c" +checksum = "597907139a488b22573158793aa7539df36ae863eba300c75f3a0d65fc475e27" dependencies = [ "pyo3", "serde", ] [[package]] -name = "quote" -version = "1.0.36" +name = "quinn" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.5.10", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.5.10", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] [[package]] -name = "rand" -version = "0.8.5" +name = "r-efi" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "libc", "rand_chacha", "rand_core", ] [[package]] name = "rand_chacha" -version = "0.3.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", "rand_core", @@ -426,27 +1053,18 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom", -] - -[[package]] -name = "redox_syscall" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" -dependencies = [ - "bitflags", + "getrandom 0.3.3", ] [[package]] name = "regex" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" dependencies = [ "aho-corasick", "memchr", @@ -456,9 +1074,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -472,31 +1090,188 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] -name = "ryu" -version = "1.0.18" +name = "reqwest" +version = "0.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] [[package]] -name = "scopeguard" -version = "1.2.0" +name = "ring" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustls" +version = "0.23.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] [[package]] name = "serde" -version = "1.0.215" +version = "1.0.224" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "6aaeb1e94f53b16384af593c71e20b095e958dab1d26939c1b70645c5cfbcc0b" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.224" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f39390fa6346e24defbcdd3d9544ba8a19985d0af74df8501fbfe9a64341ab" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.224" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "87ff78ab5e8561c9a675bfc1785cb07ae721f0ee53329a595cefd8c04c2ac4e0" dependencies = [ "proc-macro2", "quote", @@ -505,14 +1280,27 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", ] [[package]] @@ -528,9 +1316,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -538,22 +1326,60 @@ dependencies = [ ] [[package]] -name = "smallvec" -version = "1.13.2" +name = "shlex" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.85" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -568,88 +1394,310 @@ dependencies = [ "base64", "blake2", "bytes", + "futures", "headers", "hex", "http", + "http-body-util", + "icu_segmenter", "lazy_static", "log", "mime", + "once_cell", "pyo3", "pyo3-log", "pythonize", "regex", + "reqwest", "serde", "serde_json", "sha2", + "tokio", "ulid", ] [[package]] -name = "target-lexicon" -version = "0.12.14" +name = "sync_wrapper" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "target-lexicon" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.47.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "pin-project-lite", + "slab", + "socket2 0.6.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "ulid" -version = "1.1.3" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f903f293d11f31c0c29e4148f6dc0d033a7f80cebc0282bea147611667d289" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "getrandom", "rand", "web-time", ] [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unindent" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -657,10 +1705,23 @@ dependencies = [ ] [[package]] -name = "wasm-bindgen-macro" -version = "0.2.92" +name = "wasm-bindgen-futures" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -668,9 +1729,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -681,9 +1742,35 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] name = "web-time" @@ -696,10 +1783,28 @@ dependencies = [ ] [[package]] -name = "windows-targets" -version = "0.52.5" +name = "windows-sys" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -713,48 +1818,167 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/LICENSE b/LICENSE-AGPL-3.0 similarity index 100% rename from LICENSE rename to LICENSE-AGPL-3.0 diff --git a/LICENSE-COMMERCIAL b/LICENSE-COMMERCIAL new file mode 100644 index 0000000000..173e03e0c0 --- /dev/null +++ b/LICENSE-COMMERCIAL @@ -0,0 +1,6 @@ +Licensees holding a valid commercial license with Element may use this +software in accordance with the terms contained in a written agreement +between you and Element. + +To purchase a commercial license please contact our sales team at +licensing@element.io diff --git a/README.rst b/README.rst index 2fe4a7e43f..92854f631c 100644 --- a/README.rst +++ b/README.rst @@ -8,27 +8,28 @@ Synapse is an open source `Matrix `__ homeserver implementation, written and maintained by `Element `_. `Matrix `__ is the open standard for -secure and interoperable real time communications. You can directly run +secure and interoperable real-time communications. You can directly run and manage the source code in this repository, available under an AGPL -license. There is no support provided from Element unless you have a -subscription. +license (or alternatively under a commercial license from Element). +There is no support provided by Element unless you have a +subscription from Element. -Subscription alternative -======================== +Subscription +============ -Alternatively, for those that need an enterprise-ready solution, Element -Server Suite (ESS) is `available as a subscription `_. +For those that need an enterprise-ready solution, Element +Server Suite (ESS) is `available via subscription `_. ESS builds on Synapse to offer a complete Matrix-based backend including the full `Admin Console product `_, giving admins the power to easily manage an organization-wide deployment. It includes advanced identity management, auditing, -moderation and data retention options as well as Long Term Support and -SLAs. ESS can be used to support any Matrix-based frontend client. +moderation and data retention options as well as Long-Term Support and +SLAs. ESS supports any Matrix-compatible client. .. contents:: -🛠️ Installing and configuration -=============================== +🛠️ Installation and configuration +================================== The Synapse documentation describes `how to install Synapse `_. We recommend using `Docker images `_ or `Debian packages from Matrix.org @@ -132,7 +133,7 @@ connect from a client: see An easy way to get started is to login or register via Element at https://app.element.io/#/login or https://app.element.io/#/register respectively. You will need to change the server you are logging into from ``matrix.org`` -and instead specify a Homeserver URL of ``https://:8448`` +and instead specify a homeserver URL of ``https://:8448`` (or just ``https://`` if you are using a reverse proxy). If you prefer to use another client, refer to our `client breakdown `_. @@ -161,16 +162,15 @@ the public internet. Without it, anyone can freely register accounts on your hom This can be exploited by attackers to create spambots targeting the rest of the Matrix federation. -Your new user name will be formed partly from the ``server_name``, and partly -from a localpart you specify when you create the account. Your name will take -the form of:: +Your new Matrix ID will be formed partly from the ``server_name``, and partly +from a localpart you specify when you create the account in the form of:: @localpart:my.domain.name (pronounced "at localpart on my dot domain dot name"). As when logging in, you will need to specify a "Custom server". Specify your -desired ``localpart`` in the 'User name' box. +desired ``localpart`` in the 'Username' box. 🎯 Troubleshooting and support ============================== @@ -208,10 +208,10 @@ Identity servers have the job of mapping email addresses and other 3rd Party IDs (3PIDs) to Matrix user IDs, as well as verifying the ownership of 3PIDs before creating that mapping. -**They are not where accounts or credentials are stored - these live on home -servers. Identity Servers are just for mapping 3rd party IDs to matrix IDs.** +**Identity servers do not store accounts or credentials - these are stored and managed on homeservers. +Identity Servers are just for mapping 3rd Party IDs to Matrix IDs.** -This process is very security-sensitive, as there is obvious risk of spam if it +This process is highly security-sensitive, as there is an obvious risk of spam if it is too easy to sign up for Matrix accounts or harvest 3PID data. In the longer term, we hope to create a decentralised system to manage it (`matrix-doc #712 `_), but in the meantime, @@ -237,9 +237,9 @@ email address. We welcome contributions to Synapse from the community! The best place to get started is our `guide for contributors `_. -This is part of our larger `documentation `_, which includes - +This is part of our broader `documentation `_, which includes information for Synapse developers as well as Synapse administrators. + Developers might be particularly interested in: * `Synapse's database schema `_, @@ -249,6 +249,22 @@ Developers might be particularly interested in: Alongside all that, join our developer community on Matrix: `#synapse-dev:matrix.org `_, featuring real humans! +Copyright and Licensing +======================= + +| Copyright 2014-2017 OpenMarket Ltd +| Copyright 2017 Vector Creations Ltd +| Copyright 2017-2025 New Vector Ltd +| + +This software is dual-licensed by New Vector Ltd (Element). It can be used either: + +(1) for free under the terms of the GNU Affero General Public License (as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version); OR + +(2) under the terms of a paid-for Element Commercial License agreement between you and Element (the terms of which may vary depending on what you and Element have agreed to). + +Unless required by applicable law or agreed to in writing, software distributed under the Licenses is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Licenses for the specific language governing permissions and limitations under the Licenses. + .. |support| image:: https://img.shields.io/badge/matrix-community%20support-success :alt: (get community support in #synapse:matrix.org) diff --git a/build_rust.py b/build_rust.py index 662474dcb4..5c796af461 100644 --- a/build_rust.py +++ b/build_rust.py @@ -1,8 +1,10 @@ # A build script for poetry that adds the rust extension. +import itertools import os from typing import Any, Dict +from packaging.specifiers import SpecifierSet from setuptools_rust import Binding, RustExtension @@ -14,10 +16,27 @@ def build(setup_kwargs: Dict[str, Any]) -> None: target="synapse.synapse_rust", path=cargo_toml_path, binding=Binding.PyO3, + # This flag is a no-op in the latest versions. Instead, we need to + # specify this in the `bdist_wheel` config below. py_limited_api=True, - # We force always building in release mode, as we can't tell the - # difference between using `poetry` in development vs production. + # We always build in release mode, as we can't distinguish + # between using `poetry` in development vs production. debug=False, ) setup_kwargs.setdefault("rust_extensions", []).append(extension) setup_kwargs["zip_safe"] = False + + # We look up the minimum supported Python version with + # `python_requires` (e.g. ">=3.9.0,<4.0.0") and finding the first Python + # version that matches. We then convert that into the `py_limited_api` form, + # e.g. cp39 for Python 3.9. + py_limited_api: str + python_bounds = SpecifierSet(setup_kwargs["python_requires"]) + for minor_version in itertools.count(start=8): + if f"3.{minor_version}.0" in python_bounds: + py_limited_api = f"cp3{minor_version}" + break + + setup_kwargs.setdefault("options", {}).setdefault("bdist_wheel", {})[ + "py_limited_api" + ] = py_limited_api diff --git a/changelog.d/17872.doc b/changelog.d/17872.doc deleted file mode 100644 index 7f8b2d3495..0000000000 --- a/changelog.d/17872.doc +++ /dev/null @@ -1 +0,0 @@ -Add OIDC example configuration for Forgejo (fork of Gitea). diff --git a/changelog.d/17936.misc b/changelog.d/17936.misc deleted file mode 100644 index 91d976fbd9..0000000000 --- a/changelog.d/17936.misc +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect comment in new schema delta. diff --git a/changelog.d/17944.misc b/changelog.d/17944.misc deleted file mode 100644 index a8a645103f..0000000000 --- a/changelog.d/17944.misc +++ /dev/null @@ -1 +0,0 @@ -Raise setuptools_rust version cap to 1.10.2. \ No newline at end of file diff --git a/changelog.d/17945.misc b/changelog.d/17945.misc deleted file mode 100644 index eeebb92169..0000000000 --- a/changelog.d/17945.misc +++ /dev/null @@ -1 +0,0 @@ -Enable encrypted appservice related experimental features in the complement docker image. diff --git a/changelog.d/18583.removal b/changelog.d/18583.removal new file mode 100644 index 0000000000..d7baa85147 --- /dev/null +++ b/changelog.d/18583.removal @@ -0,0 +1 @@ +Remove obsolete and experimental `/sync/e2ee` endpoint. \ No newline at end of file diff --git a/changelog.d/18641.bugfix b/changelog.d/18641.bugfix new file mode 100644 index 0000000000..8f2a2e3d8b --- /dev/null +++ b/changelog.d/18641.bugfix @@ -0,0 +1 @@ +Ensure all PDUs sent via `/send` pass canonical JSON checks. diff --git a/changelog.d/18695.feature b/changelog.d/18695.feature new file mode 100644 index 0000000000..1481a27f23 --- /dev/null +++ b/changelog.d/18695.feature @@ -0,0 +1 @@ +Add experimental support for [MSC4308: Thread Subscriptions extension to Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4308) when [MSC4306: Thread Subscriptions](https://github.com/matrix-org/matrix-spec-proposals/pull/4306) and [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) are enabled. \ No newline at end of file diff --git a/changelog.d/18791.misc b/changelog.d/18791.misc new file mode 100644 index 0000000000..6ecd498286 --- /dev/null +++ b/changelog.d/18791.misc @@ -0,0 +1 @@ +Fix `LaterGauge` metrics to collect from all servers. diff --git a/changelog.d/18819.misc b/changelog.d/18819.misc new file mode 100644 index 0000000000..c76e050e79 --- /dev/null +++ b/changelog.d/18819.misc @@ -0,0 +1 @@ +Configure Synapse to run MSC4306: Thread Subscriptions Complement tests. \ No newline at end of file diff --git a/changelog.d/18823.bugfix b/changelog.d/18823.bugfix new file mode 100644 index 0000000000..473c865aa4 --- /dev/null +++ b/changelog.d/18823.bugfix @@ -0,0 +1 @@ +Fix bug where we did not send invite revocations over federation. diff --git a/changelog.d/18846.feature b/changelog.d/18846.feature new file mode 100644 index 0000000000..4a873d4446 --- /dev/null +++ b/changelog.d/18846.feature @@ -0,0 +1 @@ +Update push rules for experimental [MSC4306: Thread Subscriptions](https://github.com/matrix-org/matrix-doc/issues/4306) to follow newer draft. \ No newline at end of file diff --git a/changelog.d/18848.feature b/changelog.d/18848.feature new file mode 100644 index 0000000000..302a6e7b66 --- /dev/null +++ b/changelog.d/18848.feature @@ -0,0 +1 @@ +Add `get_media_upload_limits_for_user` and `on_media_upload_limit_exceeded` module API callbacks for media repository. diff --git a/changelog.d/18856.doc b/changelog.d/18856.doc new file mode 100644 index 0000000000..0e5e55377f --- /dev/null +++ b/changelog.d/18856.doc @@ -0,0 +1 @@ +Clarify Python dependency constraints in our deprecation policy. diff --git a/changelog.d/18870.misc b/changelog.d/18870.misc new file mode 100644 index 0000000000..e54ba4f37a --- /dev/null +++ b/changelog.d/18870.misc @@ -0,0 +1 @@ +Remove `sentinel` logcontext usage where we log in `setup`, `start` and exit. diff --git a/changelog.d/18874.misc b/changelog.d/18874.misc new file mode 100644 index 0000000000..729befb5e8 --- /dev/null +++ b/changelog.d/18874.misc @@ -0,0 +1 @@ +Use the `Enum`'s value for the dictionary key when responding to an admin request for experimental features. diff --git a/changelog.d/18875.bugfix b/changelog.d/18875.bugfix new file mode 100644 index 0000000000..3bda7a1d18 --- /dev/null +++ b/changelog.d/18875.bugfix @@ -0,0 +1 @@ +Fix prefixed support for MSC4133. diff --git a/changelog.d/18878.docker b/changelog.d/18878.docker new file mode 100644 index 0000000000..cf74f67cc8 --- /dev/null +++ b/changelog.d/18878.docker @@ -0,0 +1 @@ +Suppress "Applying schema" log noise bulk when `SYNAPSE_LOG_TESTING` is set. diff --git a/changelog.d/18886.misc b/changelog.d/18886.misc new file mode 100644 index 0000000000..d0d32e59ab --- /dev/null +++ b/changelog.d/18886.misc @@ -0,0 +1 @@ +Start background tasks after we fork the process (daemonize). diff --git a/changelog.d/18899.feature b/changelog.d/18899.feature new file mode 100644 index 0000000000..ee7141efc5 --- /dev/null +++ b/changelog.d/18899.feature @@ -0,0 +1 @@ +Add an in-memory cache to `_get_e2e_cross_signing_signatures_for_devices` to reduce DB load. \ No newline at end of file diff --git a/changelog.d/18900.misc b/changelog.d/18900.misc new file mode 100644 index 0000000000..d7d8b47eb0 --- /dev/null +++ b/changelog.d/18900.misc @@ -0,0 +1 @@ +Better explain how we manage the logcontext in `run_in_background(...)` and `run_as_background_process(...)`. diff --git a/changelog.d/18906.misc b/changelog.d/18906.misc new file mode 100644 index 0000000000..d7d8b47eb0 --- /dev/null +++ b/changelog.d/18906.misc @@ -0,0 +1 @@ +Better explain how we manage the logcontext in `run_in_background(...)` and `run_as_background_process(...)`. diff --git a/changelog.d/18909.bugfix b/changelog.d/18909.bugfix new file mode 100644 index 0000000000..10d17631f0 --- /dev/null +++ b/changelog.d/18909.bugfix @@ -0,0 +1 @@ +Fix open redirect in legacy SSO flow with the `idp` query parameter. diff --git a/changelog.d/18910.misc b/changelog.d/18910.misc new file mode 100644 index 0000000000..d5bd3ef314 --- /dev/null +++ b/changelog.d/18910.misc @@ -0,0 +1 @@ +Replace usages of the deprecated `pkg_resources` interface in preparation of setuptools dropping it soon. \ No newline at end of file diff --git a/changelog.d/18931.doc b/changelog.d/18931.doc new file mode 100644 index 0000000000..8a2dcb8654 --- /dev/null +++ b/changelog.d/18931.doc @@ -0,0 +1,2 @@ +Clarify necessary `jwt_config` parameter in OIDC documentation for authentik. +Contributed by @maxkratz. diff --git a/contrib/cmdclient/console.py b/contrib/cmdclient/console.py index ca2e72b5e8..9b5d33d2b1 100755 --- a/contrib/cmdclient/console.py +++ b/contrib/cmdclient/console.py @@ -245,7 +245,7 @@ class SynapseCmd(cmd.Cmd): if "flows" not in json_res: print("Failed to find any login flows.") - defer.returnValue(False) + return False flow = json_res["flows"][0] # assume first is the one we want. if "type" not in flow or "m.login.password" != flow["type"] or "stages" in flow: @@ -254,8 +254,8 @@ class SynapseCmd(cmd.Cmd): "Unable to login via the command line client. Please visit " "%s to login." % fallback_url ) - defer.returnValue(False) - defer.returnValue(True) + return False + return True def do_emailrequest(self, line): """Requests the association of a third party identifier diff --git a/contrib/cmdclient/http.py b/contrib/cmdclient/http.py index e6a10b5f32..54363e4259 100644 --- a/contrib/cmdclient/http.py +++ b/contrib/cmdclient/http.py @@ -78,7 +78,7 @@ class TwistedHttpClient(HttpClient): url, data, headers_dict={"Content-Type": ["application/json"]} ) body = yield readBody(response) - defer.returnValue((response.code, body)) + return response.code, body @defer.inlineCallbacks def get_json(self, url, args=None): @@ -88,7 +88,7 @@ class TwistedHttpClient(HttpClient): url = "%s?%s" % (url, qs) response = yield self._create_get_request(url) body = yield readBody(response) - defer.returnValue(json.loads(body)) + return json.loads(body) def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None): """Wrapper of _create_request to issue a PUT request""" @@ -134,7 +134,7 @@ class TwistedHttpClient(HttpClient): response = yield self._create_request(method, url) body = yield readBody(response) - defer.returnValue(json.loads(body)) + return json.loads(body) @defer.inlineCallbacks def _create_request( @@ -173,7 +173,7 @@ class TwistedHttpClient(HttpClient): if self.verbose: print("Status %s %s" % (response.code, response.phrase)) print(pformat(list(response.headers.getAllRawHeaders()))) - defer.returnValue(response) + return response def sleep(self, seconds): d = defer.Deferred() diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 89c1518bd0..fdfa96795a 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -30,3 +30,6 @@ docker-compose up -d ### More information For more information on required environment variables and mounts, see the main docker documentation at [/docker/README.md](../../docker/README.md) + +**For a more comprehensive Docker Compose example showcasing a full Matrix 2.0 stack, please see +https://github.com/element-hq/element-docker-demo** \ No newline at end of file diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 36d5fd5309..9dffc852fd 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -51,7 +51,7 @@ services: - traefik.http.routers.https-synapse.tls.certResolver=le-ssl db: - image: docker.io/postgres:12-alpine + image: docker.io/postgres:15-alpine # Change that password, of course! environment: - POSTGRES_USER=synapse diff --git a/contrib/docker_compose_workers/README.md b/contrib/docker_compose_workers/README.md index 81518f6ba1..16c8c26795 100644 --- a/contrib/docker_compose_workers/README.md +++ b/contrib/docker_compose_workers/README.md @@ -8,6 +8,9 @@ All examples and snippets assume that your Synapse service is called `synapse` i An example Docker Compose file can be found [here](docker-compose.yaml). +**For a more comprehensive Docker Compose example, showcasing a full Matrix 2.0 stack (originally based on this +docker-compose.yaml), please see https://github.com/element-hq/element-docker-demo** + ## Worker Service Examples in Docker Compose In order to start the Synapse container as a worker, you must specify an `entrypoint` that loads both the `homeserver.yaml` and the configuration for the worker (`synapse-generic-worker-1.yaml` in the example below). You must also include the worker type in the environment variable `SYNAPSE_WORKER` or alternatively pass `-m synapse.app.generic_worker` as part of the `entrypoint` after `"/start.py", "run"`). diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index 30d6d87500..e23afcf2d3 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -220,29 +220,24 @@ "yBucketBound": "auto" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_PROMETHEUS}", + "type": "prometheus" }, - "description": "", + "aliasColors": {}, + "dashLength": 10, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 }, - "hiddenSeries": false, "id": 152, "legend": { "avg": false, @@ -255,71 +250,81 @@ "values": false }, "lines": true, - "linewidth": 0, - "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "paceLength": 10, - "percentage": false, - "pluginVersion": "9.2.2", + "pluginVersion": "10.4.3", "pointradius": 5, - "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Avg", "fill": 0, - "linewidth": 3 + "linewidth": 3, + "$$hashKey": "object:48" }, { "alias": "99%", "color": "#C4162A", - "fillBelowTo": "90%" + "fillBelowTo": "90%", + "$$hashKey": "object:49" }, { "alias": "90%", "color": "#FF7383", - "fillBelowTo": "75%" + "fillBelowTo": "75%", + "$$hashKey": "object:50" }, { "alias": "75%", "color": "#FFEE52", - "fillBelowTo": "50%" + "fillBelowTo": "50%", + "$$hashKey": "object:51" }, { "alias": "50%", "color": "#73BF69", - "fillBelowTo": "25%" + "fillBelowTo": "25%", + "$$hashKey": "object:52" }, { "alias": "25%", "color": "#1F60C4", - "fillBelowTo": "5%" + "fillBelowTo": "5%", + "$$hashKey": "object:53" }, { "alias": "5%", - "lines": false + "lines": false, + "$$hashKey": "object:54" }, { "alias": "Average", "color": "rgb(255, 255, 255)", "lines": true, - "linewidth": 3 + "linewidth": 3, + "$$hashKey": "object:55" }, { - "alias": "Events", + "alias": "Local events being persisted", + "color": "#96d98D", + "points": true, + "yaxis": 2, + "zindex": -3, + "$$hashKey": "object:56" + }, + { + "$$hashKey": "object:329", "color": "#B877D9", - "hideTooltip": true, + "alias": "All events being persisted", "points": true, "yaxis": 2, "zindex": -3 } ], "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { @@ -384,7 +389,20 @@ }, "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))", "legendFormat": "Average", - "refId": "H" + "refId": "H", + "editorMode": "code", + "range": true + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))", + "hide": false, + "instant": false, + "legendFormat": "Local events being persisted", + "refId": "E", + "editorMode": "code" }, { "datasource": { @@ -393,8 +411,9 @@ "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))", "hide": false, "instant": false, - "legendFormat": "Events", - "refId": "E" + "legendFormat": "All events being persisted", + "refId": "I", + "editorMode": "code" } ], "thresholds": [ @@ -428,7 +447,9 @@ "xaxis": { "mode": "time", "show": true, - "values": [] + "values": [], + "name": null, + "buckets": null }, "yaxes": [ { @@ -450,7 +471,20 @@ ], "yaxis": { "align": false - } + }, + "bars": false, + "dashes": false, + "description": "", + "fill": 0, + "fillGradient": 0, + "hiddenSeries": false, + "linewidth": 0, + "percentage": false, + "points": false, + "stack": false, + "steppedLine": false, + "timeFrom": null, + "timeShift": null }, { "aliasColors": {}, @@ -4362,7 +4396,7 @@ "exemplar": false, "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_received_pdu_time[10m]))) / 60", "instant": false, - "legendFormat": "{{server_name}} ", + "legendFormat": "{{origin_server_name}} ", "range": true, "refId": "A" } @@ -4484,7 +4518,7 @@ "exemplar": false, "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_sent_pdu_time[10m]))) / 60", "instant": false, - "legendFormat": "{{server_name}}", + "legendFormat": "{{destination_server_name}}", "range": true, "refId": "A" } diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index 1d74fee822..9d5f3c7f4f 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -45,6 +45,10 @@ def make_graph(pdus: List[dict], filename_prefix: str) -> None: colors = {"red", "green", "blue", "yellow", "purple"} for pdu in pdus: + # TODO: The "origin" field has since been removed from events generated + # by Synapse. We should consider removing it here as well but since this + # is part of `contrib/`, it is left for the community to revise and ensure things + # still work correctly. origins.add(pdu.get("origin")) color_map = {color: color for color in colors if color in origins} diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 5fc817b607..9e7fb95c8e 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -35,7 +35,7 @@ TEMP_VENV="$(mktemp -d)" python3 -m venv "$TEMP_VENV" source "$TEMP_VENV/bin/activate" pip install -U pip -pip install poetry==1.3.2 +pip install poetry==2.1.1 poetry-plugin-export==1.9.0 poetry export \ --extras all \ --extras test \ diff --git a/debian/changelog b/debian/changelog index d7cec3fa8a..035d06ad2b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,304 @@ +matrix-synapse-py3 (1.138.0) stable; urgency=medium + + * New Synapse release 1.138.0. + + -- Synapse Packaging team Tue, 09 Sep 2025 11:21:25 +0100 + +matrix-synapse-py3 (1.138.0~rc1) stable; urgency=medium + + * New synapse release 1.138.0rc1. + + -- Synapse Packaging team Tue, 02 Sep 2025 12:16:14 +0000 + +matrix-synapse-py3 (1.137.0) stable; urgency=medium + + * New Synapse release 1.137.0. + + -- Synapse Packaging team Tue, 26 Aug 2025 10:23:41 +0100 + +matrix-synapse-py3 (1.137.0~rc1) stable; urgency=medium + + * New Synapse release 1.137.0rc1. + + -- Synapse Packaging team Tue, 19 Aug 2025 10:55:22 +0100 + +matrix-synapse-py3 (1.136.0) stable; urgency=medium + + * New Synapse release 1.136.0. + + -- Synapse Packaging team Tue, 12 Aug 2025 13:18:03 +0100 + +matrix-synapse-py3 (1.136.0~rc2) stable; urgency=medium + + * New Synapse release 1.136.0rc2. + + -- Synapse Packaging team Mon, 11 Aug 2025 12:18:52 -0600 + +matrix-synapse-py3 (1.136.0~rc1) stable; urgency=medium + + * New Synapse release 1.136.0rc1. + + -- Synapse Packaging team Tue, 05 Aug 2025 08:13:30 -0600 + +matrix-synapse-py3 (1.135.2) stable; urgency=medium + + * New Synapse release 1.135.2. + + -- Synapse Packaging team Mon, 11 Aug 2025 11:52:01 -0600 + +matrix-synapse-py3 (1.135.1) stable; urgency=medium + + * New Synapse release 1.135.1. + + -- Synapse Packaging team Mon, 11 Aug 2025 11:13:15 -0600 + +matrix-synapse-py3 (1.135.0) stable; urgency=medium + + * New Synapse release 1.135.0. + + -- Synapse Packaging team Fri, 01 Aug 2025 13:12:28 +0100 + +matrix-synapse-py3 (1.135.0~rc2) stable; urgency=medium + + * New Synapse release 1.135.0rc2. + + -- Synapse Packaging team Wed, 30 Jul 2025 12:19:14 +0100 + +matrix-synapse-py3 (1.135.0~rc1) stable; urgency=medium + + * New Synapse release 1.135.0rc1. + + -- Synapse Packaging team Tue, 22 Jul 2025 12:08:37 +0100 + +matrix-synapse-py3 (1.134.0) stable; urgency=medium + + * New Synapse release 1.134.0. + + -- Synapse Packaging team Tue, 15 Jul 2025 14:22:50 +0100 + +matrix-synapse-py3 (1.134.0~rc1) stable; urgency=medium + + * New Synapse release 1.134.0rc1. + + -- Synapse Packaging team Wed, 09 Jul 2025 11:27:13 +0100 + +matrix-synapse-py3 (1.133.0) stable; urgency=medium + + * New synapse release 1.133.0. + + -- Synapse Packaging team Tue, 01 Jul 2025 13:13:24 +0000 + +matrix-synapse-py3 (1.133.0~rc1) stable; urgency=medium + + * New Synapse release 1.133.0rc1. + + -- Synapse Packaging team Tue, 24 Jun 2025 11:57:47 +0100 + +matrix-synapse-py3 (1.132.0) stable; urgency=medium + + * New Synapse release 1.132.0. + + -- Synapse Packaging team Tue, 17 Jun 2025 13:16:20 +0100 + +matrix-synapse-py3 (1.132.0~rc1) stable; urgency=medium + + * New Synapse release 1.132.0rc1. + + -- Synapse Packaging team Tue, 10 Jun 2025 11:15:18 +0100 + +matrix-synapse-py3 (1.131.0) stable; urgency=medium + + * New Synapse release 1.131.0. + + -- Synapse Packaging team Tue, 03 Jun 2025 14:36:55 +0100 + +matrix-synapse-py3 (1.131.0~rc1) stable; urgency=medium + + * New synapse release 1.131.0rc1. + + -- Synapse Packaging team Wed, 28 May 2025 10:25:44 +0000 + +matrix-synapse-py3 (1.130.0) stable; urgency=medium + + * New Synapse release 1.130.0. + + -- Synapse Packaging team Tue, 20 May 2025 08:34:13 -0600 + +matrix-synapse-py3 (1.130.0~rc1) stable; urgency=medium + + * New Synapse release 1.130.0rc1. + + -- Synapse Packaging team Tue, 13 May 2025 10:44:04 +0100 + +matrix-synapse-py3 (1.129.0) stable; urgency=medium + + * New Synapse release 1.129.0. + + -- Synapse Packaging team Tue, 06 May 2025 12:22:11 +0100 + +matrix-synapse-py3 (1.129.0~rc2) stable; urgency=medium + + * New synapse release 1.129.0rc2. + + -- Synapse Packaging team Wed, 30 Apr 2025 13:13:16 +0000 + +matrix-synapse-py3 (1.129.0~rc1) stable; urgency=medium + + * New Synapse release 1.129.0rc1. + + -- Synapse Packaging team Tue, 15 Apr 2025 10:47:43 -0600 + +matrix-synapse-py3 (1.128.0) stable; urgency=medium + + * New Synapse release 1.128.0. + + -- Synapse Packaging team Tue, 08 Apr 2025 14:09:54 +0100 + +matrix-synapse-py3 (1.128.0~rc1) stable; urgency=medium + + * Update Poetry to 2.1.1. + * New synapse release 1.128.0rc1. + + -- Synapse Packaging team Tue, 01 Apr 2025 14:35:33 +0000 + +matrix-synapse-py3 (1.127.1) stable; urgency=medium + + * New Synapse release 1.127.1. + + -- Synapse Packaging team Wed, 26 Mar 2025 21:07:31 +0000 + +matrix-synapse-py3 (1.127.0) stable; urgency=medium + + * New Synapse release 1.127.0. + + -- Synapse Packaging team Tue, 25 Mar 2025 12:04:15 +0000 + +matrix-synapse-py3 (1.127.0~rc1) stable; urgency=medium + + * New Synapse release 1.127.0rc1. + + -- Synapse Packaging team Tue, 18 Mar 2025 13:30:05 +0000 + +matrix-synapse-py3 (1.126.0) stable; urgency=medium + + * New Synapse release 1.126.0. + + -- Synapse Packaging team Tue, 11 Mar 2025 13:11:29 +0000 + +matrix-synapse-py3 (1.126.0~rc3) stable; urgency=medium + + * New Synapse release 1.126.0rc3. + + -- Synapse Packaging team Fri, 07 Mar 2025 15:45:05 +0000 + +matrix-synapse-py3 (1.126.0~rc2) stable; urgency=medium + + * New Synapse release 1.126.0rc2. + + -- Synapse Packaging team Wed, 05 Mar 2025 14:29:12 +0000 + +matrix-synapse-py3 (1.126.0~rc1) stable; urgency=medium + + * New Synapse release 1.126.0rc1. + + -- Synapse Packaging team Tue, 04 Mar 2025 13:11:51 +0000 + +matrix-synapse-py3 (1.125.0) stable; urgency=medium + + * New Synapse release 1.125.0. + + -- Synapse Packaging team Tue, 25 Feb 2025 08:10:07 -0700 + +matrix-synapse-py3 (1.125.0~rc1) stable; urgency=medium + + * New synapse release 1.125.0rc1. + + -- Synapse Packaging team Tue, 18 Feb 2025 13:32:49 +0000 + +matrix-synapse-py3 (1.124.0) stable; urgency=medium + + * New Synapse release 1.124.0. + + -- Synapse Packaging team Tue, 11 Feb 2025 11:55:22 +0100 + +matrix-synapse-py3 (1.124.0~rc3) stable; urgency=medium + + * New Synapse release 1.124.0rc3. + + -- Synapse Packaging team Fri, 07 Feb 2025 13:42:55 +0000 + +matrix-synapse-py3 (1.124.0~rc2) stable; urgency=medium + + * New Synapse release 1.124.0rc2. + + -- Synapse Packaging team Wed, 05 Feb 2025 16:35:53 +0000 + +matrix-synapse-py3 (1.124.0~rc1) stable; urgency=medium + + * New Synapse release 1.124.0rc1. + + -- Synapse Packaging team Tue, 04 Feb 2025 11:53:05 +0000 + +matrix-synapse-py3 (1.123.0) stable; urgency=medium + + * New Synapse release 1.123.0. + + -- Synapse Packaging team Tue, 28 Jan 2025 08:37:34 -0700 + +matrix-synapse-py3 (1.123.0~rc1) stable; urgency=medium + + * New Synapse release 1.123.0rc1. + + -- Synapse Packaging team Tue, 21 Jan 2025 14:39:57 +0100 + +matrix-synapse-py3 (1.122.0) stable; urgency=medium + + * New Synapse release 1.122.0. + + -- Synapse Packaging team Tue, 14 Jan 2025 14:14:14 +0000 + +matrix-synapse-py3 (1.122.0~rc1) stable; urgency=medium + + * New Synapse release 1.122.0rc1. + + -- Synapse Packaging team Tue, 07 Jan 2025 14:06:19 +0000 + +matrix-synapse-py3 (1.121.1) stable; urgency=medium + + * New Synapse release 1.121.1. + + -- Synapse Packaging team Wed, 11 Dec 2024 18:24:48 +0000 + +matrix-synapse-py3 (1.121.0) stable; urgency=medium + + * New Synapse release 1.121.0. + + -- Synapse Packaging team Wed, 11 Dec 2024 13:12:30 +0100 + +matrix-synapse-py3 (1.121.0~rc1) stable; urgency=medium + + * New Synapse release 1.121.0rc1. + + -- Synapse Packaging team Wed, 04 Dec 2024 14:47:23 +0000 + +matrix-synapse-py3 (1.120.2) stable; urgency=medium + + * New synapse release 1.120.2. + + -- Synapse Packaging team Tue, 03 Dec 2024 15:43:37 +0000 + +matrix-synapse-py3 (1.120.1) stable; urgency=medium + + * New synapse release 1.120.1. + + -- Synapse Packaging team Tue, 03 Dec 2024 09:07:57 +0000 + +matrix-synapse-py3 (1.120.0) stable; urgency=medium + + * New synapse release 1.120.0. + + -- Synapse Packaging team Tue, 26 Nov 2024 13:10:23 +0000 + matrix-synapse-py3 (1.120.0~rc1) stable; urgency=medium * New Synapse release 1.120.0rc1. diff --git a/demo/start.sh b/demo/start.sh index 06ec6f985f..e010302bf4 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -138,6 +138,13 @@ for port in 8080 8081 8082; do per_user: per_second: 1000 burst_count: 1000 + rc_presence: + per_user: + per_second: 1000 + burst_count: 1000 + rc_delayed_event_mgmt: + per_second: 1000 + burst_count: 1000 RC ) echo "${ratelimiting}" >> "$port.config" diff --git a/docker/Dockerfile b/docker/Dockerfile index a4931011a7..15c458fa28 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,45 +20,16 @@ # `poetry export | pip install -r /dev/stdin`, but beware: we have experienced bugs in # in `poetry export` in the past. +ARG DEBIAN_VERSION=bookworm ARG PYTHON_VERSION=3.12 +ARG POETRY_VERSION=2.1.1 ### ### Stage 0: generate requirements.txt ### -# We hardcode the use of Debian bookworm here because this could change upstream -# and other Dockerfiles used for testing are expecting bookworm. -FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm AS requirements - -# RUN --mount is specific to buildkit and is documented at -# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. -# Here we use it to set up a cache for apt (and below for pip), to improve -# rebuild speeds on slow connections. -RUN \ - --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -qq && apt-get install -yqq \ - build-essential curl git libffi-dev libssl-dev pkg-config \ - && rm -rf /var/lib/apt/lists/* - -# Install rust and ensure its in the PATH. -# (Rust may be needed to compile `cryptography`---which is one of poetry's -# dependencies---on platforms that don't have a `cryptography` wheel. -ENV RUSTUP_HOME=/rust -ENV CARGO_HOME=/cargo -ENV PATH=/cargo/bin:/rust/bin:$PATH -RUN mkdir /rust /cargo - -RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal - -# arm64 builds consume a lot of memory if `CARGO_NET_GIT_FETCH_WITH_CLI` is not -# set to true, so we expose it as a build-arg. -ARG CARGO_NET_GIT_FETCH_WITH_CLI=false -ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI - -# We install poetry in its own build stage to avoid its dependencies conflicting with -# synapse's dependencies. -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --user "poetry==1.3.2" +### This stage is platform-agnostic, so we can use the build platform in case of cross-compilation. +### +FROM --platform=$BUILDPLATFORM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS requirements WORKDIR /synapse @@ -75,41 +46,30 @@ ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION # Instead, we'll just install what a regular `pip install` would from PyPI. ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE +# This silences a warning as uv isn't able to do hardlinks between its cache +# (mounted as --mount=type=cache) and the target directory. +ENV UV_LINK_MODE=copy + # Export the dependencies, but only if we're actually going to use the Poetry lockfile. # Otherwise, just create an empty requirements file so that the Dockerfile can # proceed. -RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ - /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \ +ARG POETRY_VERSION +RUN --mount=type=cache,target=/root/.cache/uv \ + if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ + uvx --with poetry-plugin-export==1.9.0 \ + poetry@${POETRY_VERSION} export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \ else \ - touch /synapse/requirements.txt; \ + touch /synapse/requirements.txt; \ fi ### ### Stage 1: builder ### -FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm AS builder - -# install the OS build deps -RUN \ - --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -qq && apt-get install -yqq \ - build-essential \ - libffi-dev \ - libjpeg-dev \ - libpq-dev \ - libssl-dev \ - libwebp-dev \ - libxml++2.6-dev \ - libxslt1-dev \ - openssl \ - zlib1g-dev \ - git \ - curl \ - libicu-dev \ - pkg-config \ - && rm -rf /var/lib/apt/lists/* +FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS builder +# This silences a warning as uv isn't able to do hardlinks between its cache +# (mounted as --mount=type=cache) and the target directory. +ENV UV_LINK_MODE=copy # Install rust and ensure its in the PATH ENV RUSTUP_HOME=/rust @@ -119,7 +79,6 @@ RUN mkdir /rust /cargo RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal - # arm64 builds consume a lot of memory if `CARGO_NET_GIT_FETCH_WITH_CLI` is not # set to true, so we expose it as a build-arg. ARG CARGO_NET_GIT_FETCH_WITH_CLI=false @@ -131,8 +90,8 @@ ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI # # This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml. COPY --from=requirements /synapse/requirements.txt /synapse/ -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --prefix="/install" --no-deps --no-warn-script-location -r /synapse/requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --prefix="/install" --no-deps -r /synapse/requirements.txt # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ @@ -146,41 +105,85 @@ ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE # Install the synapse package itself. # If we have populated requirements.txt, we don't install any dependencies # as we should already have those from the previous `pip install` step. -RUN --mount=type=cache,target=/synapse/target,sharing=locked \ +RUN \ + --mount=type=cache,target=/root/.cache/uv \ + --mount=type=cache,target=/synapse/target,sharing=locked \ --mount=type=cache,target=${CARGO_HOME}/registry,sharing=locked \ if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ - pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all]; \ + uv pip install --prefix="/install" --no-deps /synapse[all]; \ else \ - pip install --prefix="/install" --no-warn-script-location /synapse[all]; \ + uv pip install --prefix="/install" /synapse[all]; \ fi ### -### Stage 2: runtime +### Stage 2: runtime dependencies download for ARM64 and AMD64 +### +FROM --platform=$BUILDPLATFORM docker.io/library/debian:${DEBIAN_VERSION} AS runtime-deps + +# Tell apt to keep downloaded package files, as we're using cache mounts. +RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache + +# Add both target architectures +RUN dpkg --add-architecture arm64 +RUN dpkg --add-architecture amd64 + +# Fetch the runtime dependencies debs for both architectures +# We do that by building a recursive list of packages we need to download with `apt-cache depends` +# and then downloading them with `apt-get download`. +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update -qq && \ + apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances --no-pre-depends \ + curl \ + gosu \ + libjpeg62-turbo \ + libpq5 \ + libwebp7 \ + xmlsec1 \ + libjemalloc2 \ + libicu \ + | grep '^\w' > /tmp/pkg-list && \ + for arch in arm64 amd64; do \ + mkdir -p /tmp/debs-${arch} && \ + cd /tmp/debs-${arch} && \ + apt-get -o APT::Architecture="${arch}" download $(cat /tmp/pkg-list); \ + done + +# Extract the debs for each architecture +RUN \ + for arch in arm64 amd64; do \ + mkdir -p /install-${arch}/var/lib/dpkg/status.d/ && \ + for deb in /tmp/debs-${arch}/*.deb; do \ + package_name=$(dpkg-deb -I ${deb} | awk '/^ Package: .*$/ {print $2}'); \ + echo "Extracting: ${package_name}"; \ + dpkg --ctrl-tarfile $deb | tar -Ox ./control > /install-${arch}/var/lib/dpkg/status.d/${package_name}; \ + dpkg --extract $deb /install-${arch}; \ + done; \ + done + + +### +### Stage 3: runtime ### -FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm +FROM docker.io/library/python:${PYTHON_VERSION}-slim-${DEBIAN_VERSION} + +ARG TARGETARCH LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse' LABEL org.opencontainers.image.documentation='https://github.com/element-hq/synapse/blob/master/docker/README.md' LABEL org.opencontainers.image.source='https://github.com/element-hq/synapse.git' LABEL org.opencontainers.image.licenses='AGPL-3.0-or-later' -RUN \ - --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -qq && apt-get install -yqq \ - curl \ - gosu \ - libjpeg62-turbo \ - libpq5 \ - libwebp7 \ - xmlsec1 \ - libjemalloc2 \ - libicu72 \ - libssl-dev \ - openssl \ - && rm -rf /var/lib/apt/lists/* - +# On the runtime image, /lib is a symlink to /usr/lib, so we need to copy the +# libraries to the right place, else the `COPY` won't work. +# On amd64, we'll also have a /lib64 folder with ld-linux-x86-64.so.2, which is +# already present in the runtime image. +COPY --from=runtime-deps /install-${TARGETARCH}/lib /usr/lib +COPY --from=runtime-deps /install-${TARGETARCH}/etc /etc +COPY --from=runtime-deps /install-${TARGETARCH}/usr /usr +COPY --from=runtime-deps /install-${TARGETARCH}/var /var COPY --from=builder /install /usr/local COPY ./docker/start.py /start.py COPY ./docker/conf /conf diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 2ceb6ab67c..6d0fc1440b 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -2,18 +2,38 @@ ARG SYNAPSE_VERSION=latest ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION +ARG DEBIAN_VERSION=bookworm +ARG PYTHON_VERSION=3.12 -# first of all, we create a base image with an nginx which we can copy into the +# first of all, we create a base image with dependencies which we can copy into the # target image. For repeated rebuilds, this is much faster than apt installing # each time. -FROM docker.io/library/debian:bookworm-slim AS deps_base +FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS deps_base + + # Tell apt to keep downloaded package files, as we're using cache mounts. + RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache + RUN \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update -qq && \ DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends \ - redis-server nginx-light + nginx-light + + RUN \ + # remove default page + rm /etc/nginx/sites-enabled/default && \ + # have nginx log to stderr/out + ln -sf /dev/stdout /var/log/nginx/access.log && \ + ln -sf /dev/stderr /var/log/nginx/error.log + + # --link-mode=copy silences a warning as uv isn't able to do hardlinks between its cache + # (mounted as --mount=type=cache) and the target directory. + RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --link-mode=copy --prefix="/uv/usr/local" supervisor~=4.2 + + RUN mkdir -p /uv/etc/supervisor/conf.d # Similarly, a base to copy the redis server from. # @@ -21,31 +41,21 @@ FROM docker.io/library/debian:bookworm-slim AS deps_base # which makes it much easier to copy (but we need to make sure we use an image # based on the same debian version as the synapse image, to make sure we get # the expected version of libc. -FROM docker.io/library/redis:7-bookworm AS redis_base +FROM docker.io/library/redis:7-${DEBIAN_VERSION} AS redis_base # now build the final image, based on the the regular Synapse docker image FROM $FROM - # Install supervisord with pip instead of apt, to avoid installing a second - # copy of python. - RUN --mount=type=cache,target=/root/.cache/pip \ - pip install supervisor~=4.2 - RUN mkdir -p /etc/supervisor/conf.d - - # Copy over redis and nginx + # Copy over dependencies COPY --from=redis_base /usr/local/bin/redis-server /usr/local/bin - + COPY --from=deps_base /uv / COPY --from=deps_base /usr/sbin/nginx /usr/sbin COPY --from=deps_base /usr/share/nginx /usr/share/nginx COPY --from=deps_base /usr/lib/nginx /usr/lib/nginx COPY --from=deps_base /etc/nginx /etc/nginx - RUN rm /etc/nginx/sites-enabled/default - RUN mkdir /var/log/nginx /var/lib/nginx - RUN chown www-data /var/lib/nginx - - # have nginx log to stderr/out - RUN ln -sf /dev/stdout /var/log/nginx/access.log - RUN ln -sf /dev/stderr /var/log/nginx/error.log + COPY --from=deps_base /var/log/nginx /var/log/nginx + # chown to allow non-root user to write to http-*-temp-path dirs + COPY --from=deps_base --chown=www-data:root /var/lib/nginx /var/lib/nginx # Copy Synapse worker, nginx and supervisord configuration template files COPY ./docker/conf-workers/* /conf/ @@ -64,4 +74,4 @@ FROM $FROM # Replace the healthcheck with one which checks *all* the workers. The script # is generated by configure_workers_and_start.py. HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ - CMD /bin/sh /healthcheck.sh + CMD ["/healthcheck.sh"] diff --git a/docker/README.md b/docker/README.md index 8dba6fdb05..3438e9c441 100644 --- a/docker/README.md +++ b/docker/README.md @@ -114,6 +114,9 @@ The following environment variables are supported in `run` mode: is set via `docker run --user`, defaults to `991`, `991`. Note that this user must have permission to read the config files, and write to the data directories. * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`. +* `SYNAPSE_HTTP_PROXY`: Passed through to the Synapse process as the `http_proxy` environment variable. +* `SYNAPSE_HTTPS_PROXY`: Passed through to the Synapse process as the `https_proxy` environment variable. +* `SYNAPSE_NO_PROXY`: Passed through to the Synapse process as `no_proxy` environment variable. For more complex setups (e.g. for workers) you can also pass your args directly to synapse using `run` mode. For example like this: diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile index ce82c400eb..6ed084fe5d 100644 --- a/docker/complement/Dockerfile +++ b/docker/complement/Dockerfile @@ -9,6 +9,9 @@ ARG SYNAPSE_VERSION=latest # This is an intermediate image, to be built locally (not pulled from a registry). ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION +ARG DEBIAN_VERSION=bookworm + +FROM docker.io/library/postgres:13-${DEBIAN_VERSION} AS postgres_base FROM $FROM # First of all, we copy postgres server from the official postgres image, @@ -20,9 +23,9 @@ FROM $FROM # the same debian version as Synapse's docker image (so the versions of the # shared libraries match). RUN adduser --system --uid 999 postgres --home /var/lib/postgresql -COPY --from=docker.io/library/postgres:13-bookworm /usr/lib/postgresql /usr/lib/postgresql -COPY --from=docker.io/library/postgres:13-bookworm /usr/share/postgresql /usr/share/postgresql -RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql +COPY --from=postgres_base /usr/lib/postgresql /usr/lib/postgresql +COPY --from=postgres_base /usr/share/postgresql /usr/share/postgresql +COPY --from=postgres_base --chown=postgres /var/run/postgresql /var/run/postgresql ENV PATH="${PATH}:/usr/lib/postgresql/13/bin" ENV PGDATA=/var/lib/postgresql/data @@ -55,4 +58,4 @@ ENTRYPOINT ["/start_for_complement.sh"] # Update the healthcheck to have a shorter check interval HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ - CMD /bin/sh /healthcheck.sh + CMD ["/healthcheck.sh"] diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh index cc798a3210..da1b26a283 100755 --- a/docker/complement/conf/start_for_complement.sh +++ b/docker/complement/conf/start_for_complement.sh @@ -5,12 +5,12 @@ set -e echo "Complement Synapse launcher" -echo " Args: $@" +echo " Args: $*" echo " Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=$SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR" function log { - d=$(date +"%Y-%m-%d %H:%M:%S,%3N") - echo "$d $@" + d=$(printf '%(%Y-%m-%d %H:%M:%S)T,%.3s\n' ${EPOCHREALTIME/./ }) + echo "$d $*" } # Set the server name of the homeserver @@ -54,7 +54,6 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then export SYNAPSE_WORKER_TYPES="\ event_persister:2, \ background_worker, \ - frontend_proxy, \ event_creator, \ user_dir, \ media_repository, \ @@ -65,6 +64,7 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then client_reader, \ appservice, \ pusher, \ + device_lists:2, \ stream_writers=account_data+presence+receipts+to_device+typing" fi @@ -103,12 +103,11 @@ fi # Note that both the key and certificate are in PEM format (not DER). # First generate a configuration file to set up a Subject Alternative Name. -cat > /conf/server.tls.conf < /conf/server.tls.conf # Generate an RSA key openssl genrsa -out /conf/server.tls.key 2048 @@ -123,12 +122,12 @@ openssl x509 -req -in /conf/server.tls.csr \ -out /conf/server.tls.crt -extfile /conf/server.tls.conf -extensions SAN # Assert that we have a Subject Alternative Name in the certificate. -# (grep will exit with 1 here if there isn't a SAN in the certificate.) -openssl x509 -in /conf/server.tls.crt -noout -text | grep DNS: +# (the test will exit with 1 here if there isn't a SAN in the certificate.) +[[ $(openssl x509 -in /conf/server.tls.crt -noout -text) == *DNS:* ]] export SYNAPSE_TLS_CERT=/conf/server.tls.crt export SYNAPSE_TLS_KEY=/conf/server.tls.key # Run the script that writes the necessary config files and starts supervisord, which in turn # starts everything else -exec /configure_workers_and_start.py +exec /configure_workers_and_start.py "$@" diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 9a74c617bc..94e74df9d1 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -7,6 +7,7 @@ #} ## Server ## +public_baseurl: http://127.0.0.1:8008/ report_stats: False trusted_key_servers: [] enable_registration: true @@ -84,6 +85,22 @@ rc_invites: per_user: per_second: 1000 burst_count: 1000 + per_issuer: + per_second: 1000 + burst_count: 1000 + +rc_presence: + per_user: + per_second: 9999 + burst_count: 9999 + +rc_delayed_event_mgmt: + per_second: 9999 + burst_count: 9999 + +rc_room_creation: + per_second: 9999 + burst_count: 9999 federation_rr_transactions_per_room_per_second: 9999 @@ -114,6 +131,10 @@ experimental_features: msc3983_appservice_otk_claims: true # Proxy key queries to exclusive ASes msc3984_appservice_key_query: true + # Invite filtering + msc4155_enabled: true + # Thread Subscriptions + msc4306_enabled: true server_notices: system_mxid_localpart: _server @@ -130,4 +151,9 @@ caches: sync_response_cache_duration: 0 +# Complement assumes that it can publish to the room list by default. +room_list_publication_rules: + - action: allow + + {% include "shared-orig.yaml.j2" %} diff --git a/docker/conf-workers/nginx.conf.j2 b/docker/conf-workers/nginx.conf.j2 index d1e02af723..95d2f760d2 100644 --- a/docker/conf-workers/nginx.conf.j2 +++ b/docker/conf-workers/nginx.conf.j2 @@ -38,10 +38,13 @@ server { {% if using_unix_sockets %} proxy_pass http://unix:/run/main_public.sock; {% else %} + # note: do not add a path (even a single /) after the port in `proxy_pass`, + # otherwise nginx will canonicalise the URI and cause signature verification + # errors. proxy_pass http://localhost:8080; {% endif %} proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $host; + proxy_set_header Host $host:$server_port; } } diff --git a/docker/conf-workers/synapse.supervisord.conf.j2 b/docker/conf-workers/synapse.supervisord.conf.j2 index 481eb4fc92..4fb11b259e 100644 --- a/docker/conf-workers/synapse.supervisord.conf.j2 +++ b/docker/conf-workers/synapse.supervisord.conf.j2 @@ -1,5 +1,6 @@ {% if use_forking_launcher %} [program:synapse_fork] +environment=http_proxy="%(ENV_SYNAPSE_HTTP_PROXY)s",https_proxy="%(ENV_SYNAPSE_HTTPS_PROXY)s",no_proxy="%(ENV_SYNAPSE_NO_PROXY)s" command=/usr/local/bin/python -m synapse.app.complement_fork_starter {{ main_config_path }} synapse.app.homeserver @@ -20,6 +21,7 @@ exitcodes=0 {% else %} [program:synapse_main] +environment=http_proxy="%(ENV_SYNAPSE_HTTP_PROXY)s",https_proxy="%(ENV_SYNAPSE_HTTPS_PROXY)s",no_proxy="%(ENV_SYNAPSE_NO_PROXY)s" command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml @@ -36,6 +38,7 @@ exitcodes=0 {% for worker in workers %} [program:synapse_{{ worker.name }}] +environment=http_proxy="%(ENV_SYNAPSE_HTTP_PROXY)s",https_proxy="%(ENV_SYNAPSE_HTTPS_PROXY)s",no_proxy="%(ENV_SYNAPSE_NO_PROXY)s" command=/usr/local/bin/prefix-log /usr/local/bin/python -m {{ worker.app }} --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml diff --git a/docker/conf/log.config b/docker/conf/log.config index 5772321202..6fe7db66da 100644 --- a/docker/conf/log.config +++ b/docker/conf/log.config @@ -77,6 +77,13 @@ loggers: #} synapse.visibility.filtered_event_debug: level: DEBUG + + {# + If Synapse is under test, we don't care about seeing the "Applying schema" log + lines at the INFO level every time we run the tests (it's 100 lines of bulk) + #} + synapse.storage.prepare_database: + level: WARN {% endif %} root: diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 15d8d7b558..6f25653bb7 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/local/bin/python # # This file is licensed under the Affero General Public License (AGPL) version 3. # @@ -178,6 +178,9 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "^/_matrix/client/(api/v1|r0|v3|unstable)/login$", "^/_matrix/client/(api/v1|r0|v3|unstable)/account/3pid$", "^/_matrix/client/(api/v1|r0|v3|unstable)/account/whoami$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/account/deactivate$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/devices(/|$)", + "^/_matrix/client/(r0|v3)/delete_devices$", "^/_matrix/client/versions$", "^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$", "^/_matrix/client/(r0|v3|unstable)/register$", @@ -194,6 +197,9 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$", "^/_matrix/client/(r0|v3|unstable)/capabilities$", "^/_matrix/client/(r0|v3|unstable)/notifications$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload", + "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/device_signing/upload$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$", ], "shared_extra_conf": {}, "worker_extra_conf": "", @@ -202,6 +208,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "app": "synapse.app.generic_worker", "listener_resources": ["federation"], "endpoint_patterns": [ + "^/_matrix/federation/v1/version$", "^/_matrix/federation/(v1|v2)/event/", "^/_matrix/federation/(v1|v2)/state/", "^/_matrix/federation/(v1|v2)/state_ids/", @@ -264,13 +271,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, - "frontend_proxy": { - "app": "synapse.app.generic_worker", - "listener_resources": ["client", "replication"], - "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"], - "shared_extra_conf": {}, - "worker_extra_conf": "", - }, "account_data": { "app": "synapse.app.generic_worker", "listener_resources": ["client", "replication"], @@ -305,6 +305,13 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, + "device_lists": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, "typing": { "app": "synapse.app.generic_worker", "listener_resources": ["client", "replication"], @@ -321,6 +328,15 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, + "thread_subscriptions": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/unstable/io.element.msc4306/.*", + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, } # Templates for sections that may be inserted multiple times in config files @@ -351,6 +367,11 @@ def error(txt: str) -> NoReturn: def flush_buffers() -> None: + """ + Python's `print()` buffers output by default, typically waiting until ~8KB + accumulates. This method can be used to flush the buffers so we can see the output + of any print statements so far. + """ sys.stdout.flush() sys.stderr.flush() @@ -376,9 +397,11 @@ def convert(src: str, dst: str, **template_vars: object) -> None: # # We use append mode in case the files have already been written to by something else # (for instance, as part of the instructions in a dockerfile). + exists = os.path.isfile(dst) with open(dst, "a") as outfile: # In case the existing file doesn't end with a newline - outfile.write("\n") + if exists: + outfile.write("\n") outfile.write(rendered) @@ -404,16 +427,18 @@ def add_worker_roles_to_shared_config( # streams instance_map = shared_config.setdefault("instance_map", {}) - # This is a list of the stream_writers that there can be only one of. Events can be - # sharded, and therefore doesn't belong here. - singular_stream_writers = [ + # This is a list of the stream_writers. + stream_writers = { "account_data", + "events", + "device_lists", "presence", "receipts", "to_device", "typing", "push_rules", - ] + "thread_subscriptions", + } # Worker-type specific sharding config. Now a single worker can fulfill multiple # roles, check each. @@ -423,28 +448,11 @@ def add_worker_roles_to_shared_config( if "federation_sender" in worker_types_set: shared_config.setdefault("federation_sender_instances", []).append(worker_name) - if "event_persister" in worker_types_set: - # Event persisters write to the events stream, so we need to update - # the list of event stream writers - shared_config.setdefault("stream_writers", {}).setdefault("events", []).append( - worker_name - ) - - # Map of stream writer instance names to host/ports combos - if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False): - instance_map[worker_name] = { - "path": f"/run/worker.{worker_port}", - } - else: - instance_map[worker_name] = { - "host": "localhost", - "port": worker_port, - } # Update the list of stream writers. It's convenient that the name of the worker # type is the same as the stream to write. Iterate over the whole list in case there # is more than one. for worker in worker_types_set: - if worker in singular_stream_writers: + if worker in stream_writers: shared_config.setdefault("stream_writers", {}).setdefault( worker, [] ).append(worker_name) @@ -604,7 +612,7 @@ def generate_base_homeserver_config() -> None: # start.py already does this for us, so just call that. # note that this script is copied in in the official, monolith dockerfile os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT) - subprocess.run(["/usr/local/bin/python", "/start.py", "migrate_config"], check=True) + subprocess.run([sys.executable, "/start.py", "migrate_config"], check=True) def parse_worker_types( @@ -868,6 +876,13 @@ def generate_worker_files( else: healthcheck_urls.append("http://localhost:%d/health" % (worker_port,)) + # Special case for event_persister: those are just workers that write to + # the `events` stream. For other workers, the worker name is the same + # name of the stream they write to, but for some reason it is not the + # case for event_persister. + if "event_persister" in worker_types_set: + worker_types_set.add("events") + # Update the shared config with sharding-related options if necessary add_worker_roles_to_shared_config( shared_config, worker_types_set, worker_name, worker_port @@ -998,6 +1013,7 @@ def generate_worker_files( "/healthcheck.sh", healthcheck_urls=healthcheck_urls, ) + os.chmod("/healthcheck.sh", 0o755) # Ensure the logging directory exists log_dir = data_dir + "/logs" @@ -1099,6 +1115,13 @@ def main(args: List[str], environ: MutableMapping[str, str]) -> None: else: log("Could not find %s, will not use" % (jemallocpath,)) + # Empty strings are falsy in Python so this default is fine. We just can't have these + # be undefined because supervisord will complain about our + # `%(ENV_SYNAPSE_HTTP_PROXY)s` usage. + environ.setdefault("SYNAPSE_HTTP_PROXY", "") + environ.setdefault("SYNAPSE_HTTPS_PROXY", "") + environ.setdefault("SYNAPSE_NO_PROXY", "") + # Start supervisord, which will start Synapse, all of the configured worker # processes, redis, nginx etc. according to the config we created above. log("Starting supervisord") diff --git a/docker/prefix-log b/docker/prefix-log index 32dddbbfd4..2a38de5686 100755 --- a/docker/prefix-log +++ b/docker/prefix-log @@ -10,6 +10,9 @@ # '-W interactive' is a `mawk` extension which disables buffering on stdout and sets line-buffered reads on # stdin. The effect is that the output is flushed after each line, rather than being batched, which helps reduce # confusion due to to interleaving of the different processes. -exec 1> >(awk -W interactive '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0 }' >&1) -exec 2> >(awk -W interactive '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0 }' >&2) +prefixer() { + mawk -W interactive '{printf("%s | %s\n", ENVIRON["SUPERVISOR_PROCESS_NAME"], $0); fflush() }' +} +exec 1> >(prefixer) +exec 2> >(prefixer >&2) exec "$@" diff --git a/docker/start.py b/docker/start.py index 818a5355ca..0be9976a0c 100755 --- a/docker/start.py +++ b/docker/start.py @@ -22,6 +22,11 @@ def error(txt: str) -> NoReturn: def flush_buffers() -> None: + """ + Python's `print()` buffers output by default, typically waiting until ~8KB + accumulates. This method can be used to flush the buffers so we can see the output + of any print statements so far. + """ sys.stdout.flush() sys.stderr.flush() diff --git a/docs/README.md b/docs/README.md index 0b2b910c73..7802d3c3ce 100644 --- a/docs/README.md +++ b/docs/README.md @@ -63,6 +63,18 @@ mdbook serve The URL at which the docs can be viewed at will be logged. +## Synapse configuration documentation + +The [Configuration +Manual](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html) +page is generated from a YAML file, +[schema/synapse-config.schema.yaml](../schema/synapse-config.schema.yaml). To +add new options or modify existing ones, first edit that file, then run +[scripts-dev/gen_config_documentation.py](../scripts-dev/gen_config_documentation.py) +to generate an updated Configuration Manual markdown file. + +Build the book as described above to preview it in a web browser. + ## Configuration and theming The look and behaviour of the website is configured by the [book.toml](../book.toml) file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index fd91d9fa11..52f827c8df 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -49,6 +49,8 @@ - [Background update controller callbacks](modules/background_update_controller_callbacks.md) - [Account data callbacks](modules/account_data_callbacks.md) - [Add extra fields to client events unsigned section callbacks](modules/add_extra_fields_to_client_events_unsigned.md) + - [Media repository callbacks](modules/media_repository_callbacks.md) + - [Ratelimit callbacks](modules/ratelimit_callbacks.md) - [Porting a legacy module to the new interface](modules/porting_legacy_module.md) - [Workers](workers.md) - [Using `synctl` with Workers](synctl_workers.md) @@ -66,11 +68,13 @@ - [Registration Tokens](usage/administration/admin_api/registration_tokens.md) - [Manipulate Room Membership](admin_api/room_membership.md) - [Rooms](admin_api/rooms.md) + - [Scheduled tasks](admin_api/scheduled_tasks.md) - [Server Notices](admin_api/server_notices.md) - [Statistics](admin_api/statistics.md) - [Users](admin_api/user_admin_api.md) - [Server Version](admin_api/version_api.md) - [Federation](usage/administration/admin_api/federation.md) + - [Client-Server API Extensions](admin_api/client_server_api_extensions.md) - [Manhole](manhole.md) - [Monitoring](metrics-howto.md) - [Reporting Homeserver Usage Statistics](usage/administration/monitoring/reporting_homeserver_usage_statistics.md) diff --git a/docs/admin_api/client_server_api_extensions.md b/docs/admin_api/client_server_api_extensions.md new file mode 100644 index 0000000000..08fac6289b --- /dev/null +++ b/docs/admin_api/client_server_api_extensions.md @@ -0,0 +1,67 @@ +# Client-Server API Extensions + +Server administrators can set special account data to change how the Client-Server API behaves for +their clients. Setting the account data, or having it already set, as a non-admin has no effect. + +All configuration options can be set through the `io.element.synapse.admin_client_config` global +account data on the admin's user account. + +Example: +``` +PUT /_matrix/client/v3/user/{adminUserId}/account_data/io.element.synapse.admin_client_config +{ + "return_soft_failed_events": true +} +``` + +## See soft failed events + +Learn more about soft failure from [the spec](https://spec.matrix.org/v1.14/server-server-api/#soft-failure). + +To receive soft failed events in APIs like `/sync` and `/messages`, set `return_soft_failed_events` +to `true` in the admin client config. When `false`, the normal behaviour of these endpoints is to +exclude soft failed events. + +**Note**: If the policy server flagged the event as spam and that caused soft failure, that will be indicated +in the event's `unsigned` content like so: + +```json +{ + "type": "m.room.message", + "other": "event_fields_go_here", + "unsigned": { + "io.element.synapse.soft_failed": true, + "io.element.synapse.policy_server_spammy": true + } +} +``` + +Default: `false` + +## See events marked spammy by policy servers + +Learn more about policy servers from [MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284). + +Similar to `return_soft_failed_events`, clients logged in with admin accounts can see events which were +flagged by the policy server as spammy (and thus soft failed) by setting `return_policy_server_spammy_events` +to `true`. + +`return_policy_server_spammy_events` may be `true` while `return_soft_failed_events` is `false` to only see +policy server-flagged events. When `return_soft_failed_events` is `true` however, `return_policy_server_spammy_events` +is always `true`. + +Events which were flagged by the policy will be flagged as `io.element.synapse.policy_server_spammy` in the +event's `unsigned` content, like so: + +```json +{ + "type": "m.room.message", + "other": "event_fields_go_here", + "unsigned": { + "io.element.synapse.soft_failed": true, + "io.element.synapse.policy_server_spammy": true + } +} +``` + +Default: `true` if `return_soft_failed_events` is `true`, otherwise `false` diff --git a/docs/admin_api/event_reports.md b/docs/admin_api/event_reports.md index 83f7dc37f4..225b431715 100644 --- a/docs/admin_api/event_reports.md +++ b/docs/admin_api/event_reports.md @@ -60,10 +60,11 @@ paginate through. anything other than the return value of `next_token` from a previous call. Defaults to `0`. * `dir`: string - Direction of event report order. Whether to fetch the most recent first (`b`) or the oldest first (`f`). Defaults to `b`. -* `user_id`: string - Is optional and filters to only return users with user IDs that - contain this value. This is the user who reported the event and wrote the reason. -* `room_id`: string - Is optional and filters to only return rooms with room IDs that - contain this value. +* `user_id`: optional string - Filter by the user ID of the reporter. This is the user who reported the event + and wrote the reason. +* `room_id`: optional string - Filter by room id. +* `event_sender_user_id`: optional string - Filter by the sender of the reported event. This is the user who + the report was made against. **Response** @@ -116,7 +117,6 @@ It returns a JSON body like the following: "hashes": { "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw" }, - "origin": "matrix.org", "origin_server_ts": 1592291711430, "prev_events": [ "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M" diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 30833f3109..1177711c1e 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -46,6 +46,14 @@ to any local media, and any locally-cached copies of remote media. The media file itself (and any thumbnails) is not deleted from the server. +Since Synapse 1.128.0, hashes of uploaded media are tracked. If this media +is quarantined, Synapse will: + + - Quarantine any media with a matching hash that has already been uploaded. + - Quarantine any future media. + - Quarantine any existing cached remote media. + - Quarantine any future remote media. + ## Quarantining media by ID This API quarantines a single piece of local or remote media. diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index 8e3a367e90..12af87148d 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -385,6 +385,13 @@ The API is: GET /_synapse/admin/v1/rooms//state ``` +**Parameters** + +The following query parameter is available: + +* `type` - The type of room state event to filter by, eg "m.room.create". If provided, only state events + of this type will be returned (regardless of their `state_key` value). + A response body like the following is returned: ```json @@ -787,6 +794,7 @@ A response body like the following is returned: "results": [ { "delete_id": "delete_id1", + "room_id": "!roomid:example.com", "status": "failed", "error": "error message", "shutdown_room": { @@ -797,7 +805,8 @@ A response body like the following is returned: } }, { "delete_id": "delete_id2", - "status": "purging", + "room_id": "!roomid:example.com", + "status": "active", "shutdown_room": { "kicked_users": [ "@foobar:example.com" @@ -834,7 +843,9 @@ A response body like the following is returned: ```json { - "status": "purging", + "status": "active", + "delete_id": "bHkCNQpHqOaFhPtK", + "room_id": "!roomid:example.com", "shutdown_room": { "kicked_users": [ "@foobar:example.com" @@ -862,10 +873,11 @@ The following fields are returned in the JSON response body: - `results` - An array of objects, each containing information about one task. This field is omitted from the result when you query by `delete_id`. Task objects contain the following fields: - - `delete_id` - The ID for this purge if you query by `room_id`. + - `delete_id` - The ID for this purge + - `room_id` - The ID of the room being deleted - `status` - The status will be one of: - - `shutting_down` - The process is removing users from the room. - - `purging` - The process is purging the room and event data from database. + - `scheduled` - The deletion is waiting to be started + - `active` - The process is purging the room and event data from database. - `complete` - The process has completed successfully. - `failed` - The process is aborted, an error has occurred. - `error` - A string that shows an error message if `status` is `failed`. diff --git a/docs/admin_api/scheduled_tasks.md b/docs/admin_api/scheduled_tasks.md new file mode 100644 index 0000000000..b80da5083c --- /dev/null +++ b/docs/admin_api/scheduled_tasks.md @@ -0,0 +1,54 @@ +# Show scheduled tasks + +This API returns information about scheduled tasks. + +To use it, you will need to authenticate by providing an `access_token` +for a server admin: see [Admin API](../usage/administration/admin_api/). + +The api is: +``` +GET /_synapse/admin/v1/scheduled_tasks +``` + +It returns a JSON body like the following: + +```json +{ + "scheduled_tasks": [ + { + "id": "GSA124oegf1", + "action": "shutdown_room", + "status": "complete", + "timestamp_ms": 23423523, + "resource_id": "!roomid", + "result": "some result", + "error": null + } + ] +} +``` + +**Query parameters:** + +* `action_name`: string - Is optional. Returns only the scheduled tasks with the given action name. +* `resource_id`: string - Is optional. Returns only the scheduled tasks with the given resource id. +* `status`: string - Is optional. Returns only the scheduled tasks matching the given status, one of + - "scheduled" - Task is scheduled but not active + - "active" - Task is active and probably running, and if not will be run on next scheduler loop run + - "complete" - Task has completed successfully + - "failed" - Task is over and either returned a failed status, or had an exception + +* `max_timestamp`: int - Is optional. Returns only the scheduled tasks with a timestamp inferior to the specified one. + +**Response** + +The following fields are returned in the JSON response body along with a `200` HTTP status code: + +* `id`: string - ID of scheduled task. +* `action`: string - The name of the scheduled task's action. +* `status`: string - The status of the scheduled task. +* `timestamp_ms`: integer - The timestamp (in milliseconds since the unix epoch) of the given task - If the status is "scheduled" then this represents when it should be launched. + Otherwise it represents the last time this task got a change of state. +* `resource_id`: Optional string - The resource id of the scheduled task, if it possesses one +* `result`: Optional Json - Any result of the scheduled task, if given +* `error`: Optional string - If the task has the status "failed", the error associated with this failure diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 96a2994b7b..4de7e85642 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -40,6 +40,7 @@ It returns a JSON body like the following: "erased": false, "shadow_banned": 0, "creation_ts": 1560432506, + "last_seen_ts": 1732919539393, "appservice_id": null, "consent_server_notice_sent": null, "consent_version": null, @@ -55,7 +56,8 @@ It returns a JSON body like the following: } ], "user_type": null, - "locked": false + "locked": false, + "suspended": false } ``` @@ -161,7 +163,8 @@ Body parameters: - `locked` - **bool**, optional. If unspecified, locked state will be left unchanged. - `user_type` - **string** or null, optional. If not provided, the user type will be not be changed. If `null` is given, the user type will be cleared. - Other allowed options are: `bot` and `support`. + Other allowed options are: `bot` and `support` and any extra values defined in the homserver + [configuration](../usage/configuration/config_documentation.md#user_types). ## List Accounts ### List Accounts (V2) @@ -412,6 +415,32 @@ The following actions are **NOT** performed. The list may be incomplete. - Remove from monthly active users - Remove user's consent information (consent version and timestamp) +## Suspend/Unsuspend Account + +This API allows an admin to suspend/unsuspend an account. While an account is suspended, the user is +prohibited from sending invites, joining or knocking on rooms, sending messages, changing profile data, and redacting messages other than their own. + +The api is: + +``` +PUT /_synapse/admin/v1/suspend/ +``` + +with a body of: + +```json +{ + "suspend": true +} +``` + +To unsuspend a user, use the same endpoint with a body of: +```json +{ + "suspend": false +} +``` + ## Reset password **Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) @@ -476,9 +505,9 @@ with a body of: } ``` -## List room memberships of a user +## List joined rooms of a user -Gets a list of all `room_id` that a specific `user_id` is member. +Gets a list of all `room_id` that a specific `user_id` is joined to and is a member of (participating in). The API is: @@ -515,6 +544,73 @@ The following fields are returned in the JSON response body: - `joined_rooms` - An array of `room_id`. - `total` - Number of rooms. +## Get the number of invites sent by the user + +Fetches the number of invites sent by the provided user ID across all rooms +after the given timestamp. + +``` +GET /_synapse/admin/v1/users/$user_id/sent_invite_count +``` + +**Parameters** + +The following parameters should be set in the URL: + +* `user_id`: fully qualified: for example, `@user:server.com` + +The following should be set as query parameters in the URL: + +* `from_ts`: int, required. A timestamp in ms from the unix epoch. Only + invites sent at or after the provided timestamp will be returned. + This works by comparing the provided timestamp to the `received_ts` + column in the `events` table. + Note: https://currentmillis.com/ is a useful tool for converting dates + into timestamps and vice versa. + +A response body like the following is returned: + +```json +{ + "invite_count": 30 +} +``` + +_Added in Synapse 1.122.0_ + +## Get the cumulative number of rooms a user has joined after a given timestamp + +Fetches the number of rooms that the user joined after the given timestamp, even +if they have subsequently left/been banned from those rooms. + +``` +GET /_synapse/admin/v1/users/$" + "user_id": "", + "dehydrated": false }, { "device_id": "AUIECTSRND", @@ -867,7 +964,8 @@ A response body like the following is returned: "last_seen_ip": "1.2.3.5", "last_seen_user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0", "last_seen_ts": 1474491775025, - "user_id": "" + "user_id": "", + "dehydrated": false } ], "total": 2 @@ -897,6 +995,7 @@ The following fields are returned in the JSON response body: - `last_seen_ts` - The timestamp (in milliseconds since the unix epoch) when this devices was last seen. (May be a few minutes out of date, for efficiency reasons). - `user_id` - Owner of device. + - `dehydrated` - Whether the device is a dehydrated device. - `total` - Total number of user's devices. @@ -1128,7 +1227,7 @@ See also the ## Controlling whether a user is shadow-banned -Shadow-banning is a useful tool for moderating malicious or egregiously abusive users. +Shadow-banning is a useful tool for moderating malicious or egregiously abusive users. A shadow-banned users receives successful responses to their client-server API requests, but the events are not propagated into rooms. This can be an effective tool as it (hopefully) takes longer for the user to realise they are being moderated before @@ -1365,8 +1464,11 @@ _Added in Synapse 1.72.0._ ## Redact all the events of a user -This endpoint allows an admin to redact the events of a given user. There are no restrictions on redactions for a -local user. By default, we puppet the user who sent the message to redact it themselves. Redactions for non-local users are issued using the admin user, and will fail in rooms where the admin user is not admin/does not have the specified power level to issue redactions. +This endpoint allows an admin to redact the events of a given user. There are no restrictions on +redactions for a local user. By default, we puppet the user who sent the message to redact it themselves. +Redactions for non-local users are issued using the admin user, and will fail in rooms where the +admin user is not admin/does not have the specified power level to issue redactions. An option +is provided to override the default and allow the admin to issue the redactions in all cases. The API is ``` @@ -1376,7 +1478,7 @@ POST /_synapse/admin/v1/user/$user_id/redact "rooms": ["!roomid1", "!roomid2"] } ``` -If an empty list is provided as the key for `rooms`, all events in all the rooms the user is member of will be redacted, +If an empty list is provided as the key for `rooms`, all events in all the rooms the user is member of will be redacted, otherwise all the events in the rooms provided in the request will be redacted. The API starts redaction process running, and returns immediately with a JSON body with @@ -1399,12 +1501,15 @@ The following JSON body parameter must be provided: - `rooms` - A list of rooms to redact the user's events in. If an empty list is provided all events in all rooms the user is a member of will be redacted -_Added in Synapse 1.116.0._ - The following JSON body parameters are optional: - `reason` - Reason the redaction is being requested, ie "spam", "abuse", etc. This will be included in each redaction event, and be visible to users. -- `limit` - a limit on the number of the user's events to search for ones that can be redacted (events are redacted newest to oldest) in each room, defaults to 1000 if not provided +- `limit` - a limit on the number of the user's events to search for ones that can be redacted (events are redacted newest to oldest) in each room, defaults to 1000 if not provided. +- `use_admin` - If set to `true`, the admin user is used to issue the redactions, rather than puppeting the user. Useful + when the admin is also the moderator of the rooms that require redactions. Note that the redactions will fail in rooms + where the admin does not have the sufficient power level to issue the redactions. + +_Added in Synapse 1.116.0._ ## Check the status of a redaction process @@ -1443,4 +1548,6 @@ The following fields are returned in the JSON response body: - `failed_redactions` - dictionary - the keys of the dict are event ids the process was unable to redact, if any, and the values are the corresponding error that caused the redaction to fail -_Added in Synapse 1.116.0._ \ No newline at end of file +_Added in Synapse 1.116.0._ + + diff --git a/docs/changelogs/CHANGES-2023.md b/docs/changelogs/CHANGES-2023.md new file mode 100644 index 0000000000..9b6ad3de1b --- /dev/null +++ b/docs/changelogs/CHANGES-2023.md @@ -0,0 +1,2202 @@ +# Synapse 1.98.0 (2023-12-12) + +Synapse 1.98.0 will be the last Synapse release in 2023; the regular release cadence will resume in January 2024. + +Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for +proprietary dual licensing). You can read more about this here: + + - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ + - https://element.io/blog/element-to-adopt-agplv3/ + +The Matrix.org Foundation copy of the project will be archived. Any changes needed +by server administrators will be communicated via our usual announcements channels, +but we are striving to make this as seamless as possible. + + +No significant changes since 1.98.0rc1. + + + +# Synapse 1.98.0rc1 (2023-12-05) + +### Features + +- Synapse now declares support for Matrix v1.7, v1.8, and v1.9. ([\#16707](https://github.com/matrix-org/synapse/issues/16707)) +- Add `on_user_login` [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) callback for when a user logs in. ([\#15207](https://github.com/matrix-org/synapse/issues/15207)) +- Support [MSC4069: Inhibit profile propagation](https://github.com/matrix-org/matrix-spec-proposals/pull/4069). ([\#16636](https://github.com/matrix-org/synapse/issues/16636)) +- Restore tracking of requests and monthly active users when delegating authentication via [MSC3861](https://github.com/matrix-org/synapse/pull/16672) to an OIDC provider. ([\#16672](https://github.com/matrix-org/synapse/issues/16672)) +- Add an autojoin setting for server notices rooms, so users may be joined directly instead of receiving an invite. ([\#16699](https://github.com/matrix-org/synapse/issues/16699)) +- Follow redirects when downloading media over federation (per [MSC3860](https://github.com/matrix-org/matrix-spec-proposals/pull/3860)). ([\#16701](https://github.com/matrix-org/synapse/issues/16701)) + +### Bugfixes + +- Enable refreshable tokens on the admin registration endpoint. ([\#16642](https://github.com/matrix-org/synapse/issues/16642)) +- Consistently bypass rate limits when using the server notice admin API. ([\#16670](https://github.com/matrix-org/synapse/issues/16670)) +- Fix a bug introduced in Synapse 1.7.2 where rooms whose power levels lacked an `events` field could not be upgraded. ([\#16725](https://github.com/matrix-org/synapse/issues/16725)) +- Fix `GET /_synapse/admin/v1/federation/destinations` [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) returning null (instead of 0) for `retry_last_ts` and `retry_interval`. ([\#16729](https://github.com/matrix-org/synapse/issues/16729)) + +### Improved Documentation + +- Add schema rollback information to documentation. ([\#16661](https://github.com/matrix-org/synapse/issues/16661)) +- Fix poetry version typo in the [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16695](https://github.com/matrix-org/synapse/issues/16695)) +- Switch the example UNIX socket paths to `/run`. Add HAProxy example configuration for UNIX sockets. ([\#16700](https://github.com/matrix-org/synapse/issues/16700)) +- Add documentation for how to validate the configuration file with `synapse.config` script. ([\#16714](https://github.com/matrix-org/synapse/issues/16714)) + +### Internal Changes + +- Clean-up unused tables. ([\#16522](https://github.com/matrix-org/synapse/issues/16522)) +- Reduce a little database load while processing state auth chains. ([\#16552](https://github.com/matrix-org/synapse/issues/16552)) +- Reduce database load of pruning old `user_ips`. ([\#16667](https://github.com/matrix-org/synapse/issues/16667)) +- Reduce DB load when forget on leave setting is disabled. ([\#16668](https://github.com/matrix-org/synapse/issues/16668)) +- Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway. ([\#16677](https://github.com/matrix-org/synapse/issues/16677)) +- Correctly read the to-device stream ID on startup using SQLite. ([\#16682](https://github.com/matrix-org/synapse/issues/16682)) +- Reoranganise test files. ([\#16684](https://github.com/matrix-org/synapse/issues/16684)) +- Remove old full schema dumps which are no longer used. ([\#16697](https://github.com/matrix-org/synapse/issues/16697)) +- Raise poetry-core upper bound to <=1.8.1. This allows contributors to import Synapse after `poetry install`ing with Poetry 1.6 and above. Contributed by Mo Balaa. ([\#16702](https://github.com/matrix-org/synapse/issues/16702)) +- Add a workflow to try and automatically fixup linting in a PR. ([\#16704](https://github.com/matrix-org/synapse/issues/16704)) + + +### Updates to locked dependencies + +* Bump cryptography from 41.0.5 to 41.0.6. ([\#16703](https://github.com/matrix-org/synapse/issues/16703)) +* Bump cryptography from 41.0.6 to 41.0.7. ([\#16721](https://github.com/matrix-org/synapse/issues/16721)) +* Bump idna from 3.4 to 3.6. ([\#16720](https://github.com/matrix-org/synapse/issues/16720)) +* Bump jsonschema from 4.19.1 to 4.20.0. ([\#16692](https://github.com/matrix-org/synapse/issues/16692)) +* Bump matrix-org/netlify-pr-preview from 2 to 3. ([\#16719](https://github.com/matrix-org/synapse/issues/16719)) +* Bump phonenumbers from 8.13.23 to 8.13.26. ([\#16722](https://github.com/matrix-org/synapse/issues/16722)) +* Bump prometheus-client from 0.18.0 to 0.19.0. ([\#16691](https://github.com/matrix-org/synapse/issues/16691)) +* Bump pyasn1 from 0.5.0 to 0.5.1. ([\#16689](https://github.com/matrix-org/synapse/issues/16689)) +* Bump pydantic from 2.4.2 to 2.5.1. ([\#16663](https://github.com/matrix-org/synapse/issues/16663)) +* Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). ([\#16673](https://github.com/matrix-org/synapse/issues/16673)) +* Bump pyopenssl from 23.2.0 to 23.3.0. ([\#16662](https://github.com/matrix-org/synapse/issues/16662)) +* Bump ruff from 0.1.4 to 0.1.6. ([\#16690](https://github.com/matrix-org/synapse/issues/16690)) +* Bump sentry-sdk from 1.32.0 to 1.35.0. ([\#16666](https://github.com/matrix-org/synapse/issues/16666)) +* Bump serde from 1.0.192 to 1.0.193. ([\#16693](https://github.com/matrix-org/synapse/issues/16693)) +* Bump sphinx-autodoc2 from 0.4.2 to 0.5.0. ([\#16723](https://github.com/matrix-org/synapse/issues/16723)) +* Bump types-jsonschema from 4.19.0.4 to 4.20.0.0. ([\#16724](https://github.com/matrix-org/synapse/issues/16724)) +* Bump types-pillow from 10.1.0.0 to 10.1.0.2. ([\#16664](https://github.com/matrix-org/synapse/issues/16664)) +* Bump types-psycopg2 from 2.9.21.15 to 2.9.21.16. ([\#16665](https://github.com/matrix-org/synapse/issues/16665)) +* Bump types-setuptools from 68.2.0.0 to 68.2.0.2. ([\#16688](https://github.com/matrix-org/synapse/issues/16688)) + +# Synapse 1.97.0 (2023-11-28) + +Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for +proprietary dual licensing). You can read more about this here: + + - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ + - https://element.io/blog/element-to-adopt-agplv3/ + +The Matrix.org Foundation copy of the project will be archived. Any changes needed +by server administrators will be communicated via our usual announcements channels, +but we are striving to make this as seamless as possible. + + +No significant changes since 1.97.0rc1. + + +# Synapse 1.97.0rc1 (2023-11-21) + +### Features + +- Add support for asynchronous uploads as defined by [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246). Contributed by @sumnerevans at @beeper. ([\#15503](https://github.com/matrix-org/synapse/issues/15503)) +- Improve the performance of some operations in multi-worker deployments. ([\#16613](https://github.com/matrix-org/synapse/issues/16613), [\#16616](https://github.com/matrix-org/synapse/issues/16616)) + +### Bugfixes + +- Fix a long-standing bug where some queries updated the same row twice. Introduced in Synapse 1.57.0. ([\#16609](https://github.com/matrix-org/synapse/issues/16609)) +- Fix a long-standing bug where Synapse would not unbind third-party identifiers for Application Service users when deactivated and would not emit a compliant response. ([\#16617](https://github.com/matrix-org/synapse/issues/16617)) +- Fix sending out of order `POSITION` over replication, causing additional database load. ([\#16639](https://github.com/matrix-org/synapse/issues/16639)) + +### Improved Documentation + +- Note that the option [`outbound_federation_restricted_to`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#outbound_federation_restricted_to) was added in Synapse 1.89.0, and fix a nearby formatting error. ([\#16628](https://github.com/matrix-org/synapse/issues/16628)) +- Update parameter information for the `/timestamp_to_event` admin API. ([\#16631](https://github.com/matrix-org/synapse/issues/16631)) +- Provide an example for a common encrypted media response from the admin user media API and mention possible null values. ([\#16654](https://github.com/matrix-org/synapse/issues/16654)) + +### Internal Changes + +- Remove whole table locks on push rule modifications. Contributed by Nick @ Beeper (@fizzadar). ([\#16051](https://github.com/matrix-org/synapse/issues/16051)) +- Support reactor tick timings on more types of event loops. ([\#16532](https://github.com/matrix-org/synapse/issues/16532)) +- Improve type hints. ([\#16564](https://github.com/matrix-org/synapse/issues/16564), [\#16611](https://github.com/matrix-org/synapse/issues/16611), [\#16612](https://github.com/matrix-org/synapse/issues/16612)) +- Avoid executing no-op queries. ([\#16583](https://github.com/matrix-org/synapse/issues/16583)) +- Simplify persistence code to be per-room. ([\#16584](https://github.com/matrix-org/synapse/issues/16584)) +- Use standard SQL helpers in persistence code. ([\#16585](https://github.com/matrix-org/synapse/issues/16585)) +- Avoid updating the stream cache unnecessarily. ([\#16586](https://github.com/matrix-org/synapse/issues/16586)) +- Improve performance when using opentracing. ([\#16589](https://github.com/matrix-org/synapse/issues/16589)) +- Run push rule evaluator setup in parallel. ([\#16590](https://github.com/matrix-org/synapse/issues/16590)) +- Improve tests of the SQL generator. ([\#16596](https://github.com/matrix-org/synapse/issues/16596)) +- Use more generic database methods. ([\#16615](https://github.com/matrix-org/synapse/issues/16615)) +- Use `dbname` instead of the deprecated `database` connection parameter for psycopg2. ([\#16618](https://github.com/matrix-org/synapse/issues/16618)) +- Add an internal [Admin API endpoint](https://matrix-org.github.io/synapse/v1.97/usage/configuration/config_documentation.html#allow-replacing-master-cross-signing-key-without-user-interactive-auth) to temporarily grant the ability to update an existing cross-signing key without UIA. ([\#16634](https://github.com/matrix-org/synapse/issues/16634)) +- Improve references to GitHub issues. ([\#16637](https://github.com/matrix-org/synapse/issues/16637), [\#16638](https://github.com/matrix-org/synapse/issues/16638)) +- More efficiently handle no-op `POSITION` over replication. ([\#16640](https://github.com/matrix-org/synapse/issues/16640), [\#16655](https://github.com/matrix-org/synapse/issues/16655)) +- Speed up deleting of device messages when deleting a device. ([\#16643](https://github.com/matrix-org/synapse/issues/16643)) +- Speed up persisting large number of outliers. ([\#16649](https://github.com/matrix-org/synapse/issues/16649)) +- Reduce max concurrency of background tasks, reducing potential max DB load. ([\#16656](https://github.com/matrix-org/synapse/issues/16656), [\#16660](https://github.com/matrix-org/synapse/issues/16660)) +- Speed up purge room by adding an index to `event_push_summary`. ([\#16657](https://github.com/matrix-org/synapse/issues/16657)) + + + +### Updates to locked dependencies + +* Bump prometheus-client from 0.17.1 to 0.18.0. ([\#16626](https://github.com/matrix-org/synapse/issues/16626)) +* Bump pyicu from 2.11 to 2.12. ([\#16603](https://github.com/matrix-org/synapse/issues/16603)) +* Bump requests-toolbelt from 0.10.1 to 1.0.0. ([\#16659](https://github.com/matrix-org/synapse/issues/16659)) +* Bump ruff from 0.0.292 to 0.1.4. ([\#16600](https://github.com/matrix-org/synapse/issues/16600)) +* Bump serde from 1.0.190 to 1.0.192. ([\#16627](https://github.com/matrix-org/synapse/issues/16627)) +* Bump serde_json from 1.0.107 to 1.0.108. ([\#16604](https://github.com/matrix-org/synapse/issues/16604)) +* Bump setuptools-rust from 1.8.0 to 1.8.1. ([\#16601](https://github.com/matrix-org/synapse/issues/16601)) +* Bump towncrier from 23.6.0 to 23.11.0. ([\#16622](https://github.com/matrix-org/synapse/issues/16622)) +* Bump treq from 22.2.0 to 23.11.0. ([\#16623](https://github.com/matrix-org/synapse/issues/16623)) +* Bump twisted from 23.8.0 to 23.10.0. ([\#16588](https://github.com/matrix-org/synapse/issues/16588)) +* Bump types-bleach from 6.1.0.0 to 6.1.0.1. ([\#16624](https://github.com/matrix-org/synapse/issues/16624)) +* Bump types-jsonschema from 4.19.0.3 to 4.19.0.4. ([\#16599](https://github.com/matrix-org/synapse/issues/16599)) +* Bump types-pyopenssl from 23.2.0.2 to 23.3.0.0. ([\#16625](https://github.com/matrix-org/synapse/issues/16625)) +* Bump types-pyyaml from 6.0.12.11 to 6.0.12.12. ([\#16602](https://github.com/matrix-org/synapse/issues/16602)) + +# Synapse 1.96.1 (2023-11-17) + +Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for +proprietary dual licensing). You can read more about this here: + +* https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ +* https://element.io/blog/element-to-adopt-agplv3/ + +The Matrix.org Foundation copy of the project will be archived. Any changes needed +by server administrators will be communicated via our usual +[announcements channels](https://matrix.to/#/#homeowners:matrix.org), but we are +striving to make this as seamless as possible. + +This minor release was needed only because of CI-related trouble on [v1.96.0](https://github.com/matrix-org/synapse/releases/tag/v1.96.0), which was never released. + +### Internal Changes + +- Fix building of wheels in CI. ([\#16653](https://github.com/matrix-org/synapse/issues/16653)) + +# Synapse 1.96.0 (2023-11-16) + +### Bugfixes + +- Fix "'int' object is not iterable" error in `set_device_id_for_pushers` background update introduced in Synapse 1.95.0. ([\#16594](https://github.com/matrix-org/synapse/issues/16594)) + +# Synapse 1.96.0rc1 (2023-10-31) + +### Features + +- Add experimental support to allow multiple workers to write to receipts stream. ([\#16432](https://github.com/matrix-org/synapse/issues/16432)) +- Add a new module API for controller presence. ([\#16544](https://github.com/matrix-org/synapse/issues/16544)) +- Add a new module API callback that allows adding extra fields to events' unsigned section when sent down to clients. ([\#16549](https://github.com/matrix-org/synapse/issues/16549)) +- Improve the performance of claiming encryption keys. ([\#16565](https://github.com/matrix-org/synapse/issues/16565), [\#16570](https://github.com/matrix-org/synapse/issues/16570)) + +### Bugfixes + +- Fixed a bug in the example Grafana dashboard that prevents it from finding the correct datasource. Contributed by @MichaelSasser. ([\#16471](https://github.com/matrix-org/synapse/issues/16471)) +- Fix a long-standing, exceedingly rare edge case where the first event persisted by a new event persister worker might not be sent down `/sync`. ([\#16473](https://github.com/matrix-org/synapse/issues/16473), [\#16557](https://github.com/matrix-org/synapse/issues/16557), [\#16561](https://github.com/matrix-org/synapse/issues/16561), [\#16578](https://github.com/matrix-org/synapse/issues/16578), [\#16580](https://github.com/matrix-org/synapse/issues/16580)) +- Fix long-standing bug where `/sync` incorrectly did not mark a room as `limited` in a sync requests when there were missing remote events. ([\#16485](https://github.com/matrix-org/synapse/issues/16485)) +- Fix a bug introduced in Synapse 1.41 where HTTP(S) forward proxy authorization would fail when using basic HTTP authentication with a long `username:password` string. ([\#16504](https://github.com/matrix-org/synapse/issues/16504)) +- Force TLS certificate verification in user registration script. ([\#16530](https://github.com/matrix-org/synapse/issues/16530)) +- Fix long-standing bug where `/sync` could tightloop after restart when using SQLite. ([\#16540](https://github.com/matrix-org/synapse/issues/16540)) +- Fix ratelimiting of message sending when using workers, where the ratelimit would only be applied after most of the work has been done. ([\#16558](https://github.com/matrix-org/synapse/issues/16558)) +- Fix a long-standing bug where invited/knocking users would not leave during a room purge. ([\#16559](https://github.com/matrix-org/synapse/issues/16559)) + +### Improved Documentation + +- Improve documentation of presence router. ([\#16529](https://github.com/matrix-org/synapse/issues/16529)) +- Add a sentence to the [opentracing docs](https://matrix-org.github.io/synapse/latest/opentracing.html) on how you can have jaeger in a different place than synapse. ([\#16531](https://github.com/matrix-org/synapse/issues/16531)) +- Correctly describe the meaning of unspecified rule lists in the [`alias_creation_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#alias_creation_rules) and [`room_list_publication_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#room_list_publication_rules) config options and improve their descriptions more generally. ([\#16541](https://github.com/matrix-org/synapse/issues/16541)) +- Pin the recommended poetry version in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16550](https://github.com/matrix-org/synapse/issues/16550)) +- Fix a broken link to the [client breakdown](https://matrix.org/ecosystem/clients/) in the README. ([\#16569](https://github.com/matrix-org/synapse/issues/16569)) + +### Internal Changes + +- Improve performance of delete device messages query, cf issue [16479](https://github.com/matrix-org/synapse/issues/16479). ([\#16492](https://github.com/matrix-org/synapse/issues/16492)) +- Reduce memory allocations. ([\#16505](https://github.com/matrix-org/synapse/issues/16505)) +- Improve replication performance when purging rooms. ([\#16510](https://github.com/matrix-org/synapse/issues/16510)) +- Run tests against Python 3.12. ([\#16511](https://github.com/matrix-org/synapse/issues/16511)) +- Run trial & integration tests in continuous integration when `.ci` directory is modified. ([\#16512](https://github.com/matrix-org/synapse/issues/16512)) +- Remove duplicate call to mark remote server 'awake' when using a federation sending worker. ([\#16515](https://github.com/matrix-org/synapse/issues/16515)) +- Enable dirty runs on Complement CI, which is significantly faster. ([\#16520](https://github.com/matrix-org/synapse/issues/16520)) +- Stop deleting from an unused table. ([\#16521](https://github.com/matrix-org/synapse/issues/16521)) +- Improve type hints. ([\#16526](https://github.com/matrix-org/synapse/issues/16526), [\#16551](https://github.com/matrix-org/synapse/issues/16551)) +- Fix running unit tests on Twisted trunk. ([\#16528](https://github.com/matrix-org/synapse/issues/16528)) +- Reduce some spurious logging in worker mode. ([\#16555](https://github.com/matrix-org/synapse/issues/16555)) +- Stop porting a table in port db that we're going to nuke and rebuild anyway. ([\#16563](https://github.com/matrix-org/synapse/issues/16563)) +- Deal with warnings from running complement in CI. ([\#16567](https://github.com/matrix-org/synapse/issues/16567)) +- Allow building with `setuptools_rust` 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574)) + +### Updates to locked dependencies + +* Bump black from 23.10.0 to 23.10.1. ([\#16575](https://github.com/matrix-org/synapse/issues/16575)) +* Bump black from 23.9.1 to 23.10.0. ([\#16538](https://github.com/matrix-org/synapse/issues/16538)) +* Bump cryptography from 41.0.4 to 41.0.5. ([\#16572](https://github.com/matrix-org/synapse/issues/16572)) +* Bump gitpython from 3.1.37 to 3.1.40. ([\#16534](https://github.com/matrix-org/synapse/issues/16534)) +* Bump phonenumbers from 8.13.22 to 8.13.23. ([\#16576](https://github.com/matrix-org/synapse/issues/16576)) +* Bump pygithub from 1.59.1 to 2.1.1. ([\#16535](https://github.com/matrix-org/synapse/issues/16535)) +- Bump matrix-synapse-ldap3 from 0.2.2 to 0.3.0. ([\#16539](https://github.com/matrix-org/synapse/issues/16539)) +* Bump serde from 1.0.189 to 1.0.190. ([\#16577](https://github.com/matrix-org/synapse/issues/16577)) +* Bump setuptools-rust from 1.7.0 to 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574)) +* Bump types-pillow from 10.0.0.3 to 10.1.0.0. ([\#16536](https://github.com/matrix-org/synapse/issues/16536)) +* Bump types-psycopg2 from 2.9.21.14 to 2.9.21.15. ([\#16573](https://github.com/matrix-org/synapse/issues/16573)) +* Bump types-requests from 2.31.0.2 to 2.31.0.10. ([\#16537](https://github.com/matrix-org/synapse/issues/16537)) +* Bump urllib3 from 1.26.17 to 1.26.18. ([\#16516](https://github.com/matrix-org/synapse/issues/16516)) + +# Synapse 1.95.1 (2023-10-31) + +## Security advisory + +The following issue is fixed in 1.95.1. + +- [GHSA-mp92-3jfm-3575](https://github.com/matrix-org/synapse/security/advisories/GHSA-mp92-3jfm-3575) / [CVE-2023-43796](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-43796) — Moderate Severity + + Cached device information of remote users can be queried from Synapse. This can be used to enumerate the remote users known to a homeserver. + +See the advisory for more details. If you have any questions, email security@matrix.org. + + + +# Synapse 1.95.0 (2023-10-24) + +### Internal Changes + +- Build Debian packages for [Ubuntu 23.10 Mantic Minotaur](https://canonical.com/blog/canonical-releases-ubuntu-23-10-mantic-minotaur). ([\#16524](https://github.com/matrix-org/synapse/issues/16524)) + + +# Synapse 1.95.0rc1 (2023-10-17) + +### Bugfixes + +- Remove legacy unspecced `knock_state_events` field returned in some responses. ([\#16403](https://github.com/matrix-org/synapse/issues/16403)) +- Fix a bug introduced in Synapse 1.81.0 where an `AttributeError` would be raised when `_matrix/client/v3/account/whoami` is called over a unix socket. Contributed by @Sir-Photch. ([\#16404](https://github.com/matrix-org/synapse/issues/16404)) +- Properly return inline media when content types have parameters. ([\#16440](https://github.com/matrix-org/synapse/issues/16440)) +- Prevent the purging of large rooms from timing out when Postgres is in use. The timeout which causes this issue was introduced in Synapse 1.88.0. ([\#16455](https://github.com/matrix-org/synapse/issues/16455)) +- Improve the performance of purging rooms, particularly encrypted rooms. ([\#16457](https://github.com/matrix-org/synapse/issues/16457)) +- Fix a bug introduced in Synapse 1.59.0 where servers could be incorrectly marked as available after an error response was received. ([\#16506](https://github.com/matrix-org/synapse/issues/16506)) + +### Improved Documentation + +- Document internal background update mechanism. ([\#16420](https://github.com/matrix-org/synapse/issues/16420)) +- Fix a typo in the sql for [useful SQL for admins document](https://matrix-org.github.io/synapse/latest/usage/administration/useful_sql_for_admins.html). ([\#16477](https://github.com/matrix-org/synapse/issues/16477)) + +### Internal Changes + +- Bump pyo3 from 0.17.1 to 0.19.2. ([\#16162](https://github.com/matrix-org/synapse/issues/16162)) +- Update registration of media repository URLs. ([\#16419](https://github.com/matrix-org/synapse/issues/16419)) +- Improve type hints. ([\#16421](https://github.com/matrix-org/synapse/issues/16421), [\#16468](https://github.com/matrix-org/synapse/issues/16468), [\#16469](https://github.com/matrix-org/synapse/issues/16469), [\#16507](https://github.com/matrix-org/synapse/issues/16507)) +- Refactor some code to simplify and better type receipts stream adjacent code. ([\#16426](https://github.com/matrix-org/synapse/issues/16426)) +- Factor out `MultiWriter` token from `RoomStreamToken`. ([\#16427](https://github.com/matrix-org/synapse/issues/16427)) +- Improve code comments. ([\#16428](https://github.com/matrix-org/synapse/issues/16428)) +- Reduce memory allocations. ([\#16429](https://github.com/matrix-org/synapse/issues/16429), [\#16431](https://github.com/matrix-org/synapse/issues/16431), [\#16433](https://github.com/matrix-org/synapse/issues/16433), [\#16434](https://github.com/matrix-org/synapse/issues/16434), [\#16438](https://github.com/matrix-org/synapse/issues/16438), [\#16444](https://github.com/matrix-org/synapse/issues/16444)) +- Remove unused method. ([\#16435](https://github.com/matrix-org/synapse/issues/16435)) +- Improve rate limiting logic. ([\#16441](https://github.com/matrix-org/synapse/issues/16441)) +- Do not block running of CI behind the check for sign-off on PRs. ([\#16454](https://github.com/matrix-org/synapse/issues/16454)) +- Update the release script to remind releaser to check for special release notes. ([\#16461](https://github.com/matrix-org/synapse/issues/16461)) +- Update complement.sh to match new public API shape. ([\#16466](https://github.com/matrix-org/synapse/issues/16466)) +- Clean up logging on event persister endpoints. ([\#16488](https://github.com/matrix-org/synapse/issues/16488)) +- Remove useless async job to delete device messages on sync, since we only deliver (and hence delete) up to 100 device messages at a time. ([\#16491](https://github.com/matrix-org/synapse/issues/16491)) + +### Updates to locked dependencies + +* Bump bleach from 6.0.0 to 6.1.0. ([\#16451](https://github.com/matrix-org/synapse/issues/16451)) +* Bump jsonschema from 4.19.0 to 4.19.1. ([\#16500](https://github.com/matrix-org/synapse/issues/16500)) +* Bump netaddr from 0.8.0 to 0.9.0. ([\#16453](https://github.com/matrix-org/synapse/issues/16453)) +* Bump packaging from 23.1 to 23.2. ([\#16497](https://github.com/matrix-org/synapse/issues/16497)) +* Bump pillow from 10.0.1 to 10.1.0. ([\#16498](https://github.com/matrix-org/synapse/issues/16498)) +* Bump psycopg2 from 2.9.8 to 2.9.9. ([\#16452](https://github.com/matrix-org/synapse/issues/16452)) +* Bump pyo3-log from 0.8.3 to 0.8.4. ([\#16495](https://github.com/matrix-org/synapse/issues/16495)) +* Bump ruff from 0.0.290 to 0.0.292. ([\#16449](https://github.com/matrix-org/synapse/issues/16449)) +* Bump sentry-sdk from 1.31.0 to 1.32.0. ([\#16496](https://github.com/matrix-org/synapse/issues/16496)) +* Bump serde from 1.0.188 to 1.0.189. ([\#16494](https://github.com/matrix-org/synapse/issues/16494)) +* Bump types-bleach from 6.0.0.4 to 6.1.0.0. ([\#16450](https://github.com/matrix-org/synapse/issues/16450)) +* Bump types-jsonschema from 4.17.0.10 to 4.19.0.3. ([\#16499](https://github.com/matrix-org/synapse/issues/16499)) + +# Synapse 1.94.0 (2023-10-10) + +No significant changes since 1.94.0rc1. +However, please take note of the security advisory that follows. + +## Security advisory + +The following issue is fixed in 1.94.0 (and RC). + +- [GHSA-5chr-wjw5-3gq4](https://github.com/matrix-org/synapse/security/advisories/GHSA-5chr-wjw5-3gq4) / [CVE-2023-45129](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-45129) — Moderate Severity + + A malicious server ACL event can impact performance temporarily or permanently leading to a persistent denial of service. + + Homeservers running on a closed federation (which presumably do not need to use server ACLs) are not affected. + +See the advisory for more details. If you have any questions, email security@matrix.org. + + +# Synapse 1.94.0rc1 (2023-10-03) + +### Features + +- Render plain, CSS, CSV, JSON and common image formats in the browser (inline) when requested through the /download endpoint. ([\#15988](https://github.com/matrix-org/synapse/issues/15988)) +- Add experimental support for [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028) to push all encrypted events to clients. ([\#16361](https://github.com/matrix-org/synapse/issues/16361)) +- Minor performance improvement when sending presence to federated servers. ([\#16385](https://github.com/matrix-org/synapse/issues/16385)) +- Minor performance improvement by caching server ACL checking. ([\#16360](https://github.com/matrix-org/synapse/issues/16360)) + +### Improved Documentation + +- Add developer documentation concerning gradual schema migrations with column alterations. ([\#15691](https://github.com/matrix-org/synapse/issues/15691)) +- Improve documentation of the user directory search algorithm. ([\#16320](https://github.com/matrix-org/synapse/issues/16320)) +- Fix rendering of user admin API documentation around deactivation. This was broken in Synapse 1.91.0. ([\#16355](https://github.com/matrix-org/synapse/issues/16355)) +- Update documentation around message retention policies. ([\#16382](https://github.com/matrix-org/synapse/issues/16382)) +- Add note to `federation_domain_whitelist` config option to clarify its usage. ([\#16416](https://github.com/matrix-org/synapse/issues/16416)) +- Improve legacy release notes. ([\#16418](https://github.com/matrix-org/synapse/issues/16418)) + +### Deprecations and Removals + +- Remove Python version from `/_synapse/admin/v1/server_version`. ([\#16380](https://github.com/matrix-org/synapse/issues/16380)) + +### Internal Changes + +- Avoid running CI steps when the files they check have not been changed. ([\#14745](https://github.com/matrix-org/synapse/issues/14745), [\#16387](https://github.com/matrix-org/synapse/issues/16387)) +- Improve type hints. ([\#14911](https://github.com/matrix-org/synapse/issues/14911), [\#16350](https://github.com/matrix-org/synapse/issues/16350), [\#16356](https://github.com/matrix-org/synapse/issues/16356), [\#16395](https://github.com/matrix-org/synapse/issues/16395)) +- Added support for pydantic v2 in addition to pydantic v1. Contributed by Maxwell G (@gotmax23). ([\#16332](https://github.com/matrix-org/synapse/issues/16332)) +- Get CI to check PRs have been signed-off. ([\#16348](https://github.com/matrix-org/synapse/issues/16348)) +- Add missing licence header. ([\#16359](https://github.com/matrix-org/synapse/issues/16359)) +- Improve type hints, and bump types-psycopg2 from 2.9.21.11 to 2.9.21.14. ([\#16381](https://github.com/matrix-org/synapse/issues/16381)) +- Improve comments in `StateGroupBackgroundUpdateStore`. ([\#16383](https://github.com/matrix-org/synapse/issues/16383)) +- Update maturin configuration. ([\#16394](https://github.com/matrix-org/synapse/issues/16394)) +- Downgrade replication stream time out error log lines to warning. ([\#16401](https://github.com/matrix-org/synapse/issues/16401)) + +### Updates to locked dependencies + +* Bump actions/checkout from 3 to 4. ([\#16250](https://github.com/matrix-org/synapse/issues/16250)) +* Bump cryptography from 41.0.3 to 41.0.4. ([\#16362](https://github.com/matrix-org/synapse/issues/16362)) +* Bump dawidd6/action-download-artifact from 2.27.0 to 2.28.0. ([\#16374](https://github.com/matrix-org/synapse/issues/16374)) +* Bump docker/setup-buildx-action from 2 to 3. ([\#16375](https://github.com/matrix-org/synapse/issues/16375)) +* Bump gitpython from 3.1.35 to 3.1.37. ([\#16376](https://github.com/matrix-org/synapse/issues/16376)) +* Bump msgpack from 1.0.5 to 1.0.6. ([\#16377](https://github.com/matrix-org/synapse/issues/16377)) +* Bump msgpack from 1.0.6 to 1.0.7. ([\#16412](https://github.com/matrix-org/synapse/issues/16412)) +* Bump phonenumbers from 8.13.19 to 8.13.22. ([\#16413](https://github.com/matrix-org/synapse/issues/16413)) +* Bump psycopg2 from 2.9.7 to 2.9.8. ([\#16409](https://github.com/matrix-org/synapse/issues/16409)) +* Bump pydantic from 2.3.0 to 2.4.2. ([\#16410](https://github.com/matrix-org/synapse/issues/16410)) +* Bump regex from 1.9.5 to 1.9.6. ([\#16408](https://github.com/matrix-org/synapse/issues/16408)) +* Bump sentry-sdk from 1.30.0 to 1.31.0. ([\#16378](https://github.com/matrix-org/synapse/issues/16378)) +* Bump types-netaddr from 0.8.0.9 to 0.9.0.1. ([\#16411](https://github.com/matrix-org/synapse/issues/16411)) +* Bump types-psycopg2 from 2.9.21.11 to 2.9.21.14. ([\#16381](https://github.com/matrix-org/synapse/issues/16381)) +* Bump urllib3 from 1.26.15 to 1.26.17. ([\#16422](https://github.com/matrix-org/synapse/issues/16422)) + +# Synapse 1.93.0 (2023-09-26) + +No significant changes since 1.93.0rc1. + + +## Security advisory + +The following issues are fixed in 1.93.0 (and RCs). + +- [GHSA-4f74-84v3-j9q5](https://github.com/matrix-org/synapse/security/advisories/GHSA-4f74-84v3-j9q5) / [CVE-2023-41335](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-41335) — Low Severity + + Temporary storage of plaintext passwords during password changes. + +- [GHSA-7565-cq32-vx2x](https://github.com/matrix-org/synapse/security/advisories/GHSA-7565-cq32-vx2x) / [CVE-2023-42453](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-42453) — Low Severity + + Improper validation of receipts allows forged read receipts. + +See the advisories for more details. If you have any questions, email security@matrix.org. + + +# Synapse 1.93.0rc1 (2023-09-19) + +### Features + +- Add automatic purge after all users have forgotten a room. ([\#15488](https://github.com/matrix-org/synapse/issues/15488)) +- Restore room purge/shutdown after a Synapse restart. ([\#15488](https://github.com/matrix-org/synapse/issues/15488)) +- Support resolving homeservers using `matrix-fed` DNS SRV records from [MSC4040](https://github.com/matrix-org/matrix-spec-proposals/pull/4040). ([\#16137](https://github.com/matrix-org/synapse/issues/16137)) +- Add the ability to use `G` (GiB) and `T` (TiB) suffixes in configuration options that refer to numbers of bytes. ([\#16219](https://github.com/matrix-org/synapse/issues/16219)) +- Add span information to requests sent to appservices. Contributed by MTRNord. ([\#16227](https://github.com/matrix-org/synapse/issues/16227)) +- Add the ability to enable/disable registrations when using CAS. Contributed by Aurélien Grimpard. ([\#16262](https://github.com/matrix-org/synapse/issues/16262)) +- Allow the `/notifications` endpoint to be routed to workers. ([\#16265](https://github.com/matrix-org/synapse/issues/16265)) +- Enable users to easily unsubscribe to notifications emails via the `List-Unsubscribe` header. ([\#16274](https://github.com/matrix-org/synapse/issues/16274)) +- Report whether a user is `locked` in the [List Accounts admin API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html#list-accounts), and exclude locked users by default. ([\#16328](https://github.com/matrix-org/synapse/issues/16328)) + +### Bugfixes + +- Fix a long-standing bug where multi-device accounts could cause high load due to presence. ([\#16066](https://github.com/matrix-org/synapse/issues/16066), [\#16170](https://github.com/matrix-org/synapse/issues/16170), [\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172), [\#16174](https://github.com/matrix-org/synapse/issues/16174)) +- Fix a long-standing bug where appservices using [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409) to receive `to_device` messages would only get messages for one user. ([\#16251](https://github.com/matrix-org/synapse/issues/16251)) +- Fix bug when using workers where Synapse could end up re-requesting the same remote device repeatedly. ([\#16252](https://github.com/matrix-org/synapse/issues/16252)) +- Fix long-standing bug where we kept re-requesting a remote server's key repeatedly, potentially causing delays in receiving events over federation. ([\#16257](https://github.com/matrix-org/synapse/issues/16257)) +- Avoid temporary storage of sensitive information. ([\#16272](https://github.com/matrix-org/synapse/issues/16272)) +- Fix bug introduced in Synapse 1.49.0 when using dehydrated devices ([MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697)) and refresh tokens. Contributed by Hanadi. ([\#16288](https://github.com/matrix-org/synapse/issues/16288)) +- Fix a long-standing bug where invalid receipts would be accepted. ([\#16327](https://github.com/matrix-org/synapse/issues/16327)) +- Use standard name for UTF-8 charset in emails. ([\#16329](https://github.com/matrix-org/synapse/issues/16329)) +- Don't try refetching device lists for users on remote hosts that are marked as "down". ([\#16298](https://github.com/matrix-org/synapse/issues/16298)) + +### Improved Documentation + +- Fix typos in the documentation. ([\#16282](https://github.com/matrix-org/synapse/issues/16282)) +- Link to the Alpine Linux community package for Synapse. ([\#16304](https://github.com/matrix-org/synapse/issues/16304)) +- Use string for `federation_client_minimum_tls_version` documentation examples. Contributed by @jcgruenhage. ([\#16353](https://github.com/matrix-org/synapse/issues/16353)) + +### Internal Changes + +- Allow modules to delete rooms. ([\#15997](https://github.com/matrix-org/synapse/issues/15997)) +- Add GCC and GNU Make to the Nix flake development environment so that `ruff` can be compiled. ([\#16090](https://github.com/matrix-org/synapse/issues/16090), [\#16263](https://github.com/matrix-org/synapse/issues/16263)) +- Fix type checking when using the new version of Twisted. ([\#16235](https://github.com/matrix-org/synapse/issues/16235)) +- Delete device messages asynchronously and in staged batches using the task scheduler. ([\#16240](https://github.com/matrix-org/synapse/issues/16240), [\#16311](https://github.com/matrix-org/synapse/issues/16311), [\#16312](https://github.com/matrix-org/synapse/issues/16312), [\#16313](https://github.com/matrix-org/synapse/issues/16313)) +- Bump minimum supported Rust version to 1.61.0. ([\#16248](https://github.com/matrix-org/synapse/issues/16248)) +- Update rust to version 1.71.1 in the nix development environment. ([\#16260](https://github.com/matrix-org/synapse/issues/16260)) +- Simplify server key storage. ([\#16261](https://github.com/matrix-org/synapse/issues/16261)) +- Reduce CPU overhead of change password endpoint. ([\#16264](https://github.com/matrix-org/synapse/issues/16264)) +- Stop purging from tables slated for removal. ([\#16273](https://github.com/matrix-org/synapse/issues/16273)) +- Improve type hints. ([\#16276](https://github.com/matrix-org/synapse/issues/16276), [\#16301](https://github.com/matrix-org/synapse/issues/16301), [\#16325](https://github.com/matrix-org/synapse/issues/16325), [\#16326](https://github.com/matrix-org/synapse/issues/16326)) +- Raise `setuptools_rust` version cap to 1.7.0. ([\#16277](https://github.com/matrix-org/synapse/issues/16277)) +- Fix using the new task scheduler causing lots of CPU to be used. ([\#16278](https://github.com/matrix-org/synapse/issues/16278)) +- Upgrade CI run of Python 3.12 from rc1 to rc2. ([\#16280](https://github.com/matrix-org/synapse/issues/16280)) +- Include values in SQL debug when using `execute_values` with Postgres. ([\#16281](https://github.com/matrix-org/synapse/issues/16281)) +- Enable additional linting checks. ([\#16283](https://github.com/matrix-org/synapse/issues/16283)) +- Refactor `receipts_graph` Postgres transactions to stop error messages. ([\#16299](https://github.com/matrix-org/synapse/issues/16299)) +- Small improvements to logging in replication code. ([\#16309](https://github.com/matrix-org/synapse/issues/16309)) +- Remove a reference cycle in background processes. ([\#16314](https://github.com/matrix-org/synapse/issues/16314)) +- Only use literal strings for background process names. ([\#16315](https://github.com/matrix-org/synapse/issues/16315)) +- Refactor `get_user_by_id`. ([\#16316](https://github.com/matrix-org/synapse/issues/16316)) +- Speed up task to delete to-device messages. ([\#16318](https://github.com/matrix-org/synapse/issues/16318)) +- Avoid patching code in tests. ([\#16349](https://github.com/matrix-org/synapse/issues/16349)) +- Test against PostgreSQL 16. ([\#16351](https://github.com/matrix-org/synapse/issues/16351)) + +### Updates to locked dependencies + +* Bump mypy from 1.4.1 to 1.5.1. ([\#16300](https://github.com/matrix-org/synapse/issues/16300)) +* Bump black from 23.7.0 to 23.9.1. ([\#16295](https://github.com/matrix-org/synapse/issues/16295)) +* Bump docker/build-push-action from 4 to 5. ([\#16336](https://github.com/matrix-org/synapse/issues/16336)) +* Bump docker/login-action from 2 to 3. ([\#16339](https://github.com/matrix-org/synapse/issues/16339)) +* Bump docker/metadata-action from 4 to 5. ([\#16337](https://github.com/matrix-org/synapse/issues/16337)) +* Bump docker/setup-qemu-action from 2 to 3. ([\#16338](https://github.com/matrix-org/synapse/issues/16338)) +* Bump furo from 2023.8.19 to 2023.9.10. ([\#16340](https://github.com/matrix-org/synapse/issues/16340)) +* Bump gitpython from 3.1.32 to 3.1.35. ([\#16267](https://github.com/matrix-org/synapse/issues/16267), [\#16279](https://github.com/matrix-org/synapse/issues/16279)) +* Bump mypy-zope from 1.0.0 to 1.0.1. ([\#16291](https://github.com/matrix-org/synapse/issues/16291)) +* Bump pillow from 10.0.0 to 10.0.1. ([\#16344](https://github.com/matrix-org/synapse/issues/16344)) +* Bump regex from 1.9.4 to 1.9.5. ([\#16233](https://github.com/matrix-org/synapse/issues/16233)) +* Bump ruff from 0.0.286 to 0.0.290. ([\#16342](https://github.com/matrix-org/synapse/issues/16342)) +* Bump serde_json from 1.0.105 to 1.0.107. ([\#16296](https://github.com/matrix-org/synapse/issues/16296), [\#16345](https://github.com/matrix-org/synapse/issues/16345)) +* Bump twisted from 22.10.0 to 23.8.0. ([\#16235](https://github.com/matrix-org/synapse/issues/16235)) +* Bump types-pillow from 10.0.0.2 to 10.0.0.3. ([\#16293](https://github.com/matrix-org/synapse/issues/16293)) +* Bump types-setuptools from 68.0.0.3 to 68.2.0.0. ([\#16292](https://github.com/matrix-org/synapse/issues/16292)) +* Bump typing-extensions from 4.7.1 to 4.8.0. ([\#16341](https://github.com/matrix-org/synapse/issues/16341)) + +# Synapse 1.92.3 (2023-09-18) + +This is again a security update targeted at mitigating [CVE-2023-4863](https://cve.org/CVERecord?id=CVE-2023-4863). +It turns out that libwebp is bundled statically in Pillow wheels so we need to update this dependency instead of +libwebp package at the OS level. + +Unlike what was advertised in 1.92.2 changelog this release also impacts PyPI wheels and Debian packages from matrix.org. + +We encourage admins to upgrade as soon as possible. + + +### Internal Changes + +- Pillow 10.0.1 is now mandatory because of libwebp CVE-2023-4863, since Pillow provides libwebp in the wheels. ([\#16347](https://github.com/matrix-org/synapse/issues/16347)) + +### Updates to locked dependencies + +* Bump pillow from 10.0.0 to 10.0.1. ([\#16344](https://github.com/matrix-org/synapse/issues/16344)) + +# Synapse 1.92.2 (2023-09-15) + +This is a Docker-only update to mitigate [CVE-2023-4863](https://cve.org/CVERecord?id=CVE-2023-4863), a critical vulnerability in `libwebp`. Server admins not using Docker should ensure that their `libwebp` is up to date (if installed). We encourage admins to upgrade as soon as possible. + + +### Updates to the Docker image + +- Update docker image to use Debian bookworm as the base. ([\#16324](https://github.com/matrix-org/synapse/issues/16324)) + + +# Synapse 1.92.1 (2023-09-12) + +This minor release was needed only because of CI-related trouble on [v1.92.0](https://github.com/matrix-org/synapse/releases/tag/v1.92.0), which was never released. + +### Internal Changes + +- Stop building Ubuntu Kinetic since it is EOL and repos seem to be dead. + + +# Synapse 1.92.0 (2023-09-12) + +This release includes the same [bugfix](https://github.com/matrix-org/synapse/issues/16258) as Synapse 1.91.2. + +This version was never released following a CI build failure, cf [v1.92.1 changelog](https://github.com/matrix-org/synapse/releases/tag/v1.92.1). + +### Bugfixes + +- Revert [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) introspection cache, admin impersonation and account lock. ([\#16258](https://github.com/matrix-org/synapse/issues/16258)) + +### Internal Changes + +- Fix incorrect docstring for `Ratelimiter`. ([\#16255](https://github.com/matrix-org/synapse/issues/16255)) +- Update the release script to work on macOS. ([\#16266](https://github.com/matrix-org/synapse/issues/16266)) + + +# Synapse 1.91.2 (2023-09-06) + +### Bugfixes + +- Revert [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) introspection cache, admin impersonation and account lock. ([\#16258](https://github.com/matrix-org/synapse/issues/16258)) + + +# Synapse 1.92.0rc1 (2023-09-05) + +### Features + +- Add configuration setting for CAS protocol version. Contributed by Aurélien Grimpard. ([\#15816](https://github.com/matrix-org/synapse/issues/15816)) +- Suppress notifications from message edits per [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958). ([\#16113](https://github.com/matrix-org/synapse/issues/16113)) +- Experimental support for [MSC4041](https://github.com/matrix-org/matrix-spec-proposals/pull/4041): return a `Retry-After` header with `M_LIMIT_EXCEEDED` error responses. ([\#16136](https://github.com/matrix-org/synapse/issues/16136)) +- Add `last_seen_ts` to the [admin users API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html). ([\#16218](https://github.com/matrix-org/synapse/issues/16218)) +- Improve resource usage when sending data to a large number of remote hosts that are marked as "down". ([\#16223](https://github.com/matrix-org/synapse/issues/16223)) + +### Bugfixes + +- Fix IPv6-related bugs on SMTP settings, adding groundwork to fix similar issues. Contributed by @evilham and @telmich (ungleich.ch). ([\#16155](https://github.com/matrix-org/synapse/issues/16155)) +- Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `include_all_networks` as a string. ([\#16185](https://github.com/matrix-org/synapse/issues/16185)) +- Fix inaccurate error message while attempting to ban or unban a user with the same or higher PL by spliting the conditional statements. Contributed by @leviosacz. ([\#16205](https://github.com/matrix-org/synapse/issues/16205)) +- Fix a rare bug that broke looping calls, which could lead to e.g. linearly increasing memory usage. Introduced in v1.90.0. ([\#16210](https://github.com/matrix-org/synapse/issues/16210)) +- Fix a long-standing bug where uploading images would fail if we could not generate thumbnails for them. ([\#16211](https://github.com/matrix-org/synapse/issues/16211)) +- Fix a long-standing bug where we did not correctly back off from servers that had "gone" if they returned 4xx series error codes. ([\#16221](https://github.com/matrix-org/synapse/issues/16221)) + +### Improved Documentation + +- Update links to the [matrix.org blog](https://matrix.org/blog/). ([\#16008](https://github.com/matrix-org/synapse/issues/16008)) +- Document which [admin APIs](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) are disabled when experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support is enabled. ([\#16168](https://github.com/matrix-org/synapse/issues/16168)) +- Document [`exclude_rooms_from_sync`](https://matrix-org.github.io/synapse/v1.92/usage/configuration/config_documentation.html#exclude_rooms_from_sync) configuration option. ([\#16178](https://github.com/matrix-org/synapse/issues/16178)) + +### Internal Changes + +- Prepare unit tests for Python 3.12. ([\#16099](https://github.com/matrix-org/synapse/issues/16099)) +- Fix nightly CI jobs. ([\#16121](https://github.com/matrix-org/synapse/issues/16121), [\#16213](https://github.com/matrix-org/synapse/issues/16213)) +- Describe which rate limiter was hit in logs. ([\#16135](https://github.com/matrix-org/synapse/issues/16135)) +- Simplify presence code when using workers. ([\#16170](https://github.com/matrix-org/synapse/issues/16170)) +- Track per-device information in the presence code. ([\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172)) +- Stop using the `event_txn_id` table. ([\#16175](https://github.com/matrix-org/synapse/issues/16175)) +- Use `AsyncMock` instead of custom code. ([\#16179](https://github.com/matrix-org/synapse/issues/16179), [\#16180](https://github.com/matrix-org/synapse/issues/16180)) +- Improve error reporting of invalid data passed to `/_matrix/key/v2/query`. ([\#16183](https://github.com/matrix-org/synapse/issues/16183)) +- Task scheduler: add replication notify for new task to launch ASAP. ([\#16184](https://github.com/matrix-org/synapse/issues/16184)) +- Improve type hints. ([\#16186](https://github.com/matrix-org/synapse/issues/16186), [\#16188](https://github.com/matrix-org/synapse/issues/16188), [\#16201](https://github.com/matrix-org/synapse/issues/16201)) +- Bump black version to 23.7.0. ([\#16187](https://github.com/matrix-org/synapse/issues/16187)) +- Log the details of background update failures. ([\#16212](https://github.com/matrix-org/synapse/issues/16212)) +- Cache device resync requests over replication. ([\#16241](https://github.com/matrix-org/synapse/issues/16241)) + +### Updates to locked dependencies + +* Bump anyhow from 1.0.72 to 1.0.75. ([\#16141](https://github.com/matrix-org/synapse/issues/16141)) +* Bump furo from 2023.7.26 to 2023.8.19. ([\#16238](https://github.com/matrix-org/synapse/issues/16238)) +* Bump phonenumbers from 8.13.18 to 8.13.19. ([\#16237](https://github.com/matrix-org/synapse/issues/16237)) +* Bump psycopg2 from 2.9.6 to 2.9.7. ([\#16196](https://github.com/matrix-org/synapse/issues/16196)) +* Bump regex from 1.9.3 to 1.9.4. ([\#16195](https://github.com/matrix-org/synapse/issues/16195)) +* Bump ruff from 0.0.277 to 0.0.286. ([\#16198](https://github.com/matrix-org/synapse/issues/16198)) +* Bump sentry-sdk from 1.29.2 to 1.30.0. ([\#16236](https://github.com/matrix-org/synapse/issues/16236)) +* Bump serde from 1.0.184 to 1.0.188. ([\#16194](https://github.com/matrix-org/synapse/issues/16194)) +* Bump serde_json from 1.0.104 to 1.0.105. ([\#16140](https://github.com/matrix-org/synapse/issues/16140)) +* Bump types-psycopg2 from 2.9.21.10 to 2.9.21.11. ([\#16200](https://github.com/matrix-org/synapse/issues/16200)) +* Bump types-pyyaml from 6.0.12.10 to 6.0.12.11. ([\#16199](https://github.com/matrix-org/synapse/issues/16199)) + + +# Synapse 1.91.1 (2023-09-04) + +### Bugfixes + +- Fix a performance regression introduced in Synapse 1.91.0 where event persistence would cause an excessive linear growth in CPU usage. ([\#16220](https://github.com/matrix-org/synapse/issues/16220)) + + +# Synapse 1.91.0 (2023-08-30) + +No significant changes since 1.91.0rc1. + + +# Synapse 1.91.0rc1 (2023-08-23) + +### Features + +- Implements an admin API to lock an user without deactivating them. Based on [MSC3939](https://github.com/matrix-org/matrix-spec-proposals/pull/3939). ([\#15870](https://github.com/matrix-org/synapse/issues/15870)) +- Implements a task scheduler for resumable potentially long running tasks. ([\#15891](https://github.com/matrix-org/synapse/issues/15891)) +- Allow specifying `client_secret_path` as alternative to `client_secret` for OIDC providers. This avoids leaking the client secret in the homeserver config. Contributed by @Ma27. ([\#16030](https://github.com/matrix-org/synapse/issues/16030)) +- Allow customising the IdP display name, icon, and brand for SAML and CAS providers (in addition to OIDC provider). ([\#16094](https://github.com/matrix-org/synapse/issues/16094)) +- Add an `admins` query parameter to the [List Accounts](https://matrix-org.github.io/synapse/v1.91/admin_api/user_admin_api.html#list-accounts) [admin API](https://matrix-org.github.io/synapse/v1.91/usage/administration/admin_api/index.html), to include only admins or to exclude admins in user queries. ([\#16114](https://github.com/matrix-org/synapse/issues/16114)) + +### Bugfixes + +- Fix long-standing bug where concurrent requests to change a user's push rules could cause a deadlock. Contributed by Nick @ Beeper (@fizzadar). ([\#16052](https://github.com/matrix-org/synapse/issues/16052)) +- Fix a long-standing bu in `/sync` where timeout=0 does not skip caching, resulting in slow calls in cases where there are no new changes. Contributed by @PlasmaIntec. ([\#16080](https://github.com/matrix-org/synapse/issues/16080)) +- Fix performance of state resolutions for large, old rooms that did not have the full auth chain persisted. ([\#16116](https://github.com/matrix-org/synapse/issues/16116)) +- Filter out user agent references to the sliding sync proxy and rust-sdk from the user_daily_visits table to ensure that Element X can be represented fully. ([\#16124](https://github.com/matrix-org/synapse/issues/16124)) +- User constent and 3-PID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support. ([\#16127](https://github.com/matrix-org/synapse/issues/16127), [\#16134](https://github.com/matrix-org/synapse/issues/16134)) +- Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0. ([\#16133](https://github.com/matrix-org/synapse/issues/16133), [\#16169](https://github.com/matrix-org/synapse/issues/16169)) +- Fix performance degredation when there are a lot of in-flight replication requests. ([\#16148](https://github.com/matrix-org/synapse/issues/16148)) +- Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico. ([\#16156](https://github.com/matrix-org/synapse/issues/16156), [\#16164](https://github.com/matrix-org/synapse/issues/16164)) + +### Improved Documentation + +- Structured logging docs: add a link to explain the ELK stack ([\#16091](https://github.com/matrix-org/synapse/issues/16091)) + +### Internal Changes + +- Update dehydrated devices implementation. ([\#16010](https://github.com/matrix-org/synapse/issues/16010)) +- Fix database performance of read/write worker locks. ([\#16061](https://github.com/matrix-org/synapse/issues/16061)) +- Fix building the nix development environment on MacOS systems. ([\#16063](https://github.com/matrix-org/synapse/issues/16063)) +- Override global statement timeout when creating indexes in Postgres. ([\#16085](https://github.com/matrix-org/synapse/issues/16085)) +- Fix the type annotation on `run_db_interaction` in the Module API. ([\#16089](https://github.com/matrix-org/synapse/issues/16089)) +- Clean-up the presence code. ([\#16092](https://github.com/matrix-org/synapse/issues/16092)) +- Run `pyupgrade` for Python 3.8+. ([\#16110](https://github.com/matrix-org/synapse/issues/16110)) +- Rename pagination and purge locks and add comments to explain why they exist and how they work. ([\#16112](https://github.com/matrix-org/synapse/issues/16112)) +- Attempt to fix the twisted trunk job. ([\#16115](https://github.com/matrix-org/synapse/issues/16115)) +- Cache token introspection response from OIDC provider. ([\#16117](https://github.com/matrix-org/synapse/issues/16117)) +- Add cache to `get_server_keys_json_for_remote`. ([\#16123](https://github.com/matrix-org/synapse/issues/16123)) +- Add an admin endpoint to allow authorizing server to signal token revocations. ([\#16125](https://github.com/matrix-org/synapse/issues/16125)) +- Add response time metrics for introspection requests for delegated auth. ([\#16131](https://github.com/matrix-org/synapse/issues/16131)) +- MSC3861: allow impersonation by an admin user using `_oidc_admin_impersonate_user_id` query parameter. ([\#16132](https://github.com/matrix-org/synapse/issues/16132)) +- Increase performance of read/write locks. ([\#16149](https://github.com/matrix-org/synapse/issues/16149)) +- Improve presence tests. ([\#16150](https://github.com/matrix-org/synapse/issues/16150), [\#16151](https://github.com/matrix-org/synapse/issues/16151), [\#16158](https://github.com/matrix-org/synapse/issues/16158)) +- Raised the poetry-core version cap to 1.7.0. ([\#16152](https://github.com/matrix-org/synapse/issues/16152)) +- Fix assertion in user directory unit tests. ([\#16157](https://github.com/matrix-org/synapse/issues/16157)) +- Reduce scope of locks when paginating to alleviate DB contention. ([\#16159](https://github.com/matrix-org/synapse/issues/16159)) +- Reduce DB contention on worker locks. ([\#16160](https://github.com/matrix-org/synapse/issues/16160)) +- Task scheduler: mark task as active if we are scheduling as soon as possible. ([\#16165](https://github.com/matrix-org/synapse/issues/16165)) + +### Updates to locked dependencies + +* Bump click from 8.1.6 to 8.1.7. ([\#16145](https://github.com/matrix-org/synapse/issues/16145)) +* Bump gitpython from 3.1.31 to 3.1.32. ([\#16103](https://github.com/matrix-org/synapse/issues/16103)) +* Bump ijson from 3.2.1 to 3.2.3. ([\#16143](https://github.com/matrix-org/synapse/issues/16143)) +* Bump isort from 5.11.5 to 5.12.0. ([\#16108](https://github.com/matrix-org/synapse/issues/16108)) +* Bump log from 0.4.19 to 0.4.20. ([\#16109](https://github.com/matrix-org/synapse/issues/16109)) +* Bump pygithub from 1.59.0 to 1.59.1. ([\#16144](https://github.com/matrix-org/synapse/issues/16144)) +* Bump sentry-sdk from 1.28.1 to 1.29.2. ([\#16142](https://github.com/matrix-org/synapse/issues/16142)) +* Bump serde from 1.0.183 to 1.0.184. ([\#16139](https://github.com/matrix-org/synapse/issues/16139)) +* Bump txredisapi from 1.4.9 to 1.4.10. ([\#16107](https://github.com/matrix-org/synapse/issues/16107)) +* Bump types-bleach from 6.0.0.3 to 6.0.0.4. ([\#16106](https://github.com/matrix-org/synapse/issues/16106)) +* Bump types-pillow from 10.0.0.1 to 10.0.0.2. ([\#16105](https://github.com/matrix-org/synapse/issues/16105)) +* Bump types-pyopenssl from 23.2.0.1 to 23.2.0.2. ([\#16146](https://github.com/matrix-org/synapse/issues/16146)) + +# Synapse 1.91.0rc1 (2023-08-23) + +### Features + +- Implements an admin API to lock an user without deactivating them. Based on [MSC3939](https://github.com/matrix-org/matrix-spec-proposals/pull/3939). ([\#15870](https://github.com/matrix-org/synapse/issues/15870)) +- Allow specifying `client_secret_path` as alternative to `client_secret` for OIDC providers. This avoids leaking the client secret in the homeserver config. Contributed by @Ma27. ([\#16030](https://github.com/matrix-org/synapse/issues/16030)) +- Allow customising the IdP display name, icon, and brand for SAML and CAS providers (in addition to OIDC provider). ([\#16094](https://github.com/matrix-org/synapse/issues/16094)) +- Add an `admins` query parameter to the [List Accounts](https://matrix-org.github.io/synapse/v1.91/admin_api/user_admin_api.html#list-accounts) [admin API](https://matrix-org.github.io/synapse/v1.91/usage/administration/admin_api/index.html), to include only admins or to exclude admins in user queries. ([\#16114](https://github.com/matrix-org/synapse/issues/16114)) + +### Bugfixes + +- Fix long-standing bug where concurrent requests to change a user's push rules could cause a deadlock. Contributed by Nick @ Beeper (@fizzadar). ([\#16052](https://github.com/matrix-org/synapse/issues/16052)) +- Fix a long-standing bug in `/sync` where timeout=0 does not skip caching, resulting in slow calls in cases where there are no new changes. Contributed by @PlasmaIntec. ([\#16080](https://github.com/matrix-org/synapse/issues/16080)) +- Fix performance of state resolutions for large, old rooms that did not have the full auth chain persisted. ([\#16116](https://github.com/matrix-org/synapse/issues/16116)) +- Filter out user agent references to the sliding sync proxy and rust-sdk from the `user_daily_visits` table to ensure that Element X can be represented fully. ([\#16124](https://github.com/matrix-org/synapse/issues/16124)) +- User constent and third-party ID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support. ([\#16127](https://github.com/matrix-org/synapse/issues/16127), [\#16134](https://github.com/matrix-org/synapse/issues/16134)) +- Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0. ([\#16133](https://github.com/matrix-org/synapse/issues/16133), [\#16169](https://github.com/matrix-org/synapse/issues/16169)) +- Fix performance degredation when there are a lot of in-flight replication requests. ([\#16148](https://github.com/matrix-org/synapse/issues/16148)) +- Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico. ([\#16156](https://github.com/matrix-org/synapse/issues/16156), [\#16164](https://github.com/matrix-org/synapse/issues/16164)) + +### Improved Documentation + +- Structured logging docs: add a link to explain the ELK stack ([\#16091](https://github.com/matrix-org/synapse/issues/16091)) + +### Internal Changes + +- Update dehydrated devices implementation. ([\#16010](https://github.com/matrix-org/synapse/issues/16010)) +- Fix database performance of read/write worker locks. ([\#16061](https://github.com/matrix-org/synapse/issues/16061)) +- Fix building the nix development environment on MacOS systems. ([\#16063](https://github.com/matrix-org/synapse/issues/16063)) +- Override global statement timeout when creating indexes in Postgres. ([\#16085](https://github.com/matrix-org/synapse/issues/16085)) +- Fix the type annotation on `run_db_interaction` in the Module API. ([\#16089](https://github.com/matrix-org/synapse/issues/16089)) +- Clean-up the presence code. ([\#16092](https://github.com/matrix-org/synapse/issues/16092)) +- Run `pyupgrade` for Python 3.8+. ([\#16110](https://github.com/matrix-org/synapse/issues/16110)) +- Rename pagination and purge locks and add comments to explain why they exist and how they work. ([\#16112](https://github.com/matrix-org/synapse/issues/16112)) +- Attempt to fix the twisted trunk job. ([\#16115](https://github.com/matrix-org/synapse/issues/16115)) +- Cache token introspection response from OIDC provider. ([\#16117](https://github.com/matrix-org/synapse/issues/16117)) +- Add cache to `get_server_keys_json_for_remote`. ([\#16123](https://github.com/matrix-org/synapse/issues/16123)) +- Add an admin endpoint to allow authorizing server to signal token revocations. ([\#16125](https://github.com/matrix-org/synapse/issues/16125)) +- Add response time metrics for introspection requests for delegated auth. ([\#16131](https://github.com/matrix-org/synapse/issues/16131)) +- [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861): allow impersonation by an admin user using `_oidc_admin_impersonate_user_id` query parameter. ([\#16132](https://github.com/matrix-org/synapse/issues/16132)) +- Increase performance of read/write locks. ([\#16149](https://github.com/matrix-org/synapse/issues/16149)) +- Improve presence tests. ([\#16150](https://github.com/matrix-org/synapse/issues/16150), [\#16151](https://github.com/matrix-org/synapse/issues/16151), [\#16158](https://github.com/matrix-org/synapse/issues/16158)) +- Raised the poetry-core version cap to 1.7.0. ([\#16152](https://github.com/matrix-org/synapse/issues/16152)) +- Fix assertion in user directory unit tests. ([\#16157](https://github.com/matrix-org/synapse/issues/16157)) +- Reduce scope of locks when paginating to alleviate DB contention. ([\#16159](https://github.com/matrix-org/synapse/issues/16159)) +- Reduce DB contention on worker locks. ([\#16160](https://github.com/matrix-org/synapse/issues/16160)) +- Task scheduler: mark task as active if we are scheduling as soon as possible. ([\#16165](https://github.com/matrix-org/synapse/issues/16165)) +- Implements a task scheduler for resumable potentially long running tasks. ([\#15891](https://github.com/matrix-org/synapse/issues/15891)) + +### Updates to locked dependencies + +* Bump click from 8.1.6 to 8.1.7. ([\#16145](https://github.com/matrix-org/synapse/issues/16145)) +* Bump gitpython from 3.1.31 to 3.1.32. ([\#16103](https://github.com/matrix-org/synapse/issues/16103)) +* Bump ijson from 3.2.1 to 3.2.3. ([\#16143](https://github.com/matrix-org/synapse/issues/16143)) +* Bump isort from 5.11.5 to 5.12.0. ([\#16108](https://github.com/matrix-org/synapse/issues/16108)) +* Bump log from 0.4.19 to 0.4.20. ([\#16109](https://github.com/matrix-org/synapse/issues/16109)) +* Bump pygithub from 1.59.0 to 1.59.1. ([\#16144](https://github.com/matrix-org/synapse/issues/16144)) +* Bump sentry-sdk from 1.28.1 to 1.29.2. ([\#16142](https://github.com/matrix-org/synapse/issues/16142)) +* Bump serde from 1.0.183 to 1.0.184. ([\#16139](https://github.com/matrix-org/synapse/issues/16139)) +* Bump txredisapi from 1.4.9 to 1.4.10. ([\#16107](https://github.com/matrix-org/synapse/issues/16107)) +* Bump types-bleach from 6.0.0.3 to 6.0.0.4. ([\#16106](https://github.com/matrix-org/synapse/issues/16106)) +* Bump types-pillow from 10.0.0.1 to 10.0.0.2. ([\#16105](https://github.com/matrix-org/synapse/issues/16105)) +* Bump types-pyopenssl from 23.2.0.1 to 23.2.0.2. ([\#16146](https://github.com/matrix-org/synapse/issues/16146)) + +# Synapse 1.90.0 (2023-08-15) + +No significant changes since 1.90.0rc1. + + +# Synapse 1.90.0rc1 (2023-08-08) + +### Features + +- Scope transaction IDs to devices (implement [MSC3970](https://github.com/matrix-org/matrix-spec-proposals/pull/3970)). ([\#15629](https://github.com/matrix-org/synapse/issues/15629)) +- Remove old rows from the `cache_invalidation_stream_by_instance` table automatically (this table is unused in SQLite). ([\#15868](https://github.com/matrix-org/synapse/issues/15868)) + +### Bugfixes + +- Fix a long-standing bug where purging history and paginating simultaneously could lead to database corruption when using workers. ([\#15791](https://github.com/matrix-org/synapse/issues/15791)) +- Fix a long-standing bug where profile endpoint returned a 404 when the user's display name was empty. ([\#16012](https://github.com/matrix-org/synapse/issues/16012)) +- Fix a long-standing bug where the `synapse_port_db` failed to configure sequences for application services and partial stated rooms. ([\#16043](https://github.com/matrix-org/synapse/issues/16043)) +- Fix long-standing bug with deletion in dehydrated devices v2. ([\#16046](https://github.com/matrix-org/synapse/issues/16046)) + +### Updates to the Docker image + +- Add `org.opencontainers.image.version` labels to Docker containers [published by Matrix.org](https://hub.docker.com/r/matrixdotorg/synapse). Contributed by Mo Balaa. ([\#15972](https://github.com/matrix-org/synapse/issues/15972), [\#16009](https://github.com/matrix-org/synapse/issues/16009)) + +### Improved Documentation + +- Add a internal documentation page describing the ["streams" used within Synapse](https://matrix-org.github.io/synapse/v1.90/development/synapse_architecture/streams.html). ([\#16015](https://github.com/matrix-org/synapse/issues/16015)) +- Clarify comment on the keys/upload over replication enpoint. ([\#16016](https://github.com/matrix-org/synapse/issues/16016)) +- Do not expose Admin API in caddy reverse proxy example. Contributed by @NilsIrl. ([\#16027](https://github.com/matrix-org/synapse/issues/16027)) + +### Deprecations and Removals + +- Remove support for legacy application service paths. ([\#15964](https://github.com/matrix-org/synapse/issues/15964)) +- Move support for application service query parameter authorization behind a configuration option. ([\#16017](https://github.com/matrix-org/synapse/issues/16017)) + +### Internal Changes + +- Update SQL queries to inline boolean parameters as supported in SQLite 3.27. ([\#15525](https://github.com/matrix-org/synapse/issues/15525)) +- Allow for the configuration of the backoff algorithm for federation destinations. ([\#15754](https://github.com/matrix-org/synapse/issues/15754)) +- Allow modules to check whether the current worker is configured to run background tasks. ([\#15991](https://github.com/matrix-org/synapse/issues/15991)) +- Update support for [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958) to match the latest revision of the MSC. ([\#15992](https://github.com/matrix-org/synapse/issues/15992)) +- Allow modules to schedule delayed background calls. ([\#15993](https://github.com/matrix-org/synapse/issues/15993)) +- Properly overwrite the `redacts` content-property for forwards-compatibility with room versions 1 through 10. ([\#16013](https://github.com/matrix-org/synapse/issues/16013)) +- Fix building the nix development environment on MacOS systems. ([\#16019](https://github.com/matrix-org/synapse/issues/16019)) +- Remove leading and trailing spaces when setting a display name. ([\#16031](https://github.com/matrix-org/synapse/issues/16031)) +- Combine duplicated code. ([\#16023](https://github.com/matrix-org/synapse/issues/16023)) +- Collect additional metrics from `ResponseCache` for eviction. ([\#16028](https://github.com/matrix-org/synapse/issues/16028)) +- Fix endpoint improperly declaring support for MSC3814. ([\#16068](https://github.com/matrix-org/synapse/issues/16068)) +- Drop backwards compat hack for event serialization. ([\#16069](https://github.com/matrix-org/synapse/issues/16069)) + +### Updates to locked dependencies + +* Update PyYAML to 6.0.1. ([\#16011](https://github.com/matrix-org/synapse/issues/16011)) +* Bump cryptography from 41.0.2 to 41.0.3. ([\#16048](https://github.com/matrix-org/synapse/issues/16048)) +* Bump furo from 2023.5.20 to 2023.7.26. ([\#16077](https://github.com/matrix-org/synapse/issues/16077)) +* Bump immutabledict from 2.2.4 to 3.0.0. ([\#16034](https://github.com/matrix-org/synapse/issues/16034)) +* Update certifi to 2023.7.22 and pygments to 2.15.1. ([\#16044](https://github.com/matrix-org/synapse/issues/16044)) +* Bump jsonschema from 4.18.3 to 4.19.0. ([\#16081](https://github.com/matrix-org/synapse/issues/16081)) +* Bump phonenumbers from 8.13.14 to 8.13.18. ([\#16076](https://github.com/matrix-org/synapse/issues/16076)) +* Bump regex from 1.9.1 to 1.9.3. ([\#16073](https://github.com/matrix-org/synapse/issues/16073)) +* Bump serde from 1.0.171 to 1.0.175. ([\#15982](https://github.com/matrix-org/synapse/issues/15982)) +* Bump serde from 1.0.175 to 1.0.179. ([\#16033](https://github.com/matrix-org/synapse/issues/16033)) +* Bump serde from 1.0.179 to 1.0.183. ([\#16074](https://github.com/matrix-org/synapse/issues/16074)) +* Bump serde_json from 1.0.103 to 1.0.104. ([\#16032](https://github.com/matrix-org/synapse/issues/16032)) +* Bump service-identity from 21.1.0 to 23.1.0. ([\#16038](https://github.com/matrix-org/synapse/issues/16038)) +* Bump types-commonmark from 0.9.2.3 to 0.9.2.4. ([\#16037](https://github.com/matrix-org/synapse/issues/16037)) +* Bump types-jsonschema from 4.17.0.8 to 4.17.0.10. ([\#16036](https://github.com/matrix-org/synapse/issues/16036)) +* Bump types-netaddr from 0.8.0.8 to 0.8.0.9. ([\#16035](https://github.com/matrix-org/synapse/issues/16035)) +* Bump types-opentracing from 2.4.10.5 to 2.4.10.6. ([\#16078](https://github.com/matrix-org/synapse/issues/16078)) +* Bump types-setuptools from 68.0.0.0 to 68.0.0.3. ([\#16079](https://github.com/matrix-org/synapse/issues/16079)) + +# Synapse 1.89.0 (2023-08-01) + +No significant changes since 1.89.0rc1. + + +# Synapse 1.89.0rc1 (2023-07-25) + +### Features + +- Add Unix Socket support for HTTP Replication Listeners. [Document and provide usage instructions](https://matrix-org.github.io/synapse/v1.89/usage/configuration/config_documentation.html#listeners) for utilizing Unix sockets in Synapse. Contributed by Jason Little. ([\#15708](https://github.com/matrix-org/synapse/issues/15708), [\#15924](https://github.com/matrix-org/synapse/issues/15924)) +- Allow `+` in Matrix IDs, per [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009). ([\#15911](https://github.com/matrix-org/synapse/issues/15911)) +- Support room version 11 from [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820). ([\#15912](https://github.com/matrix-org/synapse/issues/15912)) +- Allow configuring the set of workers to proxy outbound federation traffic through via `outbound_federation_restricted_to`. ([\#15913](https://github.com/matrix-org/synapse/issues/15913), [\#15969](https://github.com/matrix-org/synapse/issues/15969)) +- Implement [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814), dehydrated devices v2/shrivelled sessions and move [MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697) behind a config flag. Contributed by Nico from Famedly, H-Shay and poljar. ([\#15929](https://github.com/matrix-org/synapse/issues/15929)) + +### Bugfixes + +- Fix a long-standing bug where remote invites weren't correctly pushed. ([\#15820](https://github.com/matrix-org/synapse/issues/15820)) +- Fix background schema updates failing over a large upgrade gap. ([\#15887](https://github.com/matrix-org/synapse/issues/15887)) +- Fix a bug introduced in 1.86.0 where Synapse starting with an empty `experimental_features` configuration setting. ([\#15925](https://github.com/matrix-org/synapse/issues/15925)) +- Fixed deploy annotations in the provided Grafana dashboard config, so that it shows for any homeserver and not just matrix.org. Contributed by @wrjlewis. ([\#15957](https://github.com/matrix-org/synapse/issues/15957)) +- Ensure a long state res does not starve CPU by occasionally yielding to the reactor. ([\#15960](https://github.com/matrix-org/synapse/issues/15960)) +- Properly handle redactions of creation events. ([\#15973](https://github.com/matrix-org/synapse/issues/15973)) +- Fix a bug where resyncing stale device lists could block responding to federation transactions, and thus delay receiving new data from the remote server. ([\#15975](https://github.com/matrix-org/synapse/issues/15975)) + +### Improved Documentation + +- Better clarify how to run a worker instance (pass both configs). ([\#15921](https://github.com/matrix-org/synapse/issues/15921)) +- Improve [the documentation](https://matrix-org.github.io/synapse/v1.89/admin_api/user_admin_api.html#login-as-a-user) for the login as a user admin API. ([\#15938](https://github.com/matrix-org/synapse/issues/15938)) +- Fix broken Arch Linux package link. Contributed by @SnipeXandrej. ([\#15981](https://github.com/matrix-org/synapse/issues/15981)) + +### Deprecations and Removals + +- Remove support for calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15928](https://github.com/matrix-org/synapse/issues/15928)) + +### Internal Changes + +- Mark `get_user_in_directory` private since it is only used in tests. Also remove the cache from it. ([\#15884](https://github.com/matrix-org/synapse/issues/15884)) +- Document which Python version runs on a given Linux distribution so we can more easily clean up later. ([\#15909](https://github.com/matrix-org/synapse/issues/15909)) +- Add details to warning in log when we fail to fetch an alias. ([\#15922](https://github.com/matrix-org/synapse/issues/15922)) +- Remove unneeded `__init__`. ([\#15926](https://github.com/matrix-org/synapse/issues/15926)) +- Fix bug with read/write lock implementation. This is currently unused so has no observable effects. ([\#15933](https://github.com/matrix-org/synapse/issues/15933), [\#15958](https://github.com/matrix-org/synapse/issues/15958)) +- Unbreak the nix development environment by pinning the Rust version to 1.70.0. ([\#15940](https://github.com/matrix-org/synapse/issues/15940)) +- Update presence metrics to differentiate remote vs local users. ([\#15952](https://github.com/matrix-org/synapse/issues/15952)) +- Stop reading from column `user_id` of table `profiles`. ([\#15955](https://github.com/matrix-org/synapse/issues/15955)) +- Build packages for Debian Trixie. ([\#15961](https://github.com/matrix-org/synapse/issues/15961)) +- Reduce the amount of state we pull out. ([\#15968](https://github.com/matrix-org/synapse/issues/15968)) +- Speed up updating state in large rooms. ([\#15971](https://github.com/matrix-org/synapse/issues/15971)) + +### Updates to locked dependencies + +* Bump anyhow from 1.0.71 to 1.0.72. ([\#15949](https://github.com/matrix-org/synapse/issues/15949)) +* Bump click from 8.1.3 to 8.1.6. ([\#15984](https://github.com/matrix-org/synapse/issues/15984)) +* Bump cryptography from 41.0.1 to 41.0.2. ([\#15943](https://github.com/matrix-org/synapse/issues/15943)) +* Bump jsonschema from 4.17.3 to 4.18.3. ([\#15948](https://github.com/matrix-org/synapse/issues/15948)) +* Bump pillow from 9.4.0 to 10.0.0. ([\#15986](https://github.com/matrix-org/synapse/issues/15986)) +* Bump prometheus-client from 0.17.0 to 0.17.1. ([\#15945](https://github.com/matrix-org/synapse/issues/15945)) +* Bump pydantic from 1.10.10 to 1.10.11. ([\#15946](https://github.com/matrix-org/synapse/issues/15946)) +* Bump pygithub from 1.58.2 to 1.59.0. ([\#15834](https://github.com/matrix-org/synapse/issues/15834)) +* Bump pyo3-log from 0.8.2 to 0.8.3. ([\#15951](https://github.com/matrix-org/synapse/issues/15951)) +* Bump sentry-sdk from 1.26.0 to 1.28.1. ([\#15985](https://github.com/matrix-org/synapse/issues/15985)) +* Bump serde_json from 1.0.100 to 1.0.103. ([\#15950](https://github.com/matrix-org/synapse/issues/15950)) +* Bump types-pillow from 9.5.0.4 to 10.0.0.1. ([\#15932](https://github.com/matrix-org/synapse/issues/15932)) +* Bump types-requests from 2.31.0.1 to 2.31.0.2. ([\#15983](https://github.com/matrix-org/synapse/issues/15983)) +* Bump typing-extensions from 4.5.0 to 4.7.1. ([\#15947](https://github.com/matrix-org/synapse/issues/15947)) + +# Synapse 1.88.0 (2023-07-18) + +This release + - raises the minimum supported version of Python to 3.8, as Python 3.7 is now [end-of-life](https://devguide.python.org/versions/), and + - removes deprecated config options related to worker deployment. + +See [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.88/docs/upgrade.md#upgrading-to-v1880) for more information. + + +### Bugfixes + +- Revert "Stop writing to column `user_id` of tables `profiles` and `user_filters`", which was introduced in Synapse 1.88.0rc1. ([\#15953](https://github.com/matrix-org/synapse/issues/15953)) + + +# Synapse 1.88.0rc1 (2023-07-11) + +### Features + +- Add `not_user_type` param to the [list accounts admin API](https://matrix-org.github.io/synapse/v1.88/admin_api/user_admin_api.html#list-accounts). ([\#15844](https://github.com/matrix-org/synapse/issues/15844)) + +### Bugfixes + +- Pin `pydantic` to `^=1.7.4` to avoid backwards-incompatible API changes from the 2.0.0 release. + Contributed by @PaarthShah. ([\#15862](https://github.com/matrix-org/synapse/issues/15862)) +- Correctly resize thumbnails with pillow version >=10. ([\#15876](https://github.com/matrix-org/synapse/issues/15876)) + +### Improved Documentation + +- Fixed header levels on the [Admin API "Users"](https://matrix-org.github.io/synapse/v1.87/admin_api/user_admin_api.html) documentation page. Contributed by @sumnerevans at @beeper. ([\#15852](https://github.com/matrix-org/synapse/issues/15852)) +- Remove deprecated `worker_replication_host`, `worker_replication_http_port` and `worker_replication_http_tls` configuration options. ([\#15872](https://github.com/matrix-org/synapse/issues/15872)) + +### Deprecations and Removals + +- **Remove deprecated `worker_replication_host`, `worker_replication_http_port` and `worker_replication_http_tls` configuration options.** See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.88/docs/upgrade.md#removal-of-worker_replication_-settings) for more details. ([\#15860](https://github.com/matrix-org/synapse/issues/15860)) +- Remove support for Python 3.7 and hence for Debian Buster. ([\#15851](https://github.com/matrix-org/synapse/issues/15851), [\#15892](https://github.com/matrix-org/synapse/issues/15892), [\#15893](https://github.com/matrix-org/synapse/issues/15893), [\#15917](https://github.com/matrix-org/synapse/pull/15917)) + +### Internal Changes + +- Add foreign key constraint to `event_forward_extremities`. ([\#15751](https://github.com/matrix-org/synapse/issues/15751), [\#15907](https://github.com/matrix-org/synapse/issues/15907)) +- Add read/write style cross-worker locks. ([\#15782](https://github.com/matrix-org/synapse/issues/15782)) +- Stop writing to column `user_id` of tables `profiles` and `user_filters`. ([\#15787](https://github.com/matrix-org/synapse/issues/15787)) +- Use lower isolation level when cleaning old presence stream data to avoid serialization errors. ([\#15826](https://github.com/matrix-org/synapse/issues/15826)) +- Add tracing to media `/upload` code paths. ([\#15850](https://github.com/matrix-org/synapse/issues/15850), [\#15888](https://github.com/matrix-org/synapse/issues/15888)) +- Add a timeout that aborts any Postgres statement taking more than 1 hour. ([\#15853](https://github.com/matrix-org/synapse/issues/15853)) +- Fix the `devenv up` configuration which was ignoring the config overrides. ([\#15854](https://github.com/matrix-org/synapse/issues/15854)) +- Optimised cleanup of old entries in `device_lists_stream`. ([\#15861](https://github.com/matrix-org/synapse/issues/15861)) +- Update the Matrix clients link in the _It works! Synapse is running_ landing page. ([\#15874](https://github.com/matrix-org/synapse/issues/15874)) +- Fix building Synapse with the nightly Rust compiler. ([\#15906](https://github.com/matrix-org/synapse/issues/15906)) +- Add `Server` to Access-Control-Expose-Headers header. ([\#15908](https://github.com/matrix-org/synapse/issues/15908)) + +### Updates to locked dependencies + +* Bump authlib from 1.2.0 to 1.2.1. ([\#15864](https://github.com/matrix-org/synapse/issues/15864)) +* Bump importlib-metadata from 6.6.0 to 6.7.0. ([\#15865](https://github.com/matrix-org/synapse/issues/15865)) +* Bump lxml from 4.9.2 to 4.9.3. ([\#15897](https://github.com/matrix-org/synapse/issues/15897)) +* Bump regex from 1.8.4 to 1.9.1. ([\#15902](https://github.com/matrix-org/synapse/issues/15902)) +* Bump ruff from 0.0.275 to 0.0.277. ([\#15900](https://github.com/matrix-org/synapse/issues/15900)) +* Bump sentry-sdk from 1.25.1 to 1.26.0. ([\#15867](https://github.com/matrix-org/synapse/issues/15867)) +* Bump serde_json from 1.0.99 to 1.0.100. ([\#15901](https://github.com/matrix-org/synapse/issues/15901)) +* Bump types-pyopenssl from 23.2.0.0 to 23.2.0.1. ([\#15866](https://github.com/matrix-org/synapse/issues/15866)) + +# Synapse 1.87.0 (2023-07-04) + +Please note that this will be the last release of Synapse that is compatible with +Python 3.7 and earlier. +This is due to Python 3.7 now having reached End of Life; see our [deprecation policy](https://matrix-org.github.io/synapse/v1.87/deprecation_policy.html) +for more details. + +### Bugfixes + +- Pin `pydantic` to `^1.7.4` to avoid backwards-incompatible API changes from the 2.0.0 release. + Resolves https://github.com/matrix-org/synapse/issues/15858. + Contributed by @PaarthShah. ([\#15862](https://github.com/matrix-org/synapse/issues/15862)) + +### Internal Changes + +- Split out 2022 changes from the changelog so the rendered version in GitHub doesn't timeout as much. ([\#15846](https://github.com/matrix-org/synapse/issues/15846)) + + +# Synapse 1.87.0rc1 (2023-06-27) + +### Features + +- Improve `/messages` response time by avoiding backfill when we already have messages to return. ([\#15737](https://github.com/matrix-org/synapse/issues/15737)) +- Add spam checker module API for logins. ([\#15838](https://github.com/matrix-org/synapse/issues/15838)) + +### Bugfixes + +- Fix a long-standing bug where media files were served in an unsafe manner. Contributed by @joshqou. ([\#15680](https://github.com/matrix-org/synapse/issues/15680)) +- Avoid invalidating a cache that was just prefilled. ([\#15758](https://github.com/matrix-org/synapse/issues/15758)) +- Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983). ([\#15770](https://github.com/matrix-org/synapse/issues/15770)) +- Fix joining rooms through aliases where the alias server isn't a real homeserver. Contributed by @tulir @ Beeper. ([\#15776](https://github.com/matrix-org/synapse/issues/15776)) +- Fix a bug in push rules handling leading to an invalid (per spec) `is_user_mention` rule sent to clients. Also fix wrong rule names for `is_user_mention` and `is_room_mention`. ([\#15781](https://github.com/matrix-org/synapse/issues/15781)) +- Fix a bug introduced in 1.57.0 where the wrong table would be locked on updating database rows when using SQLite as the database backend. ([\#15788](https://github.com/matrix-org/synapse/issues/15788)) +- Fix Sytest environmental variable evaluation in CI. ([\#15804](https://github.com/matrix-org/synapse/issues/15804)) +- Fix forgotten rooms missing from initial sync after rejoining them. Contributed by Nico from Famedly. ([\#15815](https://github.com/matrix-org/synapse/issues/15815)) +- Fix sqlite `user_filters` upgrade introduced in v1.86.0. ([\#15817](https://github.com/matrix-org/synapse/issues/15817)) + +### Improved Documentation + +- Document `looping_call()` functionality that will wait for the given function to finish before scheduling another. ([\#15772](https://github.com/matrix-org/synapse/issues/15772)) +- Fix a typo in the [Admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html). ([\#15805](https://github.com/matrix-org/synapse/issues/15805)) +- Fix typo in MSC number in faster remote room join architecture doc. ([\#15812](https://github.com/matrix-org/synapse/issues/15812)) + +### Deprecations and Removals + +- Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms. ([\#15748](https://github.com/matrix-org/synapse/issues/15748)) + +### Internal Changes + +- Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`. ([\#15233](https://github.com/matrix-org/synapse/issues/15233)) +- Regularly try to send transactions to other servers after they failed instead of waiting for a new event to be available before trying. ([\#15743](https://github.com/matrix-org/synapse/issues/15743)) +- Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983). ([\#15755](https://github.com/matrix-org/synapse/issues/15755)) +- Allow for the configuration of max request retries and min/max retry delays in the matrix federation client. ([\#15783](https://github.com/matrix-org/synapse/issues/15783)) +- Switch from `matrix://` to `matrix-federation://` scheme for internal Synapse routing of outbound federation traffic. ([\#15806](https://github.com/matrix-org/synapse/issues/15806)) +- Fix harmless exceptions being printed when running the port DB script. ([\#15814](https://github.com/matrix-org/synapse/issues/15814)) + +### Updates to locked dependencies + +* Bump attrs from 22.2.0 to 23.1.0. ([\#15801](https://github.com/matrix-org/synapse/issues/15801)) +* Bump cryptography from 40.0.2 to 41.0.1. ([\#15800](https://github.com/matrix-org/synapse/issues/15800)) +* Bump ijson from 3.2.0.post0 to 3.2.1. ([\#15802](https://github.com/matrix-org/synapse/issues/15802)) +* Bump phonenumbers from 8.13.13 to 8.13.14. ([\#15798](https://github.com/matrix-org/synapse/issues/15798)) +* Bump ruff from 0.0.265 to 0.0.272. ([\#15799](https://github.com/matrix-org/synapse/issues/15799)) +* Bump ruff from 0.0.272 to 0.0.275. ([\#15833](https://github.com/matrix-org/synapse/issues/15833)) +* Bump serde_json from 1.0.96 to 1.0.97. ([\#15797](https://github.com/matrix-org/synapse/issues/15797)) +* Bump serde_json from 1.0.97 to 1.0.99. ([\#15832](https://github.com/matrix-org/synapse/issues/15832)) +* Bump towncrier from 22.12.0 to 23.6.0. ([\#15831](https://github.com/matrix-org/synapse/issues/15831)) +* Bump types-opentracing from 2.4.10.4 to 2.4.10.5. ([\#15830](https://github.com/matrix-org/synapse/issues/15830)) +* Bump types-setuptools from 67.8.0.0 to 68.0.0.0. ([\#15835](https://github.com/matrix-org/synapse/issues/15835)) + +Synapse 1.86.0 (2023-06-20) +=========================== + +No significant changes since 1.86.0rc2. + + +Synapse 1.86.0rc2 (2023-06-14) +============================== + +Bugfixes +-------- + +- Fix an error when having workers of different versions running. ([\#15774](https://github.com/matrix-org/synapse/issues/15774)) + + +Synapse 1.86.0rc1 (2023-06-13) +============================== + +This version was tagged but never released. + +Features +-------- + +- Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session. ([\#15388](https://github.com/matrix-org/synapse/issues/15388)) +- Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API. ([\#15450](https://github.com/matrix-org/synapse/issues/15450)) +- Enable support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#15520](https://github.com/matrix-org/synapse/issues/15520)) +- Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider. ([\#15582](https://github.com/matrix-org/synapse/issues/15582)) +- Add Synapse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions. ([\#15674](https://github.com/matrix-org/synapse/issues/15674)) +- Add a catch-all * to the supported relation types when redacting an event and its related events. This is an update to [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation. ([\#15705](https://github.com/matrix-org/synapse/issues/15705)) +- Speed up `/messages` by backfilling in the background when there are no backward extremities where we are directly paginating. ([\#15710](https://github.com/matrix-org/synapse/issues/15710)) +- Expose a metric reporting the database background update status. ([\#15740](https://github.com/matrix-org/synapse/issues/15740)) + + +Bugfixes +-------- + +- Correctly clear caches when we delete a room. ([\#15609](https://github.com/matrix-org/synapse/issues/15609)) +- Check permissions for enabling encryption earlier during room creation to avoid creating broken rooms. ([\#15695](https://github.com/matrix-org/synapse/issues/15695)) + + +Improved Documentation +---------------------- + +- Simplify query to find participating servers in a room. ([\#15732](https://github.com/matrix-org/synapse/issues/15732)) + + +Internal Changes +---------------- + +- Log when events are (maybe unexpectedly) filtered out of responses in tests. ([\#14213](https://github.com/matrix-org/synapse/issues/14213)) +- Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`. ([\#15649](https://github.com/matrix-org/synapse/issues/15649)) +- Add support for tracing functions which return `Awaitable`s. ([\#15650](https://github.com/matrix-org/synapse/issues/15650)) +- Cache requests for user's devices over federation. ([\#15675](https://github.com/matrix-org/synapse/issues/15675)) +- Add fully qualified docker image names to Dockerfiles. ([\#15689](https://github.com/matrix-org/synapse/issues/15689)) +- Remove some unused code. ([\#15690](https://github.com/matrix-org/synapse/issues/15690)) +- Improve type hints. ([\#15694](https://github.com/matrix-org/synapse/issues/15694), [\#15697](https://github.com/matrix-org/synapse/issues/15697)) +- Update docstring and traces on `maybe_backfill()` functions. ([\#15709](https://github.com/matrix-org/synapse/issues/15709)) +- Add context for when/why to use the `long_retries` option when sending Federation requests. ([\#15721](https://github.com/matrix-org/synapse/issues/15721)) +- Removed some unused fields. ([\#15723](https://github.com/matrix-org/synapse/issues/15723)) +- Update federation error to more plainly explain we can only authorize our own membership events. ([\#15725](https://github.com/matrix-org/synapse/issues/15725)) +- Prevent the `latest_deps` and `twisted_trunk` daily GitHub Actions workflows from running on forks of the codebase. ([\#15726](https://github.com/matrix-org/synapse/issues/15726)) +- Improve performance of user directory search. ([\#15729](https://github.com/matrix-org/synapse/issues/15729)) +- Remove redundant table join with `room_memberships` when doing a `is_host_joined()`/`is_host_invited()` call (`membership` is already part of the `current_state_events`). ([\#15731](https://github.com/matrix-org/synapse/issues/15731)) +- Remove superfluous `room_memberships` join from background update. ([\#15733](https://github.com/matrix-org/synapse/issues/15733)) +- Speed up typechecking CI. ([\#15752](https://github.com/matrix-org/synapse/issues/15752)) +- Bump minimum supported Rust version to 1.60.0. ([\#15768](https://github.com/matrix-org/synapse/issues/15768)) + +### Updates to locked dependencies + +* Bump importlib-metadata from 6.1.0 to 6.6.0. ([\#15711](https://github.com/matrix-org/synapse/issues/15711)) +* Bump library/redis from 6-bullseye to 7-bullseye in /docker. ([\#15712](https://github.com/matrix-org/synapse/issues/15712)) +* Bump log from 0.4.18 to 0.4.19. ([\#15761](https://github.com/matrix-org/synapse/issues/15761)) +* Bump phonenumbers from 8.13.11 to 8.13.13. ([\#15763](https://github.com/matrix-org/synapse/issues/15763)) +* Bump pyasn1 from 0.4.8 to 0.5.0. ([\#15713](https://github.com/matrix-org/synapse/issues/15713)) +* Bump pydantic from 1.10.8 to 1.10.9. ([\#15762](https://github.com/matrix-org/synapse/issues/15762)) +* Bump pyo3-log from 0.8.1 to 0.8.2. ([\#15759](https://github.com/matrix-org/synapse/issues/15759)) +* Bump pyopenssl from 23.1.1 to 23.2.0. ([\#15765](https://github.com/matrix-org/synapse/issues/15765)) +* Bump regex from 1.7.3 to 1.8.4. ([\#15769](https://github.com/matrix-org/synapse/issues/15769)) +* Bump sentry-sdk from 1.22.1 to 1.25.0. ([\#15714](https://github.com/matrix-org/synapse/issues/15714)) +* Bump sentry-sdk from 1.25.0 to 1.25.1. ([\#15764](https://github.com/matrix-org/synapse/issues/15764)) +* Bump serde from 1.0.163 to 1.0.164. ([\#15760](https://github.com/matrix-org/synapse/issues/15760)) +* Bump types-jsonschema from 4.17.0.7 to 4.17.0.8. ([\#15716](https://github.com/matrix-org/synapse/issues/15716)) +* Bump types-pyopenssl from 23.1.0.2 to 23.2.0.0. ([\#15766](https://github.com/matrix-org/synapse/issues/15766)) +* Bump types-requests from 2.31.0.0 to 2.31.0.1. ([\#15715](https://github.com/matrix-org/synapse/issues/15715)) + +Synapse 1.85.2 (2023-06-08) +=========================== + +Bugfixes +-------- + +- Fix regression where using TLS for HTTP replication between workers did not work. Introduced in v1.85.0. ([\#15746](https://github.com/matrix-org/synapse/issues/15746)) + + +Synapse 1.85.1 (2023-06-07) +=========================== + +Note: this release only fixes a bug that stopped some deployments from upgrading to v1.85.0. There is no need to upgrade to v1.85.1 if successfully running v1.85.0. + +Bugfixes +-------- + +- Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. ([\#15738](https://github.com/matrix-org/synapse/issues/15738), [\#15739](https://github.com/matrix-org/synapse/issues/15739)) + + +Synapse 1.85.0 (2023-06-06) +=========================== + +No significant changes since 1.85.0rc2. + + +## Security advisory + +The following issues are fixed in 1.85.0 (and RCs). + +- [GHSA-26c5-ppr8-f33p](https://github.com/matrix-org/synapse/security/advisories/GHSA-26c5-ppr8-f33p) / [CVE-2023-32682](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32682) — Low Severity + + It may be possible for a deactivated user to login when using uncommon configurations. + +- [GHSA-98px-6486-j7qc](https://github.com/matrix-org/synapse/security/advisories/GHSA-98px-6486-j7qc) / [CVE-2023-32683](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32683) — Low Severity + + A discovered oEmbed or image URL can bypass the `url_preview_url_blacklist` setting potentially allowing server side request forgery or bypassing network policies. Impact is limited to IP addresses allowed by the `url_preview_ip_range_blacklist` setting (by default this only allows public IPs). + +See the advisories for more details. If you have any questions, email security@matrix.org. + + +Synapse 1.85.0rc2 (2023-06-01) +============================== + +Bugfixes +-------- + +- Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. ([\#15693](https://github.com/matrix-org/synapse/issues/15693)) + + +Deprecations and Removals +------------------------- + +- Deprecate calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15703](https://github.com/matrix-org/synapse/issues/15703)) + + +Internal Changes +---------------- + +- Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. ([\#15700](https://github.com/matrix-org/synapse/issues/15700)) + + +Synapse 1.85.0rc1 (2023-05-30) +============================== + +Features +-------- + +- Improve performance of backfill requests by performing backfill of previously failed requests in the background. ([\#15585](https://github.com/matrix-org/synapse/issues/15585)) +- Add a new [admin API](https://matrix-org.github.io/synapse/v1.85/usage/administration/admin_api/index.html) to [create a new device for a user](https://matrix-org.github.io/synapse/v1.85/admin_api/user_admin_api.html#create-a-device). ([\#15611](https://github.com/matrix-org/synapse/issues/15611)) +- Add Unix socket support for Redis connections. Contributed by Jason Little. ([\#15644](https://github.com/matrix-org/synapse/issues/15644)) + + +Bugfixes +-------- + +- Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar). ([\#15464](https://github.com/matrix-org/synapse/issues/15464)) +- Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL. ([\#15601](https://github.com/matrix-org/synapse/issues/15601)) +- Fix a long-standing bug where filters with multiple backslashes were rejected. ([\#15607](https://github.com/matrix-org/synapse/issues/15607)) +- Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted. ([\#15614](https://github.com/matrix-org/synapse/issues/15614)) +- Fix a long-standing bug where deactivated users were still able to login using the custom `org.matrix.login.jwt` login type (if enabled). ([\#15624](https://github.com/matrix-org/synapse/issues/15624)) +- Fix a long-standing bug where deactivated users were able to login in uncommon situations. ([\#15634](https://github.com/matrix-org/synapse/issues/15634)) + + +Improved Documentation +---------------------- + +- Warn users that at least 3.75GB of space is needed for the nix Synapse development environment. ([\#15613](https://github.com/matrix-org/synapse/issues/15613)) +- Remove outdated comment from the generated and sample homeserver log configs. ([\#15648](https://github.com/matrix-org/synapse/issues/15648)) +- Improve contributor docs to make it more clear that Rust is a necessary prerequisite. Contributed by @grantm. ([\#15668](https://github.com/matrix-org/synapse/issues/15668)) + + +Deprecations and Removals +------------------------- + +- Remove the old version of the R30 (30-day retained users) phone-home metric. ([\#10428](https://github.com/matrix-org/synapse/issues/10428)) + + +Internal Changes +---------------- + +- Create dependabot changelogs at release time. ([\#15481](https://github.com/matrix-org/synapse/issues/15481)) +- Add not null constraint to column `full_user_id` of tables `profiles` and `user_filters`. ([\#15537](https://github.com/matrix-org/synapse/issues/15537)) +- Allow connecting to HTTP Replication Endpoints by using `worker_name` when constructing the request. ([\#15578](https://github.com/matrix-org/synapse/issues/15578)) +- Make the `thread_id` column on `event_push_actions`, `event_push_actions_staging`, and `event_push_summary` non-null. ([\#15597](https://github.com/matrix-org/synapse/issues/15597)) +- Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible. ([\#15602](https://github.com/matrix-org/synapse/issues/15602)) +- Fix subscriptable type usage in Python <3.9. ([\#15604](https://github.com/matrix-org/synapse/issues/15604)) +- Update internal terminology. ([\#15606](https://github.com/matrix-org/synapse/issues/15606), [\#15620](https://github.com/matrix-org/synapse/issues/15620)) +- Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing. ([\#15610](https://github.com/matrix-org/synapse/issues/15610), [\#15647](https://github.com/matrix-org/synapse/issues/15647)) +- Trace how many new events from the backfill response we need to process. ([\#15633](https://github.com/matrix-org/synapse/issues/15633)) +- Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s. ([\#15615](https://github.com/matrix-org/synapse/issues/15615)) +- Update Mutual Rooms ([MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666)) implementation to match new proposal text. ([\#15621](https://github.com/matrix-org/synapse/issues/15621)) +- Remove the unstable identifiers from faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706)). ([\#15625](https://github.com/matrix-org/synapse/issues/15625)) +- Fix the olddeps CI. ([\#15626](https://github.com/matrix-org/synapse/issues/15626)) +- Remove duplicate timestamp from test logs (`_trial_temp/test.log`). ([\#15636](https://github.com/matrix-org/synapse/issues/15636)) +- Fix two memory leaks in `trial` test runs. ([\#15630](https://github.com/matrix-org/synapse/issues/15630)) +- Limit the size of the `HomeServerConfig` cache in trial test runs. ([\#15646](https://github.com/matrix-org/synapse/issues/15646)) +- Improve type hints. ([\#15658](https://github.com/matrix-org/synapse/issues/15658), [\#15659](https://github.com/matrix-org/synapse/issues/15659)) +- Add requesting user id parameter to key claim methods in `TransportLayerClient`. ([\#15663](https://github.com/matrix-org/synapse/issues/15663)) +- Speed up rebuilding of the user directory for local users. ([\#15665](https://github.com/matrix-org/synapse/issues/15665)) +- Implement "option 2" for [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820): Room version 11. ([\#15666](https://github.com/matrix-org/synapse/issues/15666), [\#15678](https://github.com/matrix-org/synapse/issues/15678)) + +### Updates to locked dependencies + +* Bump furo from 2023.3.27 to 2023.5.20. ([\#15642](https://github.com/matrix-org/synapse/issues/15642)) +* Bump log from 0.4.17 to 0.4.18. ([\#15681](https://github.com/matrix-org/synapse/issues/15681)) +* Bump prometheus-client from 0.16.0 to 0.17.0. ([\#15682](https://github.com/matrix-org/synapse/issues/15682)) +* Bump pydantic from 1.10.7 to 1.10.8. ([\#15685](https://github.com/matrix-org/synapse/issues/15685)) +* Bump pygithub from 1.58.1 to 1.58.2. ([\#15643](https://github.com/matrix-org/synapse/issues/15643)) +* Bump requests from 2.28.2 to 2.31.0. ([\#15651](https://github.com/matrix-org/synapse/issues/15651)) +* Bump sphinx from 6.1.3 to 6.2.1. ([\#15641](https://github.com/matrix-org/synapse/issues/15641)) +* Bump types-bleach from 6.0.0.1 to 6.0.0.3. ([\#15686](https://github.com/matrix-org/synapse/issues/15686)) +* Bump types-pillow from 9.5.0.2 to 9.5.0.4. ([\#15640](https://github.com/matrix-org/synapse/issues/15640)) +* Bump types-pyyaml from 6.0.12.9 to 6.0.12.10. ([\#15683](https://github.com/matrix-org/synapse/issues/15683)) +* Bump types-requests from 2.30.0.0 to 2.31.0.0. ([\#15684](https://github.com/matrix-org/synapse/issues/15684)) +* Bump types-setuptools from 67.7.0.2 to 67.8.0.0. ([\#15639](https://github.com/matrix-org/synapse/issues/15639)) + +Synapse 1.84.1 (2023-05-26) +=========================== + +This patch release fixes a major issue with homeservers that do not have an `instance_map` defined but which do use workers. +If you have already upgraded to Synapse 1.84.0 and your homeserver is working normally, then there is no need to update to this patch release. + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse v1.84.0 where workers do not start up when no `instance_map` was provided. ([\#15672](https://github.com/matrix-org/synapse/issues/15672)) + + +Internal Changes +---------------- + +- Add `dch` and `notify-send` to the development Nix flake so that the release script can be used. ([\#15673](https://github.com/matrix-org/synapse/issues/15673)) + + +Synapse 1.84.0 (2023-05-23) +=========================== + +The `worker_replication_*` configuration settings have been deprecated in favour of configuring the main process consistently with other instances in the `instance_map`. The deprecated settings will be removed in Synapse v1.88.0, but changing your configuration in advance is recommended. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.84/docs/upgrade.md#upgrading-to-v1840) for more information. + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.84.0rc1 where errors during startup were not reported correctly on Python < 3.10. ([\#15599](https://github.com/matrix-org/synapse/issues/15599)) + + +Synapse 1.84.0rc1 (2023-05-16) +============================== + +Features +-------- + +- Add an option to prevent media downloads from configured domains. ([\#15197](https://github.com/matrix-org/synapse/issues/15197)) +- Add `forget_rooms_on_leave` config option to automatically forget rooms when users leave them or are removed from them. ([\#15224](https://github.com/matrix-org/synapse/issues/15224)) +- Add redis TLS configuration options. ([\#15312](https://github.com/matrix-org/synapse/issues/15312)) +- Add a config option to delay push notifications by a random amount, to discourage time-based profiling. ([\#15516](https://github.com/matrix-org/synapse/issues/15516)) +- Stabilize support for [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15528](https://github.com/matrix-org/synapse/issues/15528)) +- Implement [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009) to expand the supported characters in Matrix IDs. ([\#15536](https://github.com/matrix-org/synapse/issues/15536)) +- Advertise support for Matrix 1.6 on `/_matrix/client/versions`. ([\#15559](https://github.com/matrix-org/synapse/issues/15559)) +- Print full error and stack-trace of any exception that occurs during startup/initialization. ([\#15569](https://github.com/matrix-org/synapse/issues/15569)) + + +Bugfixes +-------- + +- Don't fail on federation over TOR where SRV queries are not supported. Contributed by Zdzichu. ([\#15523](https://github.com/matrix-org/synapse/issues/15523)) +- Experimental support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which rejects setting the `"m.push_rules"` via account data. ([\#15554](https://github.com/matrix-org/synapse/issues/15554), [\#15555](https://github.com/matrix-org/synapse/issues/15555)) +- Fix a long-standing bug where an invalid membership event could cause an internal server error. ([\#15564](https://github.com/matrix-org/synapse/issues/15564)) +- Require at least poetry-core v1.1.0. ([\#15566](https://github.com/matrix-org/synapse/issues/15566), [\#15571](https://github.com/matrix-org/synapse/issues/15571)) + + +Deprecations and Removals +------------------------- + +- Remove need for `worker_replication_*` based settings in worker configuration yaml by placing this data directly on the `instance_map` instead. ([\#15491](https://github.com/matrix-org/synapse/issues/15491)) + + +Updates to the Docker image +--------------------------- + +- Add pkg-config package to Stage 0 to be able to build Dockerfile on ppc64le architecture. ([\#15567](https://github.com/matrix-org/synapse/issues/15567)) + + +Improved Documentation +---------------------- + +- Clarify documentation of the "Create or modify account" Admin API. ([\#15544](https://github.com/matrix-org/synapse/issues/15544)) +- Fix path to the `statistics/database/rooms` admin API in documentation. ([\#15560](https://github.com/matrix-org/synapse/issues/15560)) +- Update and improve Mastodon Single Sign-On documentation. ([\#15587](https://github.com/matrix-org/synapse/issues/15587)) + + +Internal Changes +---------------- + +- Use oEmbed to generate URL previews for YouTube Shorts. ([\#15025](https://github.com/matrix-org/synapse/issues/15025)) +- Create new `Client` for use with HTTP Replication between workers. Contributed by Jason Little. ([\#15470](https://github.com/matrix-org/synapse/issues/15470)) +- Bump pyicu from 2.10.2 to 2.11. ([\#15509](https://github.com/matrix-org/synapse/issues/15509)) +- Remove references to supporting per-user flag for [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654). ([\#15522](https://github.com/matrix-org/synapse/issues/15522)) +- Don't use a trusted key server when running the demo scripts. ([\#15527](https://github.com/matrix-org/synapse/issues/15527)) +- Speed up rebuilding of the user directory for local users. ([\#15529](https://github.com/matrix-org/synapse/issues/15529)) +- Speed up deleting of old rows in `event_push_actions`. ([\#15531](https://github.com/matrix-org/synapse/issues/15531)) +- Install the `xmlsec` and `mdbook` packages and switch back to the upstream [cachix/devenv](https://github.com/cachix/devenv) repo in the nix development environment. ([\#15532](https://github.com/matrix-org/synapse/issues/15532), [\#15533](https://github.com/matrix-org/synapse/issues/15533), [\#15545](https://github.com/matrix-org/synapse/issues/15545)) +- Implement [MSC3987](https://github.com/matrix-org/matrix-spec-proposals/pull/3987) by removing `"dont_notify"` from the list of actions in default push rules. ([\#15534](https://github.com/matrix-org/synapse/issues/15534)) +- Move various module API callback registration methods to a dedicated class. ([\#15535](https://github.com/matrix-org/synapse/issues/15535)) +- Proxy `/user/devices` federation queries to application services for [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984). ([\#15539](https://github.com/matrix-org/synapse/issues/15539)) +- Factor out an `is_mine_server_name` method. ([\#15542](https://github.com/matrix-org/synapse/issues/15542)) +- Allow running Complement tests using [podman](https://podman.io/) by adding a `PODMAN` environment variable to `scripts-dev/complement.sh`. ([\#15543](https://github.com/matrix-org/synapse/issues/15543)) +- Bump serde from 1.0.160 to 1.0.162. ([\#15548](https://github.com/matrix-org/synapse/issues/15548)) +- Bump types-setuptools from 67.6.0.5 to 67.7.0.1. ([\#15549](https://github.com/matrix-org/synapse/issues/15549)) +- Bump sentry-sdk from 1.19.1 to 1.22.1. ([\#15550](https://github.com/matrix-org/synapse/issues/15550)) +- Bump ruff from 0.0.259 to 0.0.265. ([\#15551](https://github.com/matrix-org/synapse/issues/15551)) +- Bump hiredis from 2.2.2 to 2.2.3. ([\#15552](https://github.com/matrix-org/synapse/issues/15552)) +- Bump types-requests from 2.29.0.0 to 2.30.0.0. ([\#15553](https://github.com/matrix-org/synapse/issues/15553)) +- Add `org.matrix.msc3981` info to `/_matrix/client/versions`. ([\#15558](https://github.com/matrix-org/synapse/issues/15558)) +- Declare unstable support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) under `/_matrix/client/versions` if the experimental implementation is enabled. ([\#15562](https://github.com/matrix-org/synapse/issues/15562)) +- Implement [MSC3821](https://github.com/matrix-org/matrix-spec-proposals/pull/3821) to update the redaction rules. ([\#15563](https://github.com/matrix-org/synapse/issues/15563)) +- Implement updated redaction rules from [MSC3389](https://github.com/matrix-org/matrix-spec-proposals/pull/3389). ([\#15565](https://github.com/matrix-org/synapse/issues/15565)) +- Allow `pip install` to use setuptools_rust 1.6.0 when building Synapse. ([\#15570](https://github.com/matrix-org/synapse/issues/15570)) +- Deal with upcoming Github Actions deprecations. ([\#15576](https://github.com/matrix-org/synapse/issues/15576)) +- Export `run_as_background_process` from the module API. ([\#15577](https://github.com/matrix-org/synapse/issues/15577)) +- Update build system requirements to allow building with poetry-core==1.6.0. ([\#15588](https://github.com/matrix-org/synapse/issues/15588)) +- Bump serde from 1.0.162 to 1.0.163. ([\#15589](https://github.com/matrix-org/synapse/issues/15589)) +- Bump phonenumbers from 8.13.7 to 8.13.11. ([\#15590](https://github.com/matrix-org/synapse/issues/15590)) +- Bump types-psycopg2 from 2.9.21.9 to 2.9.21.10. ([\#15591](https://github.com/matrix-org/synapse/issues/15591)) +- Bump types-commonmark from 0.9.2.2 to 0.9.2.3. ([\#15592](https://github.com/matrix-org/synapse/issues/15592)) +- Bump types-setuptools from 67.7.0.1 to 67.7.0.2. ([\#15594](https://github.com/matrix-org/synapse/issues/15594)) + + +Synapse 1.83.0 (2023-05-09) +=========================== + +No significant changes since 1.83.0rc1. + + +Synapse 1.83.0rc1 (2023-05-02) +============================== + +Features +-------- + +- Experimental support to recursively provide relations per [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981). ([\#15315](https://github.com/matrix-org/synapse/issues/15315)) +- Experimental support for [MSC3970](https://github.com/matrix-org/matrix-spec-proposals/pull/3970): Scope transaction IDs to devices. ([\#15318](https://github.com/matrix-org/synapse/issues/15318)) +- Add an [admin API endpoint](https://matrix-org.github.io/synapse/v1.83/admin_api/experimental_features.html) to support per-user feature flags. ([\#15344](https://github.com/matrix-org/synapse/issues/15344)) +- Add a module API to send an HTTP push notification. ([\#15387](https://github.com/matrix-org/synapse/issues/15387)) +- Add an [admin API endpoint](https://matrix-org.github.io/synapse/v1.83/admin_api/statistics.html#get-largest-rooms-by-size-in-database) to query the largest rooms by disk space used in the database. ([\#15482](https://github.com/matrix-org/synapse/issues/15482)) + + +Bugfixes +-------- + +- Disable push rule evaluation for rooms excluded from sync. ([\#15361](https://github.com/matrix-org/synapse/issues/15361)) +- Fix a long-standing bug where cached server key results which were directly fetched would not be properly re-used. ([\#15417](https://github.com/matrix-org/synapse/issues/15417)) +- Fix a bug introduced in Synapse 1.73.0 where some experimental push rules were returned by default. ([\#15494](https://github.com/matrix-org/synapse/issues/15494)) + + +Improved Documentation +---------------------- + +- Add Nginx loadbalancing example with sticky mxid for workers. ([\#15411](https://github.com/matrix-org/synapse/issues/15411)) +- Update outdated development docs that mention restrictions in versions of SQLite that we no longer support. ([\#15498](https://github.com/matrix-org/synapse/issues/15498)) + + +Internal Changes +---------------- + +- Speedup tests by caching HomeServerConfig instances. ([\#15284](https://github.com/matrix-org/synapse/issues/15284)) +- Add denormalised event stream ordering column to membership state tables for future use. Contributed by Nick @ Beeper (@fizzadar). ([\#15356](https://github.com/matrix-org/synapse/issues/15356)) +- Always use multi-user device resync replication endpoints. ([\#15418](https://github.com/matrix-org/synapse/issues/15418)) +- Add column `full_user_id` to tables `profiles` and `user_filters`. ([\#15458](https://github.com/matrix-org/synapse/issues/15458)) +- Update support for [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) to allow always returning fallback-keys in a `/keys/claim` request. ([\#15462](https://github.com/matrix-org/synapse/issues/15462)) +- Improve type hints. ([\#15465](https://github.com/matrix-org/synapse/issues/15465), [\#15496](https://github.com/matrix-org/synapse/issues/15496), [\#15497](https://github.com/matrix-org/synapse/issues/15497)) +- Support claiming more than one OTK at a time. ([\#15468](https://github.com/matrix-org/synapse/issues/15468)) +- Bump types-pyyaml from 6.0.12.8 to 6.0.12.9. ([\#15471](https://github.com/matrix-org/synapse/issues/15471)) +- Bump pyasn1-modules from 0.2.8 to 0.3.0. ([\#15473](https://github.com/matrix-org/synapse/issues/15473)) +- Bump cryptography from 40.0.1 to 40.0.2. ([\#15474](https://github.com/matrix-org/synapse/issues/15474)) +- Bump types-netaddr from 0.8.0.7 to 0.8.0.8. ([\#15475](https://github.com/matrix-org/synapse/issues/15475)) +- Bump types-jsonschema from 4.17.0.6 to 4.17.0.7. ([\#15476](https://github.com/matrix-org/synapse/issues/15476)) +- Ask bug reporters to provide logs as text. ([\#15479](https://github.com/matrix-org/synapse/issues/15479)) +- Add a Nix flake for use as a development environment. ([\#15495](https://github.com/matrix-org/synapse/issues/15495)) +- Bump anyhow from 1.0.70 to 1.0.71. ([\#15507](https://github.com/matrix-org/synapse/issues/15507)) +- Bump types-pillow from 9.4.0.19 to 9.5.0.2. ([\#15508](https://github.com/matrix-org/synapse/issues/15508)) +- Bump packaging from 23.0 to 23.1. ([\#15510](https://github.com/matrix-org/synapse/issues/15510)) +- Bump types-requests from 2.28.11.16 to 2.29.0.0. ([\#15511](https://github.com/matrix-org/synapse/issues/15511)) +- Bump setuptools-rust from 1.5.2 to 1.6.0. ([\#15512](https://github.com/matrix-org/synapse/issues/15512)) +- Update the check_schema_delta script to account for when the schema version has been bumped locally. ([\#15466](https://github.com/matrix-org/synapse/issues/15466)) + + +Synapse 1.82.0 (2023-04-25) +=========================== + +No significant changes since 1.82.0rc1. + + +Synapse 1.82.0rc1 (2023-04-18) +============================== + +Features +-------- + +- Allow loading the `/directory/room/{roomAlias}` endpoint on workers. ([\#15333](https://github.com/matrix-org/synapse/issues/15333)) +- Add some validation to `instance_map` configuration loading. ([\#15431](https://github.com/matrix-org/synapse/issues/15431)) +- Allow loading the `/capabilities` endpoint on workers. ([\#15436](https://github.com/matrix-org/synapse/issues/15436)) + + +Bugfixes +-------- + +- Delete server-side backup keys when deactivating an account. ([\#15181](https://github.com/matrix-org/synapse/issues/15181)) +- Fix and document untold assumption that `on_logged_out` module hooks will be called before the deletion of pushers. ([\#15410](https://github.com/matrix-org/synapse/issues/15410)) +- Improve robustness when handling a perspective key response by deduplicating received server keys. ([\#15423](https://github.com/matrix-org/synapse/issues/15423)) +- Synapse now correctly fails to start if the config option `app_service_config_files` is not a list. ([\#15425](https://github.com/matrix-org/synapse/issues/15425)) +- Disable loading `RefreshTokenServlet` (`/_matrix/client/(r0|v3|unstable)/refresh`) on workers. ([\#15428](https://github.com/matrix-org/synapse/issues/15428)) + + +Improved Documentation +---------------------- + +- Note that the `delete_stale_devices_after` background job always runs on the main process. ([\#15452](https://github.com/matrix-org/synapse/issues/15452)) + + +Deprecations and Removals +------------------------- + +- Remove the broken, unspecced registration fallback. Note that the *login* fallback is unaffected by this change. ([\#15405](https://github.com/matrix-org/synapse/issues/15405)) + + +Internal Changes +---------------- + +- Bump black from 23.1.0 to 23.3.0. ([\#15372](https://github.com/matrix-org/synapse/issues/15372)) +- Bump pyopenssl from 23.1.0 to 23.1.1. ([\#15373](https://github.com/matrix-org/synapse/issues/15373)) +- Bump types-psycopg2 from 2.9.21.8 to 2.9.21.9. ([\#15374](https://github.com/matrix-org/synapse/issues/15374)) +- Bump types-netaddr from 0.8.0.6 to 0.8.0.7. ([\#15375](https://github.com/matrix-org/synapse/issues/15375)) +- Bump types-opentracing from 2.4.10.3 to 2.4.10.4. ([\#15376](https://github.com/matrix-org/synapse/issues/15376)) +- Bump dawidd6/action-download-artifact from 2.26.0 to 2.26.1. ([\#15404](https://github.com/matrix-org/synapse/issues/15404)) +- Bump parameterized from 0.8.1 to 0.9.0. ([\#15412](https://github.com/matrix-org/synapse/issues/15412)) +- Bump types-pillow from 9.4.0.17 to 9.4.0.19. ([\#15413](https://github.com/matrix-org/synapse/issues/15413)) +- Bump sentry-sdk from 1.17.0 to 1.19.1. ([\#15414](https://github.com/matrix-org/synapse/issues/15414)) +- Bump immutabledict from 2.2.3 to 2.2.4. ([\#15415](https://github.com/matrix-org/synapse/issues/15415)) +- Bump dawidd6/action-download-artifact from 2.26.1 to 2.27.0. ([\#15441](https://github.com/matrix-org/synapse/issues/15441)) +- Bump serde_json from 1.0.95 to 1.0.96. ([\#15442](https://github.com/matrix-org/synapse/issues/15442)) +- Bump serde from 1.0.159 to 1.0.160. ([\#15443](https://github.com/matrix-org/synapse/issues/15443)) +- Bump pillow from 9.4.0 to 9.5.0. ([\#15444](https://github.com/matrix-org/synapse/issues/15444)) +- Bump furo from 2023.3.23 to 2023.3.27. ([\#15445](https://github.com/matrix-org/synapse/issues/15445)) +- Bump types-pyopenssl from 23.1.0.0 to 23.1.0.2. ([\#15446](https://github.com/matrix-org/synapse/issues/15446)) +- Bump mypy from 1.0.0 to 1.0.1. ([\#15447](https://github.com/matrix-org/synapse/issues/15447)) +- Bump psycopg2 from 2.9.5 to 2.9.6. ([\#15448](https://github.com/matrix-org/synapse/issues/15448)) +- Improve DB performance of clearing out old data from `stream_ordering_to_exterm`. ([\#15382](https://github.com/matrix-org/synapse/issues/15382), [\#15429](https://github.com/matrix-org/synapse/issues/15429)) +- Implement [MSC3989](https://github.com/matrix-org/matrix-spec-proposals/pull/3989) redaction algorithm. ([\#15393](https://github.com/matrix-org/synapse/issues/15393)) +- Implement [MSC2175](https://github.com/matrix-org/matrix-doc/pull/2175) to stop adding `creator` to create events. ([\#15394](https://github.com/matrix-org/synapse/issues/15394)) +- Implement [MSC2174](https://github.com/matrix-org/matrix-spec-proposals/pull/2174) to move the `redacts` key to a `content` property. ([\#15395](https://github.com/matrix-org/synapse/issues/15395)) +- Trust dtonlay/rust-toolchain in CI. ([\#15406](https://github.com/matrix-org/synapse/issues/15406)) +- Explicitly install Synapse during typechecking in CI. ([\#15409](https://github.com/matrix-org/synapse/issues/15409)) +- Only load the SSO redirect servlet if SSO is enabled. ([\#15421](https://github.com/matrix-org/synapse/issues/15421)) +- Refactor `SimpleHttpClient` to pull out a base class. ([\#15427](https://github.com/matrix-org/synapse/issues/15427)) +- Improve type hints. ([\#15432](https://github.com/matrix-org/synapse/issues/15432)) +- Convert async to normal tests in `TestSSOHandler`. ([\#15433](https://github.com/matrix-org/synapse/issues/15433)) +- Speed up the user directory background update. ([\#15435](https://github.com/matrix-org/synapse/issues/15435)) +- Disable directory listing for static resources in `/_matrix/static/`. ([\#15438](https://github.com/matrix-org/synapse/issues/15438)) +- Move various module API callback registration methods to a dedicated class. ([\#15453](https://github.com/matrix-org/synapse/issues/15453)) + + +Synapse 1.81.0 (2023-04-11) +=========================== + +Synapse now attempts the versioned appservice paths before falling back to the +[legacy paths](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). +Usage of the legacy routes should be considered deprecated. + +Additionally, Synapse has supported sending the application service access token +via [the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization) +since v1.70.0. For backwards compatibility it is *also* sent as the `access_token` +query parameter. This is insecure and should be considered deprecated. + +A future version of Synapse (v1.88.0 or later) will remove support for legacy +application service routes and query parameter authorization. + + +No significant changes since 1.81.0rc2. + + +Synapse 1.81.0rc2 (2023-04-06) +============================== + +Bugfixes +-------- + +- Fix the `set_device_id_for_pushers_txn` background update crash. ([\#15391](https://github.com/matrix-org/synapse/issues/15391)) + + +Internal Changes +---------------- + +- Update CI to run complement under the latest stable go version. ([\#15403](https://github.com/matrix-org/synapse/issues/15403)) + + +Synapse 1.81.0rc1 (2023-04-04) +============================== + +Features +-------- + +- Add the ability to enable/disable registrations when in the OIDC flow. ([\#14978](https://github.com/matrix-org/synapse/issues/14978)) +- Add a primitive helper script for listing worker endpoints. ([\#15243](https://github.com/matrix-org/synapse/issues/15243)) +- Experimental support for passing One Time Key and device key requests to application services ([MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) and [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984)). ([\#15314](https://github.com/matrix-org/synapse/issues/15314), [\#15321](https://github.com/matrix-org/synapse/issues/15321)) +- Allow loading `/password_policy` endpoint on workers. ([\#15331](https://github.com/matrix-org/synapse/issues/15331)) +- Add experimental support for Unix sockets. Contributed by Jason Little. ([\#15353](https://github.com/matrix-org/synapse/issues/15353)) +- Build Debian packages for Ubuntu 23.04 (Lunar Lobster). ([\#15381](https://github.com/matrix-org/synapse/issues/15381)) + + +Bugfixes +-------- + +- Fix a long-standing bug where edits of non-`m.room.message` events would not be correctly bundled. ([\#15295](https://github.com/matrix-org/synapse/issues/15295)) +- Fix a bug introduced in Synapse v1.55.0 which could delay remote homeservers being able to decrypt encrypted messages sent by local users. ([\#15297](https://github.com/matrix-org/synapse/issues/15297)) +- Add a check to [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) + to ensure that the sqlite database passed to the script exists before trying to port from it. ([\#15306](https://github.com/matrix-org/synapse/issues/15306)) +- Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key. ([\#15309](https://github.com/matrix-org/synapse/issues/15309)) +- Fix a long-standing bug that Synpase only used the [legacy appservice routes](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). ([\#15317](https://github.com/matrix-org/synapse/issues/15317)) +- Fix a long-standing bug preventing users from rejoining rooms after being banned and unbanned over federation. Contributed by Nico. ([\#15323](https://github.com/matrix-org/synapse/issues/15323)) +- Fix bug in worker mode where on a rolling restart of workers the "typing" worker would consume 100% CPU until it got restarted. ([\#15332](https://github.com/matrix-org/synapse/issues/15332)) +- Fix a long-standing bug where some to_device messages could be dropped when using workers. ([\#15349](https://github.com/matrix-org/synapse/issues/15349)) +- Fix a bug introduced in Synapse 1.70.0 where the background sync from a faster join could spin for hours when one of the events involved had been marked for backoff. ([\#15351](https://github.com/matrix-org/synapse/issues/15351)) +- Fix missing app variable in mail subject for password resets. Contributed by Cyberes. ([\#15352](https://github.com/matrix-org/synapse/issues/15352)) +- Fix a rare bug introduced in Synapse 1.66.0 where initial syncs would fail when the user had been kicked from a faster joined room that had not finished syncing. ([\#15383](https://github.com/matrix-org/synapse/issues/15383)) + + +Improved Documentation +---------------------- + +- Fix a typo in login requests ratelimit defaults. ([\#15341](https://github.com/matrix-org/synapse/issues/15341)) +- Add some clarification to the doc/comments regarding TCP replication. ([\#15354](https://github.com/matrix-org/synapse/issues/15354)) +- Note that Synapse 1.74 queued a rebuild of the user directory tables. ([\#15386](https://github.com/matrix-org/synapse/issues/15386)) + + +Internal Changes +---------------- + +- Use `immutabledict` instead of `frozendict`. ([\#15113](https://github.com/matrix-org/synapse/issues/15113)) +- Add developer documentation for the Federation Sender and add a documentation mechanism using Sphinx. ([\#15265](https://github.com/matrix-org/synapse/issues/15265), [\#15336](https://github.com/matrix-org/synapse/issues/15336)) +- Make the pushers rely on the `device_id` instead of the `access_token_id` for various operations. ([\#15280](https://github.com/matrix-org/synapse/issues/15280)) +- Bump sentry-sdk from 1.15.0 to 1.17.0. ([\#15285](https://github.com/matrix-org/synapse/issues/15285)) +- Allow running the Twisted trunk job against other branches. ([\#15302](https://github.com/matrix-org/synapse/issues/15302)) +- Remind the releaser to ask for changelog feedback in [#synapse-dev](https://matrix.to/#/#synapse-dev:matrix.org). ([\#15303](https://github.com/matrix-org/synapse/issues/15303)) +- Bump dtolnay/rust-toolchain from e12eda571dc9a5ee5d58eecf4738ec291c66f295 to fc3253060d0c959bea12a59f10f8391454a0b02d. ([\#15304](https://github.com/matrix-org/synapse/issues/15304)) +- Reject events with an invalid "mentions" property per [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15311](https://github.com/matrix-org/synapse/issues/15311)) +- As an optimisation, use `TRUNCATE` on Postgres when clearing the user directory tables. ([\#15316](https://github.com/matrix-org/synapse/issues/15316)) +- Fix `.gitignore` rule for the Complement source tarball downloaded automatically by `complement.sh`. ([\#15319](https://github.com/matrix-org/synapse/issues/15319)) +- Bump serde from 1.0.157 to 1.0.158. ([\#15324](https://github.com/matrix-org/synapse/issues/15324)) +- Bump regex from 1.7.1 to 1.7.3. ([\#15325](https://github.com/matrix-org/synapse/issues/15325)) +- Bump types-pyopenssl from 23.0.0.4 to 23.1.0.0. ([\#15326](https://github.com/matrix-org/synapse/issues/15326)) +- Bump furo from 2022.12.7 to 2023.3.23. ([\#15327](https://github.com/matrix-org/synapse/issues/15327)) +- Bump ruff from 0.0.252 to 0.0.259. ([\#15328](https://github.com/matrix-org/synapse/issues/15328)) +- Bump cryptography from 40.0.0 to 40.0.1. ([\#15329](https://github.com/matrix-org/synapse/issues/15329)) +- Bump mypy-zope from 0.9.0 to 0.9.1. ([\#15330](https://github.com/matrix-org/synapse/issues/15330)) +- Speed up unit tests when using SQLite3. ([\#15334](https://github.com/matrix-org/synapse/issues/15334)) +- Speed up pydantic CI job. ([\#15339](https://github.com/matrix-org/synapse/issues/15339)) +- Speed up sample config CI job. ([\#15340](https://github.com/matrix-org/synapse/issues/15340)) +- Fix copyright year in SSO footer template. ([\#15358](https://github.com/matrix-org/synapse/issues/15358)) +- Bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3. ([\#15369](https://github.com/matrix-org/synapse/issues/15369)) +- Bump serde from 1.0.158 to 1.0.159. ([\#15370](https://github.com/matrix-org/synapse/issues/15370)) +- Bump serde_json from 1.0.94 to 1.0.95. ([\#15371](https://github.com/matrix-org/synapse/issues/15371)) +- Speed up membership queries for users with forgotten rooms. ([\#15385](https://github.com/matrix-org/synapse/issues/15385)) + + +Synapse 1.80.0 (2023-03-28) +=========================== + +No significant changes since 1.80.0rc2. + + +Synapse 1.80.0rc2 (2023-03-22) +============================== + +Bugfixes +-------- + +- Fix a bug in which the [`POST /_matrix/client/v3/rooms/{roomId}/report/{eventId}`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3roomsroomidreporteventid) endpoint would return the wrong error if the user did not have permission to view the event. This aligns Synapse's implementation with [MSC2249](https://github.com/matrix-org/matrix-spec-proposals/pull/2249). ([\#15298](https://github.com/matrix-org/synapse/issues/15298), [\#15300](https://github.com/matrix-org/synapse/issues/15300)) +- Fix a bug introduced in Synapse 1.75.0rc1 where the [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) + would fail to open the SQLite database. ([\#15301](https://github.com/matrix-org/synapse/issues/15301)) + + +Synapse 1.80.0rc1 (2023-03-21) +============================== + +Features +-------- + +- Stabilise support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): `event_property_contains` push condition. ([\#15187](https://github.com/matrix-org/synapse/issues/15187)) +- Implement [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15249](https://github.com/matrix-org/synapse/issues/15249)) +- Allow loading `/register/available` endpoint on workers. ([\#15268](https://github.com/matrix-org/synapse/issues/15268)) +- Improve performance of creating and authenticating events. ([\#15195](https://github.com/matrix-org/synapse/issues/15195)) +- Add topic and name events to group of events that are batch persisted when creating a room. ([\#15229](https://github.com/matrix-org/synapse/issues/15229)) + + +Bugfixes +-------- + +- Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change. ([\#14755](https://github.com/matrix-org/synapse/issues/14755), [\#14756](https://github.com/matrix-org/synapse/issues/14756)) +- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to fix a long-standing bug where properties with dots were handled ambiguously in push rules. ([\#15190](https://github.com/matrix-org/synapse/issues/15190)) +- Faster joins: Fix a bug introduced in Synapse 1.66 where spurious "Failed to find memberships ..." errors would be logged. ([\#15232](https://github.com/matrix-org/synapse/issues/15232)) +- Fix a long-standing error when sending message into deleted room. ([\#15235](https://github.com/matrix-org/synapse/issues/15235)) + + +Updates to the Docker image +--------------------------- + +- Ensure the Dockerfile builds on platforms that don't have a `cryptography` wheel. ([\#15239](https://github.com/matrix-org/synapse/issues/15239)) +- Mirror images to the GitHub Container Registry (`ghcr.io/matrix-org/synapse`). ([\#15281](https://github.com/matrix-org/synapse/issues/15281), [\#15282](https://github.com/matrix-org/synapse/issues/15282)) + + +Improved Documentation +---------------------- + +- Add a missing endpoint to the workers documentation. ([\#15223](https://github.com/matrix-org/synapse/issues/15223)) + + +Internal Changes +---------------- + +- Add additional functionality to declaring worker types when starting Complement in worker mode. ([\#14921](https://github.com/matrix-org/synapse/issues/14921)) +- Add `Synapse-Trace-Id` to `access-control-expose-headers` header. ([\#14974](https://github.com/matrix-org/synapse/issues/14974)) +- Make the `HttpTransactionCache` use the `Requester` in addition of the just the `Request` to build the transaction key. ([\#15200](https://github.com/matrix-org/synapse/issues/15200)) +- Improve log lines when purging rooms. ([\#15222](https://github.com/matrix-org/synapse/issues/15222)) +- Improve type hints. ([\#15230](https://github.com/matrix-org/synapse/issues/15230), [\#15231](https://github.com/matrix-org/synapse/issues/15231), [\#15238](https://github.com/matrix-org/synapse/issues/15238)) +- Move various module API callback registration methods to a dedicated class. ([\#15237](https://github.com/matrix-org/synapse/issues/15237)) +- Configure GitHub Actions for merge queues. ([\#15244](https://github.com/matrix-org/synapse/issues/15244)) +- Add schema comments about the `destinations` and `destination_rooms` tables. ([\#15247](https://github.com/matrix-org/synapse/issues/15247)) +- Skip processing of auto-join room behaviour if there are no auto-join rooms configured. ([\#15262](https://github.com/matrix-org/synapse/issues/15262)) +- Remove unused store method `_set_destination_retry_timings_emulated`. ([\#15266](https://github.com/matrix-org/synapse/issues/15266)) +- Reorganize URL preview code. ([\#15269](https://github.com/matrix-org/synapse/issues/15269)) +- Clean-up direct TCP replication code. ([\#15272](https://github.com/matrix-org/synapse/issues/15272), [\#15274](https://github.com/matrix-org/synapse/issues/15274)) +- Make `configure_workers_and_start` script used in Complement tests compatible with older versions of Python. ([\#15275](https://github.com/matrix-org/synapse/issues/15275)) +- Add a `/versions` flag for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15293](https://github.com/matrix-org/synapse/issues/15293)) +- Bump hiredis from 2.2.1 to 2.2.2. ([\#15252](https://github.com/matrix-org/synapse/issues/15252)) +- Bump serde from 1.0.152 to 1.0.155. ([\#15253](https://github.com/matrix-org/synapse/issues/15253)) +- Bump pysaml2 from 7.2.1 to 7.3.1. ([\#15254](https://github.com/matrix-org/synapse/issues/15254)) +- Bump msgpack from 1.0.4 to 1.0.5. ([\#15255](https://github.com/matrix-org/synapse/issues/15255)) +- Bump gitpython from 3.1.30 to 3.1.31. ([\#15256](https://github.com/matrix-org/synapse/issues/15256)) +- Bump cryptography from 39.0.1 to 39.0.2. ([\#15257](https://github.com/matrix-org/synapse/issues/15257)) +- Bump pydantic from 1.10.4 to 1.10.6. ([\#15286](https://github.com/matrix-org/synapse/issues/15286)) +- Bump serde from 1.0.155 to 1.0.157. ([\#15287](https://github.com/matrix-org/synapse/issues/15287)) +- Bump anyhow from 1.0.69 to 1.0.70. ([\#15288](https://github.com/matrix-org/synapse/issues/15288)) +- Bump txredisapi from 1.4.7 to 1.4.9. ([\#15289](https://github.com/matrix-org/synapse/issues/15289)) +- Bump pygithub from 1.57 to 1.58.1. ([\#15290](https://github.com/matrix-org/synapse/issues/15290)) +- Bump types-requests from 2.28.11.12 to 2.28.11.15. ([\#15291](https://github.com/matrix-org/synapse/issues/15291)) + + + +Synapse 1.79.0 (2023-03-14) +=========================== + +No significant changes since 1.79.0rc2. + + +Synapse 1.79.0rc2 (2023-03-13) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.79.0rc1 where attempting to register a `on_remove_user_third_party_identifier` module API callback would be a no-op. ([\#15227](https://github.com/matrix-org/synapse/issues/15227)) +- Fix a rare bug introduced in Synapse 1.73 where events could remain unsent to other homeservers after a faster-join to a room. ([\#15248](https://github.com/matrix-org/synapse/issues/15248)) + + +Internal Changes +---------------- + +- Refactor `filter_events_for_server`. ([\#15240](https://github.com/matrix-org/synapse/issues/15240)) + + +Synapse 1.79.0rc1 (2023-03-07) +============================== + +Features +-------- + +- Add two new Third Party Rules module API callbacks: [`on_add_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_add_user_third_party_identifier) and [`on_remove_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_remove_user_third_party_identifier). ([\#15044](https://github.com/matrix-org/synapse/issues/15044)) +- Experimental support for [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967) to not require UIA for setting up cross-signing on first use. ([\#15077](https://github.com/matrix-org/synapse/issues/15077)) +- Add media information to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.79/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#15107](https://github.com/matrix-org/synapse/issues/15107)) +- Add an [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) to delete a [specific event report](https://spec.matrix.org/v1.6/client-server-api/#reporting-content). ([\#15116](https://github.com/matrix-org/synapse/issues/15116)) +- Add support for knocking to workers. ([\#15133](https://github.com/matrix-org/synapse/issues/15133)) +- Allow use of the `/filter` Client-Server APIs on workers. ([\#15134](https://github.com/matrix-org/synapse/issues/15134)) +- Update support for [MSC2677](https://github.com/matrix-org/matrix-spec-proposals/pull/2677): remove support for server-side aggregation of reactions. ([\#15172](https://github.com/matrix-org/synapse/issues/15172)) +- Stabilise support for [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758): `event_property_is` push condition. ([\#15185](https://github.com/matrix-org/synapse/issues/15185)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.75 that caused experimental support for deleting account data to raise an internal server error while using an account data writer worker. ([\#14869](https://github.com/matrix-org/synapse/issues/14869)) +- Fix a long-standing bug where Synapse handled an unspecced field on push rules. ([\#15088](https://github.com/matrix-org/synapse/issues/15088)) +- Fix a long-standing bug where a URL preview would break if the discovered oEmbed failed to download. ([\#15092](https://github.com/matrix-org/synapse/issues/15092)) +- Fix a long-standing bug where an initial sync would not respond to changes to the list of ignored users if there was an initial sync cached. ([\#15163](https://github.com/matrix-org/synapse/issues/15163)) +- Add the `transaction_id` in the events included in many endpoints' responses. ([\#15174](https://github.com/matrix-org/synapse/issues/15174)) +- Fix a bug introduced in Synapse 1.78.0 where requests to claim dehydrated devices would fail with a `405` error. ([\#15180](https://github.com/matrix-org/synapse/issues/15180)) +- Stop applying edits when bundling aggregations, per [MSC3925](https://github.com/matrix-org/matrix-spec-proposals/pull/3925). ([\#15193](https://github.com/matrix-org/synapse/issues/15193)) +- Fix a long-standing bug where the user directory search was not case-insensitive for accented characters. ([\#15143](https://github.com/matrix-org/synapse/issues/15143)) + + +Updates to the Docker image +--------------------------- + +- Improve startup logging in the with-workers Docker image. ([\#15186](https://github.com/matrix-org/synapse/issues/15186)) + + +Improved Documentation +---------------------- + +- Document how to use caches in a module. ([\#14026](https://github.com/matrix-org/synapse/issues/14026)) +- Clarify which worker processes the ThirdPartyRules' [`on_new_event`](https://matrix-org.github.io/synapse/v1.78/modules/third_party_rules_callbacks.html#on_new_event) module API callback runs on. ([\#15071](https://github.com/matrix-org/synapse/issues/15071)) +- Document using [Shibboleth](https://www.shibboleth.net/) as an OpenID Provider. ([\#15112](https://github.com/matrix-org/synapse/issues/15112)) +- Correct reference to `federation_verify_certificates` in configuration documentation. ([\#15139](https://github.com/matrix-org/synapse/issues/15139)) +- Correct small documentation errors in some `MatrixFederationHttpClient` methods. ([\#15148](https://github.com/matrix-org/synapse/issues/15148)) +- Correct the description of the behavior of `registration_shared_secret_path` on startup. ([\#15168](https://github.com/matrix-org/synapse/issues/15168)) + + +Deprecations and Removals +------------------------- + +- Deprecate the `on_threepid_bind` module callback, to be replaced by [`on_add_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_add_user_third_party_identifier). See [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.79/docs/upgrade.md#upgrading-to-v1790). ([\#15044](https://github.com/matrix-org/synapse/issues/15044)) +- Remove the unspecced `room_alias` field from the [`/createRoom`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3createroom) response. ([\#15093](https://github.com/matrix-org/synapse/issues/15093)) +- Remove the unspecced `PUT` on the `/knock/{roomIdOrAlias}` endpoint. ([\#15189](https://github.com/matrix-org/synapse/issues/15189)) +- Remove the undocumented and unspecced `type` parameter to the `/thumbnail` endpoint. ([\#15137](https://github.com/matrix-org/synapse/issues/15137)) +- Remove unspecced and buggy `PUT` method on the unstable `/rooms//batch_send` endpoint. ([\#15199](https://github.com/matrix-org/synapse/issues/15199)) + + +Internal Changes +---------------- + +- Run the integration test suites with the asyncio reactor enabled in CI. ([\#14101](https://github.com/matrix-org/synapse/issues/14101)) +- Batch up storing state groups when creating a new room. ([\#14918](https://github.com/matrix-org/synapse/issues/14918)) +- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15051](https://github.com/matrix-org/synapse/issues/15051)) +- Refactor writing json data in `FileExfiltrationWriter`. ([\#15095](https://github.com/matrix-org/synapse/issues/15095)) +- Tighten the login ratelimit defaults. ([\#15135](https://github.com/matrix-org/synapse/issues/15135)) +- Fix a typo in an experimental config setting. ([\#15138](https://github.com/matrix-org/synapse/issues/15138)) +- Refactor the media modules. ([\#15146](https://github.com/matrix-org/synapse/issues/15146), [\#15175](https://github.com/matrix-org/synapse/issues/15175)) +- Improve type hints. ([\#15164](https://github.com/matrix-org/synapse/issues/15164)) +- Move `get_event_report` and `get_event_reports_paginate` from `RoomStore` to `RoomWorkerStore`. ([\#15165](https://github.com/matrix-org/synapse/issues/15165)) +- Remove dangling reference to being a reference implementation in docstring. ([\#15167](https://github.com/matrix-org/synapse/issues/15167)) +- Add an option to force a rebuild of the "editable" complement image. ([\#15184](https://github.com/matrix-org/synapse/issues/15184)) +- Use nightly rustfmt in CI. ([\#15188](https://github.com/matrix-org/synapse/issues/15188)) +- Add a `get_next_txn` method to `StreamIdGenerator` to match `MultiWriterIdGenerator`. ([\#15191](https://github.com/matrix-org/synapse/issues/15191)) +- Combine `AbstractStreamIdTracker` and `AbstractStreamIdGenerator`. ([\#15192](https://github.com/matrix-org/synapse/issues/15192)) +- Automatically fix errors with `ruff`. ([\#15194](https://github.com/matrix-org/synapse/issues/15194)) +- Refactor database transaction for query users' devices to reduce database pool contention. ([\#15215](https://github.com/matrix-org/synapse/issues/15215)) +- Correct `test_icu_word_boundary_punctuation` so that it passes with the ICU versions available in Alpine and macOS. ([\#15177](https://github.com/matrix-org/synapse/issues/15177)) + +
Locked dependency updates + + - Bump actions/checkout from 2 to 3. ([\#15155](https://github.com/matrix-org/synapse/issues/15155)) + - Bump black from 22.12.0 to 23.1.0. ([\#15103](https://github.com/matrix-org/synapse/issues/15103)) + - Bump dawidd6/action-download-artifact from 2.25.0 to 2.26.0. ([\#15152](https://github.com/matrix-org/synapse/issues/15152)) + - Bump docker/login-action from 1 to 2. ([\#15154](https://github.com/matrix-org/synapse/issues/15154)) + - Bump matrix-org/backend-meta from 1 to 2. ([\#15156](https://github.com/matrix-org/synapse/issues/15156)) + - Bump ruff from 0.0.237 to 0.0.252. ([\#15159](https://github.com/matrix-org/synapse/issues/15159)) + - Bump serde_json from 1.0.93 to 1.0.94. ([\#15214](https://github.com/matrix-org/synapse/issues/15214)) + - Bump types-commonmark from 0.9.2.1 to 0.9.2.2. ([\#15209](https://github.com/matrix-org/synapse/issues/15209)) + - Bump types-opentracing from 2.4.10.1 to 2.4.10.3. ([\#15158](https://github.com/matrix-org/synapse/issues/15158)) + - Bump types-pillow from 9.4.0.13 to 9.4.0.17. ([\#15211](https://github.com/matrix-org/synapse/issues/15211)) + - Bump types-psycopg2 from 2.9.21.4 to 2.9.21.8. ([\#15210](https://github.com/matrix-org/synapse/issues/15210)) + - Bump types-pyopenssl from 22.1.0.2 to 23.0.0.4. ([\#15213](https://github.com/matrix-org/synapse/issues/15213)) + - Bump types-setuptools from 67.3.0.1 to 67.4.0.3. ([\#15160](https://github.com/matrix-org/synapse/issues/15160)) + - Bump types-setuptools from 67.4.0.3 to 67.5.0.0. ([\#15212](https://github.com/matrix-org/synapse/issues/15212)) + - Bump typing-extensions from 4.4.0 to 4.5.0. ([\#15157](https://github.com/matrix-org/synapse/issues/15157)) +
+ + +Synapse 1.78.0 (2023-02-28) +=========================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.76 where 5s delays would occasionally occur in deployments using workers. ([\#15150](https://github.com/matrix-org/synapse/issues/15150)) + + +Synapse 1.78.0rc1 (2023-02-21) +============================== + +Features +-------- + +- Implement the experimental `exact_event_match` push rule condition from [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758). ([\#14964](https://github.com/matrix-org/synapse/issues/14964)) +- Add account data to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.78/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14969](https://github.com/matrix-org/synapse/issues/14969)) +- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to disambiguate push rule keys with dots in them. ([\#15004](https://github.com/matrix-org/synapse/issues/15004)) +- Allow Synapse to use a specific Redis [logical database](https://redis.io/commands/select/) in worker-mode deployments. ([\#15034](https://github.com/matrix-org/synapse/issues/15034)) +- Tag opentracing spans for federation requests with the name of the worker serving the request. ([\#15042](https://github.com/matrix-org/synapse/issues/15042)) +- Implement the experimental `exact_event_property_contains` push rule condition from [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966). ([\#15045](https://github.com/matrix-org/synapse/issues/15045)) +- Remove spurious `dont_notify` action from the defaults for the `.m.rule.reaction` pushrule. ([\#15073](https://github.com/matrix-org/synapse/issues/15073)) +- Update the error code returned when user sends a duplicate annotation. ([\#15075](https://github.com/matrix-org/synapse/issues/15075)) + + +Bugfixes +-------- + +- Prevent clients from reporting nonexistent events. ([\#13779](https://github.com/matrix-org/synapse/issues/13779)) +- Return spec-compliant JSON errors when unknown endpoints are requested. ([\#14605](https://github.com/matrix-org/synapse/issues/14605)) +- Fix a long-standing bug where the room aliases returned could be corrupted. ([\#15038](https://github.com/matrix-org/synapse/issues/15038)) +- Fix a bug introduced in Synapse 1.76.0 where partially-joined rooms could not be deleted using the [purge room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api). ([\#15068](https://github.com/matrix-org/synapse/issues/15068)) +- Fix a long-standing bug where federated joins would fail if the first server in the list of servers to try is not in the room. ([\#15074](https://github.com/matrix-org/synapse/issues/15074)) +- Fix a bug introduced in Synapse v1.74.0 where searching with colons when using ICU for search term tokenisation would fail with an error. ([\#15079](https://github.com/matrix-org/synapse/issues/15079)) +- Reduce the likelihood of a rare race condition where rejoining a restricted room over federation would fail. ([\#15080](https://github.com/matrix-org/synapse/issues/15080)) +- Fix a bug introduced in Synapse 1.76 where workers would fail to start if the `health` listener was configured. ([\#15096](https://github.com/matrix-org/synapse/issues/15096)) +- Fix a bug introduced in Synapse 1.75 where the [portdb script](https://matrix-org.github.io/synapse/release-v1.78/postgres.html#porting-from-sqlite) would fail to run after a room had been faster-joined. ([\#15108](https://github.com/matrix-org/synapse/issues/15108)) + + +Improved Documentation +---------------------- + +- Document how to start Synapse with Poetry. Contributed by @thezaidbintariq. ([\#14892](https://github.com/matrix-org/synapse/issues/14892), [\#15022](https://github.com/matrix-org/synapse/issues/15022)) +- Update delegation documentation to clarify that SRV DNS delegation does not eliminate all needs to serve files from .well-known locations. Contributed by @williamkray. ([\#14959](https://github.com/matrix-org/synapse/issues/14959)) +- Fix a mistake in registration_shared_secret_path docs. ([\#15078](https://github.com/matrix-org/synapse/issues/15078)) +- Refer to a more recent blog post on the [Database Maintenance Tools](https://matrix-org.github.io/synapse/latest/usage/administration/database_maintenance_tools.html) page. Contributed by @jahway603. ([\#15083](https://github.com/matrix-org/synapse/issues/15083)) + + +Internal Changes +---------------- + +- Re-type hint some collections as read-only. ([\#13755](https://github.com/matrix-org/synapse/issues/13755)) +- Faster joins: don't stall when another user joins during a partial-state room resync. ([\#14606](https://github.com/matrix-org/synapse/issues/14606)) +- Add a class `UnpersistedEventContext` to allow for the batching up of storing state groups. ([\#14675](https://github.com/matrix-org/synapse/issues/14675)) +- Add a check to ensure that locked dependencies have source distributions available. ([\#14742](https://github.com/matrix-org/synapse/issues/14742)) +- Tweak comment on `_is_local_room_accessible` as part of room visibility in `/hierarchy` to clarify the condition for a room being visible. ([\#14834](https://github.com/matrix-org/synapse/issues/14834)) +- Prevent `WARNING: there is already a transaction in progress` lines appearing in PostgreSQL's logs on some occasions. ([\#14840](https://github.com/matrix-org/synapse/issues/14840)) +- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14929](https://github.com/matrix-org/synapse/issues/14929)) +- Improve performance of `/sync` in a few situations. ([\#14973](https://github.com/matrix-org/synapse/issues/14973)) +- Limit concurrent event creation for a room to avoid state resolution when sending bursts of events to a local room. ([\#14977](https://github.com/matrix-org/synapse/issues/14977)) +- Skip calculating unread push actions in /sync when enable_push is false. ([\#14980](https://github.com/matrix-org/synapse/issues/14980)) +- Add a schema dump symlinks inside `contrib`, to make it easier for IDEs to interrogate Synapse's database schema. ([\#14982](https://github.com/matrix-org/synapse/issues/14982)) +- Improve type hints. ([\#15008](https://github.com/matrix-org/synapse/issues/15008), [\#15026](https://github.com/matrix-org/synapse/issues/15026), [\#15027](https://github.com/matrix-org/synapse/issues/15027), [\#15028](https://github.com/matrix-org/synapse/issues/15028), [\#15031](https://github.com/matrix-org/synapse/issues/15031), [\#15035](https://github.com/matrix-org/synapse/issues/15035), [\#15052](https://github.com/matrix-org/synapse/issues/15052), [\#15072](https://github.com/matrix-org/synapse/issues/15072), [\#15084](https://github.com/matrix-org/synapse/issues/15084)) +- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15037](https://github.com/matrix-org/synapse/issues/15037)) +- Avoid mutating a cached value in `get_user_devices_from_cache`. ([\#15040](https://github.com/matrix-org/synapse/issues/15040)) +- Fix a rare exception in logs on start up. ([\#15041](https://github.com/matrix-org/synapse/issues/15041)) +- Update pyo3-log to v0.8.1. ([\#15043](https://github.com/matrix-org/synapse/issues/15043)) +- Avoid mutating cached values in `_generate_sync_entry_for_account_data`. ([\#15047](https://github.com/matrix-org/synapse/issues/15047)) +- Refactor arguments of `try_unbind_threepid` and `_try_unbind_threepid_with_id_server` to not use dictionaries. ([\#15053](https://github.com/matrix-org/synapse/issues/15053)) +- Merge debug logging from the hotfixes branch. ([\#15054](https://github.com/matrix-org/synapse/issues/15054)) +- Faster joins: omit device list updates originating from partial state rooms in /sync responses without lazy loading of members enabled. ([\#15069](https://github.com/matrix-org/synapse/issues/15069)) +- Fix clashing database transaction name. ([\#15070](https://github.com/matrix-org/synapse/issues/15070)) +- Upper-bound frozendict dependency. This works around us being unable to test installing our wheels against Python 3.11 in CI. ([\#15114](https://github.com/matrix-org/synapse/issues/15114)) +- Tweak logging for when a worker waits for its view of a replication stream to catch up. ([\#15120](https://github.com/matrix-org/synapse/issues/15120)) + +
Locked dependency updates + +- Bump bleach from 5.0.1 to 6.0.0. ([\#15059](https://github.com/matrix-org/synapse/issues/15059)) +- Bump cryptography from 38.0.4 to 39.0.1. ([\#15020](https://github.com/matrix-org/synapse/issues/15020)) +- Bump ruff version from 0.0.230 to 0.0.237. ([\#15033](https://github.com/matrix-org/synapse/issues/15033)) +- Bump dtolnay/rust-toolchain from 9cd00a88a73addc8617065438eff914dd08d0955 to 25dc93b901a87e864900a8aec6c12e9aa794c0c3. ([\#15060](https://github.com/matrix-org/synapse/issues/15060)) +- Bump systemd-python from 234 to 235. ([\#15061](https://github.com/matrix-org/synapse/issues/15061)) +- Bump serde_json from 1.0.92 to 1.0.93. ([\#15062](https://github.com/matrix-org/synapse/issues/15062)) +- Bump types-requests from 2.28.11.8 to 2.28.11.12. ([\#15063](https://github.com/matrix-org/synapse/issues/15063)) +- Bump types-pillow from 9.4.0.5 to 9.4.0.10. ([\#15064](https://github.com/matrix-org/synapse/issues/15064)) +- Bump sentry-sdk from 1.13.0 to 1.15.0. ([\#15065](https://github.com/matrix-org/synapse/issues/15065)) +- Bump types-jsonschema from 4.17.0.3 to 4.17.0.5. ([\#15099](https://github.com/matrix-org/synapse/issues/15099)) +- Bump types-bleach from 5.0.3.1 to 6.0.0.0. ([\#15100](https://github.com/matrix-org/synapse/issues/15100)) +- Bump dtolnay/rust-toolchain from 25dc93b901a87e864900a8aec6c12e9aa794c0c3 to e12eda571dc9a5ee5d58eecf4738ec291c66f295. ([\#15101](https://github.com/matrix-org/synapse/issues/15101)) +- Bump dawidd6/action-download-artifact from 2.24.3 to 2.25.0. ([\#15102](https://github.com/matrix-org/synapse/issues/15102)) +- Bump types-pillow from 9.4.0.10 to 9.4.0.13. ([\#15104](https://github.com/matrix-org/synapse/issues/15104)) +- Bump types-setuptools from 67.1.0.0 to 67.3.0.1. ([\#15105](https://github.com/matrix-org/synapse/issues/15105)) + + +
+ + +Synapse 1.77.0 (2023-02-14) +=========================== + +No significant changes since 1.77.0rc2. + + +Synapse 1.77.0rc2 (2023-02-10) +============================== + +Bugfixes +-------- + +- Fix bug where retried replication requests would return a failure. Introduced in v1.76.0. ([\#15024](https://github.com/matrix-org/synapse/issues/15024)) + + +Internal Changes +---------------- + +- Prepare for future database schema changes. ([\#15036](https://github.com/matrix-org/synapse/issues/15036)) + + +Synapse 1.77.0rc1 (2023-02-07) +============================== + +Features +-------- + +- Experimental support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#14823](https://github.com/matrix-org/synapse/issues/14823), [\#14943](https://github.com/matrix-org/synapse/issues/14943), [\#14957](https://github.com/matrix-org/synapse/issues/14957), [\#14958](https://github.com/matrix-org/synapse/issues/14958)) +- Experimental support to suppress notifications from message edits ([MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958)). ([\#14960](https://github.com/matrix-org/synapse/issues/14960), [\#15016](https://github.com/matrix-org/synapse/issues/15016)) +- Add profile information, devices and connections to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.77/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14894](https://github.com/matrix-org/synapse/issues/14894)) +- Improve performance when joining or sending an event in large rooms. ([\#14962](https://github.com/matrix-org/synapse/issues/14962)) +- Improve performance of joining and leaving large rooms with many local users. ([\#14971](https://github.com/matrix-org/synapse/issues/14971)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.53.0 where `next_batch` tokens from `/sync` could not be used with the `/relations` endpoint. ([\#14866](https://github.com/matrix-org/synapse/issues/14866)) +- Fix a bug introduced in Synapse 1.35.0 where the module API's `send_local_online_presence_to` would fail to send presence updates over federation. ([\#14880](https://github.com/matrix-org/synapse/issues/14880)) +- Fix a bug introduced in Synapse 1.70.0 where the background updates to add non-thread unique indexes on receipts could fail when upgrading from 1.67.0 or earlier. ([\#14915](https://github.com/matrix-org/synapse/issues/14915)) +- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14926](https://github.com/matrix-org/synapse/issues/14926)) +- Fix a bug introduced in Synapse 1.68.0 where we were unable to service remote joins in rooms with `@room` notification levels set to `null` in their (malformed) power levels. ([\#14942](https://github.com/matrix-org/synapse/issues/14942)) +- Fix a bug introduced in Synapse 1.64.0 where boolean power levels were erroneously permitted in [v10 rooms](https://spec.matrix.org/v1.5/rooms/v10/). ([\#14944](https://github.com/matrix-org/synapse/issues/14944)) +- Fix a long-standing bug where sending messages on servers with presence enabled would spam "Re-starting finished log context" log lines. ([\#14947](https://github.com/matrix-org/synapse/issues/14947)) +- Fix a bug introduced in Synapse 1.68.0 where logging from the Rust module was not properly logged. ([\#14976](https://github.com/matrix-org/synapse/issues/14976)) +- Fix various long-standing bugs in Synapse's config, event and request handling where booleans were unintentionally accepted where an integer was expected. ([\#14945](https://github.com/matrix-org/synapse/issues/14945)) + + +Internal Changes +---------------- + +- Add missing type hints. ([\#14879](https://github.com/matrix-org/synapse/issues/14879), [\#14886](https://github.com/matrix-org/synapse/issues/14886), [\#14887](https://github.com/matrix-org/synapse/issues/14887), [\#14904](https://github.com/matrix-org/synapse/issues/14904), [\#14927](https://github.com/matrix-org/synapse/issues/14927), [\#14956](https://github.com/matrix-org/synapse/issues/14956), [\#14983](https://github.com/matrix-org/synapse/issues/14983), [\#14984](https://github.com/matrix-org/synapse/issues/14984), [\#14985](https://github.com/matrix-org/synapse/issues/14985), [\#14987](https://github.com/matrix-org/synapse/issues/14987), [\#14988](https://github.com/matrix-org/synapse/issues/14988), [\#14990](https://github.com/matrix-org/synapse/issues/14990), [\#14991](https://github.com/matrix-org/synapse/issues/14991), [\#14992](https://github.com/matrix-org/synapse/issues/14992), [\#15007](https://github.com/matrix-org/synapse/issues/15007)) +- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14922](https://github.com/matrix-org/synapse/issues/14922)) +- Allow running the complement tests suites with the asyncio reactor enabled. ([\#14858](https://github.com/matrix-org/synapse/issues/14858)) +- Improve performance of `/sync` in a few situations. ([\#14908](https://github.com/matrix-org/synapse/issues/14908), [\#14970](https://github.com/matrix-org/synapse/issues/14970)) +- Document how to handle Dependabot pull requests. ([\#14916](https://github.com/matrix-org/synapse/issues/14916)) +- Fix typo in release script. ([\#14920](https://github.com/matrix-org/synapse/issues/14920)) +- Update build system requirements to allow building with poetry-core 1.5.0. ([\#14949](https://github.com/matrix-org/synapse/issues/14949), [\#15019](https://github.com/matrix-org/synapse/issues/15019)) +- Add an [lnav](https://lnav.org) config file for Synapse logs to `/contrib/lnav`. ([\#14953](https://github.com/matrix-org/synapse/issues/14953)) +- Faster joins: Refactor internal handling of servers in room to never store an empty list. ([\#14954](https://github.com/matrix-org/synapse/issues/14954)) +- Faster joins: tag `v2/send_join/` requests to indicate if they served a partial join response. ([\#14950](https://github.com/matrix-org/synapse/issues/14950)) +- Allow running `cargo` without the `extension-module` option. ([\#14965](https://github.com/matrix-org/synapse/issues/14965)) +- Preparatory work for adding a denormalised event stream ordering column in the future. Contributed by Nick @ Beeper (@fizzadar). ([\#14979](https://github.com/matrix-org/synapse/issues/14979), [9cd7610](https://github.com/matrix-org/synapse/commit/9cd7610f86ab5051c9365dd38d1eec405a5f8ca6), [f10caa7](https://github.com/matrix-org/synapse/commit/f10caa73eee0caa91cf373966104d1ededae2aee); see [\#15014](https://github.com/matrix-org/synapse/issues/15014)) +- Add tests for `_flatten_dict`. ([\#14981](https://github.com/matrix-org/synapse/issues/14981), [\#15002](https://github.com/matrix-org/synapse/issues/15002)) + +
Locked dependency updates + +- Bump dtolnay/rust-toolchain from e645b0cf01249a964ec099494d38d2da0f0b349f to 9cd00a88a73addc8617065438eff914dd08d0955. ([\#14968](https://github.com/matrix-org/synapse/issues/14968)) +- Bump docker/build-push-action from 3 to 4. ([\#14952](https://github.com/matrix-org/synapse/issues/14952)) +- Bump ijson from 3.1.4 to 3.2.0.post0. ([\#14935](https://github.com/matrix-org/synapse/issues/14935)) +- Bump types-pyyaml from 6.0.12.2 to 6.0.12.3. ([\#14936](https://github.com/matrix-org/synapse/issues/14936)) +- Bump types-jsonschema from 4.17.0.2 to 4.17.0.3. ([\#14937](https://github.com/matrix-org/synapse/issues/14937)) +- Bump types-pillow from 9.4.0.3 to 9.4.0.5. ([\#14938](https://github.com/matrix-org/synapse/issues/14938)) +- Bump hiredis from 2.0.0 to 2.1.1. ([\#14939](https://github.com/matrix-org/synapse/issues/14939)) +- Bump hiredis from 2.1.1 to 2.2.1. ([\#14993](https://github.com/matrix-org/synapse/issues/14993)) +- Bump types-setuptools from 65.6.0.3 to 67.1.0.0. ([\#14994](https://github.com/matrix-org/synapse/issues/14994)) +- Bump prometheus-client from 0.15.0 to 0.16.0. ([\#14995](https://github.com/matrix-org/synapse/issues/14995)) +- Bump anyhow from 1.0.68 to 1.0.69. ([\#14996](https://github.com/matrix-org/synapse/issues/14996)) +- Bump serde_json from 1.0.91 to 1.0.92. ([\#14997](https://github.com/matrix-org/synapse/issues/14997)) +- Bump isort from 5.11.4 to 5.11.5. ([\#14998](https://github.com/matrix-org/synapse/issues/14998)) +- Bump phonenumbers from 8.13.4 to 8.13.5. ([\#14999](https://github.com/matrix-org/synapse/issues/14999)) +
+ +Synapse 1.76.0 (2023-01-31) +=========================== + +The 1.76 release is the first to enable faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706) and [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902)) by default. Admins can opt-out: see [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#faster-joins-are-enabled-by-default) for more details. + +The upgrade from 1.75 to 1.76 changes the account data replication streams in a backwards-incompatible manner. Server operators running a multi-worker deployment should consult [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#changes-to-the-account-data-replication-streams). + +Those who are `poetry install`ing from source using our lockfile should ensure their poetry version is 1.3.2 or higher; [see upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#minimum-version-of-poetry-is-now-132). + + +Notes on faster joins +--------------------- + +The faster joins project sees the most benefit when joining a room with a large number of members (joined or historical). We expect it to be particularly useful for joining large public rooms like the [Matrix HQ](https://matrix.to/#/#matrix:matrix.org) or [Synapse Admins](https://matrix.to/#/#synapse:matrix.org) rooms. + +After a faster join, Synapse considers that room "partially joined". In this state, you should be able to + +- read incoming messages; +- see incoming state changes, e.g. room topic changes; and +- send messages, if the room is unencrypted. + +Synapse has to spend more effort to complete the join in the background. Once this finishes, you will be able to + +- send messages, if the room is in encrypted; +- retrieve room history from before your join, if permitted by the room settings; and +- access the full list of room members. + + +Improved Documentation +---------------------- + +- Describe the ideas and the internal machinery behind faster joins. ([\#14677](https://github.com/matrix-org/synapse/issues/14677)) + + +Synapse 1.76.0rc2 (2023-01-27) +============================== + +Bugfixes +-------- + +- Faster joins: Fix a bug introduced in Synapse 1.69 where device list EDUs could fail to be handled after a restart when a faster join sync is in progress. ([\#14914](https://github.com/matrix-org/synapse/issues/14914)) + + +Internal Changes +---------------- + +- Faster joins: Improve performance of looking up partial-state status of rooms. ([\#14917](https://github.com/matrix-org/synapse/issues/14917)) + + +Synapse 1.76.0rc1 (2023-01-25) +============================== + +Features +-------- + +- Update the default room version to [v10](https://spec.matrix.org/v1.5/rooms/v10/) ([MSC 3904](https://github.com/matrix-org/matrix-spec-proposals/pull/3904)). Contributed by @FSG-Cat. ([\#14111](https://github.com/matrix-org/synapse/issues/14111)) +- Add a `set_displayname()` method to the module API for setting a user's display name. ([\#14629](https://github.com/matrix-org/synapse/issues/14629)) +- Add a dedicated listener configuration for `health` endpoint. ([\#14747](https://github.com/matrix-org/synapse/issues/14747)) +- Implement support for [MSC3890](https://github.com/matrix-org/matrix-spec-proposals/pull/3890): Remotely silence local notifications. ([\#14775](https://github.com/matrix-org/synapse/issues/14775)) +- Implement experimental support for [MSC3930](https://github.com/matrix-org/matrix-spec-proposals/pull/3930): Push rules for ([MSC3381](https://github.com/matrix-org/matrix-spec-proposals/pull/3381)) Polls. ([\#14787](https://github.com/matrix-org/synapse/issues/14787)) +- Per [MSC3925](https://github.com/matrix-org/matrix-spec-proposals/pull/3925), bundle the whole of the replacement with any edited events, and optionally inhibit server-side replacement. ([\#14811](https://github.com/matrix-org/synapse/issues/14811)) +- Faster joins: always serve a partial join response to servers that request it with the stable query param. ([\#14839](https://github.com/matrix-org/synapse/issues/14839)) +- Faster joins: allow non-lazy-loading ("eager") syncs to complete after a partial join by omitting partial state rooms until they become fully stated. ([\#14870](https://github.com/matrix-org/synapse/issues/14870)) +- Faster joins: request partial joins by default. Admins can opt-out of this for the time being---see the upgrade notes. ([\#14905](https://github.com/matrix-org/synapse/issues/14905)) + + +Bugfixes +-------- + +- Add index to improve performance of the `/timestamp_to_event` endpoint used for jumping to a specific date in the timeline of a room. ([\#14799](https://github.com/matrix-org/synapse/issues/14799)) +- Fix a long-standing bug where Synapse would exhaust the stack when processing many federation requests where the remote homeserver has disconencted early. ([\#14812](https://github.com/matrix-org/synapse/issues/14812), [\#14842](https://github.com/matrix-org/synapse/issues/14842)) +- Fix rare races when using workers. ([\#14820](https://github.com/matrix-org/synapse/issues/14820)) +- Fix a bug introduced in Synapse 1.64.0 when using room version 10 with frozen events enabled. ([\#14864](https://github.com/matrix-org/synapse/issues/14864)) +- Fix a long-standing bug where the `populate_room_stats` background job could fail on broken rooms. ([\#14873](https://github.com/matrix-org/synapse/issues/14873)) +- Faster joins: Fix a bug in worker deployments where the room stats and user directory would not get updated when finishing a fast join until another event is sent or received. ([\#14874](https://github.com/matrix-org/synapse/issues/14874)) +- Faster joins: Fix incompatibility with joins into restricted rooms where no local users have the ability to invite. ([\#14882](https://github.com/matrix-org/synapse/issues/14882)) +- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14910](https://github.com/matrix-org/synapse/issues/14910)) + + +Updates to the Docker image +--------------------------- + +- Bump default Python version in the Dockerfile from 3.9 to 3.11. ([\#14875](https://github.com/matrix-org/synapse/issues/14875)) + + +Improved Documentation +---------------------- + +- Include `x_forwarded` entry in the HTTP listener example configs and remove the remaining `worker_main_http_uri` entries. ([\#14667](https://github.com/matrix-org/synapse/issues/14667)) +- Remove duplicate commands from the Code Style documentation page; point to the Contributing Guide instead. ([\#14773](https://github.com/matrix-org/synapse/issues/14773)) +- Add missing documentation for `tag` to `listeners` section. ([\#14803](https://github.com/matrix-org/synapse/issues/14803)) +- Updated documentation in configuration manual for `user_directory.search_all_users`. ([\#14818](https://github.com/matrix-org/synapse/issues/14818)) +- Add `worker_manhole` to configuration manual. ([\#14824](https://github.com/matrix-org/synapse/issues/14824)) +- Fix the example config missing the `id` field in [application service documentation](https://matrix-org.github.io/synapse/latest/application_services.html). ([\#14845](https://github.com/matrix-org/synapse/issues/14845)) +- Minor corrections to the logging configuration documentation. ([\#14868](https://github.com/matrix-org/synapse/issues/14868)) +- Document the export user data command. Contributed by @thezaidbintariq. ([\#14883](https://github.com/matrix-org/synapse/issues/14883)) + + +Deprecations and Removals +------------------------- + +- Poetry 1.3.2 or higher is now required when `poetry install`ing from source. ([\#14860](https://github.com/matrix-org/synapse/issues/14860)) + + +Internal Changes +---------------- + +- Faster remote room joins (worker mode): do not populate external hosts-in-room cache when sending events as this requires blocking for full state. ([\#14749](https://github.com/matrix-org/synapse/issues/14749)) +- Enable Complement tests for Faster Remote Room Joins against worker-mode Synapse. ([\#14752](https://github.com/matrix-org/synapse/issues/14752)) +- Add some clarifying comments and refactor a portion of the `Keyring` class for readability. ([\#14804](https://github.com/matrix-org/synapse/issues/14804)) +- Add local poetry config files (`poetry.toml`) to `.gitignore`. ([\#14807](https://github.com/matrix-org/synapse/issues/14807)) +- Add missing type hints. ([\#14816](https://github.com/matrix-org/synapse/issues/14816), [\#14885](https://github.com/matrix-org/synapse/issues/14885), [\#14889](https://github.com/matrix-org/synapse/issues/14889)) +- Refactor push tests. ([\#14819](https://github.com/matrix-org/synapse/issues/14819)) +- Re-enable some linting that was disabled when we switched to ruff. ([\#14821](https://github.com/matrix-org/synapse/issues/14821)) +- Add `cargo fmt` and `cargo clippy` to the lint script. ([\#14822](https://github.com/matrix-org/synapse/issues/14822)) +- Drop unused table `presence`. ([\#14825](https://github.com/matrix-org/synapse/issues/14825)) +- Merge the two account data and the two device list replication streams. ([\#14826](https://github.com/matrix-org/synapse/issues/14826), [\#14833](https://github.com/matrix-org/synapse/issues/14833)) +- Faster joins: use stable identifiers from [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). ([\#14832](https://github.com/matrix-org/synapse/issues/14832), [\#14841](https://github.com/matrix-org/synapse/issues/14841)) +- Add a parameter to control whether the federation client performs a partial state join. ([\#14843](https://github.com/matrix-org/synapse/issues/14843)) +- Add check to avoid starting duplicate partial state syncs. ([\#14844](https://github.com/matrix-org/synapse/issues/14844)) +- Add an early return when handling no-op presence updates. ([\#14855](https://github.com/matrix-org/synapse/issues/14855)) +- Fix `wait_for_stream_position` to correctly wait for the right instance to advance its token. ([\#14856](https://github.com/matrix-org/synapse/issues/14856), [\#14872](https://github.com/matrix-org/synapse/issues/14872)) +- Always notify replication when a stream advances automatically. ([\#14877](https://github.com/matrix-org/synapse/issues/14877)) +- Reduce max time we wait for stream positions. ([\#14881](https://github.com/matrix-org/synapse/issues/14881)) +- Faster joins: allow the resync process more time to fetch `/state` ids. ([\#14912](https://github.com/matrix-org/synapse/issues/14912)) +- Bump regex from 1.7.0 to 1.7.1. ([\#14848](https://github.com/matrix-org/synapse/issues/14848)) +- Bump peaceiris/actions-gh-pages from 3.9.1 to 3.9.2. ([\#14861](https://github.com/matrix-org/synapse/issues/14861)) +- Bump ruff from 0.0.215 to 0.0.224. ([\#14862](https://github.com/matrix-org/synapse/issues/14862)) +- Bump types-pillow from 9.4.0.0 to 9.4.0.3. ([\#14863](https://github.com/matrix-org/synapse/issues/14863)) +- Bump types-opentracing from 2.4.10 to 2.4.10.1. ([\#14896](https://github.com/matrix-org/synapse/issues/14896)) +- Bump ruff from 0.0.224 to 0.0.230. ([\#14897](https://github.com/matrix-org/synapse/issues/14897)) +- Bump types-requests from 2.28.11.7 to 2.28.11.8. ([\#14899](https://github.com/matrix-org/synapse/issues/14899)) +- Bump types-psycopg2 from 2.9.21.2 to 2.9.21.4. ([\#14900](https://github.com/matrix-org/synapse/issues/14900)) +- Bump types-commonmark from 0.9.2 to 0.9.2.1. ([\#14901](https://github.com/matrix-org/synapse/issues/14901)) + + +Synapse 1.75.0 (2023-01-17) +=========================== + +No significant changes since 1.75.0rc2. + + +Synapse 1.75.0rc2 (2023-01-12) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.75.0rc1 where device lists could be miscalculated with some sync filters. ([\#14810](https://github.com/matrix-org/synapse/issues/14810)) +- Fix race where calling `/members` or `/state` with an `at` parameter could fail for newly created rooms, when using multiple workers. ([\#14817](https://github.com/matrix-org/synapse/issues/14817)) + + +Synapse 1.75.0rc1 (2023-01-10) +============================== + +Features +-------- + +- Add a `cached` function to `synapse.module_api` that returns a decorator to cache return values of functions. ([\#14663](https://github.com/matrix-org/synapse/issues/14663)) +- Add experimental support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) (removing account data). ([\#14714](https://github.com/matrix-org/synapse/issues/14714)) +- Support [RFC7636](https://datatracker.ietf.org/doc/html/rfc7636) Proof Key for Code Exchange for OAuth single sign-on. ([\#14750](https://github.com/matrix-org/synapse/issues/14750)) +- Support non-OpenID compliant userinfo claims for subject and picture. ([\#14753](https://github.com/matrix-org/synapse/issues/14753)) +- Improve performance of `/sync` when filtering all rooms, message types, or senders. ([\#14786](https://github.com/matrix-org/synapse/issues/14786)) +- Improve performance of the `/hierarchy` endpoint. ([\#14263](https://github.com/matrix-org/synapse/issues/14263)) + + +Bugfixes +-------- + +- Fix the *MAU Limits* section of the Grafana dashboard relying on a specific `job` name for the workers of a Synapse deployment. ([\#14644](https://github.com/matrix-org/synapse/issues/14644)) +- Fix a bug introduced in Synapse 1.70.0 which could cause spurious `UNIQUE constraint failed` errors in the `rotate_notifs` background job. ([\#14669](https://github.com/matrix-org/synapse/issues/14669)) +- Ensure stream IDs are always updated after caches get invalidated with workers. Contributed by Nick @ Beeper (@fizzadar). ([\#14723](https://github.com/matrix-org/synapse/issues/14723)) +- Remove the unspecced `device` field from `/pushrules` responses. ([\#14727](https://github.com/matrix-org/synapse/issues/14727)) +- Fix a bug introduced in Synapse 1.73.0 where the `picture_claim` configured under `oidc_providers` was unused (the default value of `"picture"` was used instead). ([\#14751](https://github.com/matrix-org/synapse/issues/14751)) +- Unescape HTML entities in URL preview titles making use of oEmbed responses. ([\#14781](https://github.com/matrix-org/synapse/issues/14781)) +- Disable sending confirmation email when 3pid is disabled. ([\#14725](https://github.com/matrix-org/synapse/issues/14725)) + + +Improved Documentation +---------------------- + +- Declare support for Python 3.11. ([\#14673](https://github.com/matrix-org/synapse/issues/14673)) +- Fix `target_memory_usage` being used in the description for the actual `cache_autotune` sub-option `target_cache_memory_usage`. ([\#14674](https://github.com/matrix-org/synapse/issues/14674)) +- Move `email` to Server section in config file documentation. ([\#14730](https://github.com/matrix-org/synapse/issues/14730)) +- Fix broken links in the Synapse documentation. ([\#14744](https://github.com/matrix-org/synapse/issues/14744)) +- Add missing worker settings to shared configuration documentation. ([\#14748](https://github.com/matrix-org/synapse/issues/14748)) +- Document using Twitter as a OAuth 2.0 authentication provider. ([\#14778](https://github.com/matrix-org/synapse/issues/14778)) +- Fix Synapse 1.74 upgrade notes to correctly explain how to install pyICU when installing Synapse from PyPI. ([\#14797](https://github.com/matrix-org/synapse/issues/14797)) +- Update link to towncrier in contribution guide. ([\#14801](https://github.com/matrix-org/synapse/issues/14801)) +- Use `htmltest` to check links in the Synapse documentation. ([\#14743](https://github.com/matrix-org/synapse/issues/14743)) + + +Internal Changes +---------------- + +- Faster remote room joins: stream the un-partial-stating of events over replication. ([\#14545](https://github.com/matrix-org/synapse/issues/14545), [\#14546](https://github.com/matrix-org/synapse/issues/14546)) +- Use [ruff](https://github.com/charliermarsh/ruff/) instead of flake8. ([\#14633](https://github.com/matrix-org/synapse/issues/14633), [\#14741](https://github.com/matrix-org/synapse/issues/14741)) +- Change `handle_new_client_event` signature so that a 429 does not reach clients on `PartialStateConflictError`, and internally retry when needed instead. ([\#14665](https://github.com/matrix-org/synapse/issues/14665)) +- Remove dependency on jQuery on reCAPTCHA page. ([\#14672](https://github.com/matrix-org/synapse/issues/14672)) +- Faster joins: make `compute_state_after_events` consistent with other state-fetching functions that take a `StateFilter`. ([\#14676](https://github.com/matrix-org/synapse/issues/14676)) +- Add missing type hints. ([\#14680](https://github.com/matrix-org/synapse/issues/14680), [\#14681](https://github.com/matrix-org/synapse/issues/14681), [\#14687](https://github.com/matrix-org/synapse/issues/14687)) +- Improve type annotations for the helper methods on a `CachedFunction`. ([\#14685](https://github.com/matrix-org/synapse/issues/14685)) +- Check that the SQLite database file exists before porting to PostgreSQL. ([\#14692](https://github.com/matrix-org/synapse/issues/14692)) +- Add `.direnv/` directory to .gitignore to prevent local state generated by the [direnv](https://direnv.net/) development tool from being committed. ([\#14707](https://github.com/matrix-org/synapse/issues/14707)) +- Batch up replication requests to request the resyncing of remote users's devices. ([\#14716](https://github.com/matrix-org/synapse/issues/14716)) +- If debug logging is enabled, log the `msgid`s of any to-device messages that are returned over `/sync`. ([\#14724](https://github.com/matrix-org/synapse/issues/14724)) +- Change GHA CI job to follow best practices. ([\#14772](https://github.com/matrix-org/synapse/issues/14772)) +- Switch to our fork of `dh-virtualenv` to work around an upstream Python 3.11 incompatibility. ([\#14774](https://github.com/matrix-org/synapse/issues/14774)) +- Skip testing built wheels for PyPy 3.7 on Linux x86_64 as we lack new required dependencies in the build environment. ([\#14802](https://github.com/matrix-org/synapse/issues/14802)) + +### Dependabot updates + +
+ +- Bump JasonEtco/create-an-issue from 2.8.1 to 2.8.2. ([\#14693](https://github.com/matrix-org/synapse/issues/14693)) +- Bump anyhow from 1.0.66 to 1.0.68. ([\#14694](https://github.com/matrix-org/synapse/issues/14694)) +- Bump blake2 from 0.10.5 to 0.10.6. ([\#14695](https://github.com/matrix-org/synapse/issues/14695)) +- Bump serde_json from 1.0.89 to 1.0.91. ([\#14696](https://github.com/matrix-org/synapse/issues/14696)) +- Bump serde from 1.0.150 to 1.0.151. ([\#14697](https://github.com/matrix-org/synapse/issues/14697)) +- Bump lxml from 4.9.1 to 4.9.2. ([\#14698](https://github.com/matrix-org/synapse/issues/14698)) +- Bump types-jsonschema from 4.17.0.1 to 4.17.0.2. ([\#14700](https://github.com/matrix-org/synapse/issues/14700)) +- Bump sentry-sdk from 1.11.1 to 1.12.0. ([\#14701](https://github.com/matrix-org/synapse/issues/14701)) +- Bump types-setuptools from 65.6.0.1 to 65.6.0.2. ([\#14702](https://github.com/matrix-org/synapse/issues/14702)) +- Bump minimum PyYAML to 3.13. ([\#14720](https://github.com/matrix-org/synapse/issues/14720)) +- Bump JasonEtco/create-an-issue from 2.8.2 to 2.9.1. ([\#14731](https://github.com/matrix-org/synapse/issues/14731)) +- Bump towncrier from 22.8.0 to 22.12.0. ([\#14732](https://github.com/matrix-org/synapse/issues/14732)) +- Bump isort from 5.10.1 to 5.11.4. ([\#14733](https://github.com/matrix-org/synapse/issues/14733)) +- Bump attrs from 22.1.0 to 22.2.0. ([\#14734](https://github.com/matrix-org/synapse/issues/14734)) +- Bump black from 22.10.0 to 22.12.0. ([\#14735](https://github.com/matrix-org/synapse/issues/14735)) +- Bump sentry-sdk from 1.12.0 to 1.12.1. ([\#14736](https://github.com/matrix-org/synapse/issues/14736)) +- Bump setuptools from 65.3.0 to 65.5.1. ([\#14738](https://github.com/matrix-org/synapse/issues/14738)) +- Bump serde from 1.0.151 to 1.0.152. ([\#14758](https://github.com/matrix-org/synapse/issues/14758)) +- Bump ruff from 0.0.189 to 0.0.206. ([\#14759](https://github.com/matrix-org/synapse/issues/14759)) +- Bump pydantic from 1.10.2 to 1.10.4. ([\#14760](https://github.com/matrix-org/synapse/issues/14760)) +- Bump gitpython from 3.1.29 to 3.1.30. ([\#14761](https://github.com/matrix-org/synapse/issues/14761)) +- Bump pillow from 9.3.0 to 9.4.0. ([\#14762](https://github.com/matrix-org/synapse/issues/14762)) +- Bump types-requests from 2.28.11.5 to 2.28.11.7. ([\#14763](https://github.com/matrix-org/synapse/issues/14763)) +- Bump dawidd6/action-download-artifact from 2.24.2 to 2.24.3. ([\#14779](https://github.com/matrix-org/synapse/issues/14779)) +- Bump peaceiris/actions-gh-pages from 3.9.0 to 3.9.1. ([\#14791](https://github.com/matrix-org/synapse/issues/14791)) +- Bump types-pillow from 9.3.0.4 to 9.4.0.0. ([\#14792](https://github.com/matrix-org/synapse/issues/14792)) +- Bump pyopenssl from 22.1.0 to 23.0.0. ([\#14793](https://github.com/matrix-org/synapse/issues/14793)) +- Bump types-setuptools from 65.6.0.2 to 65.6.0.3. ([\#14794](https://github.com/matrix-org/synapse/issues/14794)) +- Bump importlib-metadata from 4.2.0 to 6.0.0. ([\#14795](https://github.com/matrix-org/synapse/issues/14795)) +- Bump ruff from 0.0.206 to 0.0.215. ([\#14796](https://github.com/matrix-org/synapse/issues/14796)) +
diff --git a/docs/changelogs/CHANGES-2024.md b/docs/changelogs/CHANGES-2024.md new file mode 100644 index 0000000000..ee354f1573 --- /dev/null +++ b/docs/changelogs/CHANGES-2024.md @@ -0,0 +1,1586 @@ +# Synapse 1.121.1 (2024-12-11) + +This release contains a fix for our docker build CI. It is functionally identical to 1.121.0, whose changelog is below. + +### Internal Changes + +- Downgrade the Ubuntu GHA runner when building docker images. ([\#18026](https://github.com/element-hq/synapse/issues/18026)) + + + +# Synapse 1.121.0 (2024-12-11) + +### Internal Changes + +- Fix release process to not create duplicate releases. ([\#18025](https://github.com/element-hq/synapse/issues/18025)) + + + +# Synapse 1.121.0rc1 (2024-12-04) + +### Features + +- Support for [MSC4190](https://github.com/matrix-org/matrix-spec-proposals/pull/4190): device management for Application Services. ([\#17705](https://github.com/element-hq/synapse/issues/17705)) +- Update [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync to include invite, ban, kick, targets when `$LAZY`-loading room members. ([\#17947](https://github.com/element-hq/synapse/issues/17947)) +- Use stable `M_USER_LOCKED` error code for locked accounts, as per [Matrix 1.12](https://spec.matrix.org/v1.12/client-server-api/#account-locking). ([\#17965](https://github.com/element-hq/synapse/issues/17965)) +- [MSC4076](https://github.com/matrix-org/matrix-spec-proposals/pull/4076): Add `disable_badge_count` to pusher configuration. ([\#17975](https://github.com/element-hq/synapse/issues/17975)) + +### Bugfixes + +- Fix long-standing bug where read receipts could get overly delayed being sent over federation. ([\#17933](https://github.com/element-hq/synapse/issues/17933)) + +### Improved Documentation + +- Add OIDC example configuration for Forgejo (fork of Gitea). ([\#17872](https://github.com/element-hq/synapse/issues/17872)) +- Link to element-docker-demo from contrib/docker*. ([\#17953](https://github.com/element-hq/synapse/issues/17953)) + +### Internal Changes + +- [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108): Add a `Content-Type` header on the `PUT` response to work around a faulty behavior in some caching reverse proxies. ([\#17253](https://github.com/element-hq/synapse/issues/17253)) +- Fix incorrect comment in new schema delta. ([\#17936](https://github.com/element-hq/synapse/issues/17936)) +- Raise setuptools_rust version cap to 1.10.2. ([\#17944](https://github.com/element-hq/synapse/issues/17944)) +- Enable encrypted appservice related experimental features in the complement docker image. ([\#17945](https://github.com/element-hq/synapse/issues/17945)) +- Return whether the user is suspended when querying the user account in the Admin API. ([\#17952](https://github.com/element-hq/synapse/issues/17952)) +- Fix new scheduled tasks jumping the queue. ([\#17962](https://github.com/element-hq/synapse/issues/17962)) +- Bump pyo3 and dependencies to v0.23.2. ([\#17966](https://github.com/element-hq/synapse/issues/17966)) +- Update setuptools-rust and fix building abi3 wheels in latest version. ([\#17969](https://github.com/element-hq/synapse/issues/17969)) +- Consolidate SSO redirects through `/_matrix/client/v3/login/sso/redirect(/{idpId})`. ([\#17972](https://github.com/element-hq/synapse/issues/17972)) +- Fix Docker and Complement config to be able to use `public_baseurl`. ([\#17986](https://github.com/element-hq/synapse/issues/17986)) +- Fix building wheels for MacOS which was temporarily disabled in Synapse 1.120.2. ([\#17993](https://github.com/element-hq/synapse/issues/17993)) +- Fix release process to not create duplicate releases. ([\#17970](https://github.com/element-hq/synapse/issues/17970), [\#17995](https://github.com/element-hq/synapse/issues/17995)) + + +### Updates to locked dependencies + +* Bump bytes from 1.8.0 to 1.9.0. ([\#17982](https://github.com/element-hq/synapse/issues/17982)) +* Bump pysaml2 from 7.3.1 to 7.5.0. ([\#17978](https://github.com/element-hq/synapse/issues/17978)) +* Bump serde_json from 1.0.132 to 1.0.133. ([\#17939](https://github.com/element-hq/synapse/issues/17939)) +* Bump tomli from 2.0.2 to 2.1.0. ([\#17959](https://github.com/element-hq/synapse/issues/17959)) +* Bump tomli from 2.1.0 to 2.2.1. ([\#17979](https://github.com/element-hq/synapse/issues/17979)) +* Bump tornado from 6.4.1 to 6.4.2. ([\#17955](https://github.com/element-hq/synapse/issues/17955)) + +# Synapse 1.120.2 (2024-12-03) + +This version has building of wheels for macOS disabled. +It is functionally identical to 1.120.1, which contains multiple security fixes. +If you are already using 1.120.1, there is no need to upgrade to this version. + + + +# Synapse 1.120.1 (2024-12-03) + +This patch release fixes multiple security vulnerabilities, some affecting all prior versions of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild. + +Administrators who are unable to update Synapse may use the workarounds described in the linked GitHub Security Advisory below. + +### Security advisory + +The following issues are fixed in 1.120.1. + +- [GHSA-rfq8-j7rh-8hf2](https://github.com/element-hq/synapse/security/advisories/GHSA-rfq8-j7rh-8hf2) / [CVE-2024-52805](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-52805): **Unsupported content types can lead to memory exhaustion** + + Synapse instances which have a high `max_upload_size` and which don't have a reverse proxy in front of them that would otherwise limit upload size are affected. + + Fixed by [4b7154c58501b4bf5e1c2d6c11ebef96529f2fdf](https://github.com/element-hq/synapse/commit/4b7154c58501b4bf5e1c2d6c11ebef96529f2fdf). + +- [GHSA-f3r3-h2mq-hx2h](https://github.com/element-hq/synapse/security/advisories/GHSA-f3r3-h2mq-hx2h) / [CVE-2024-52815](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-52815): **Malicious invites via federation can break a user's sync** + + Fixed by [d82e1ed357b7ee21dff83d06cba7a67840cfd464](https://github.com/element-hq/synapse/commit/d82e1ed357b7ee21dff83d06cba7a67840cfd464). + +- [GHSA-vp6v-whfm-rv3g](https://github.com/element-hq/synapse/security/advisories/GHSA-vp6v-whfm-rv3g) / [CVE-2024-53863](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-53863): **Synapse can be forced to thumbnail unexpected file formats, invoking potentially untrustworthy decoders** + + Synapse instances can disable dynamic thumbnailing by setting `dynamic_thumbnails` to `false` in the configuration file. + + Fixed by [b64a4e5fbbbf119b6c65aedf0d999b4237d55503](https://github.com/element-hq/synapse/commit/b64a4e5fbbbf119b6c65aedf0d999b4237d55503). + +- [GHSA-56w4-5538-8v8h](https://github.com/element-hq/synapse/security/advisories/GHSA-56w4-5538-8v8h) / [CVE-2024-53867](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-53867): **The Sliding Sync feature on Synapse versions between 1.113.0rc1 and 1.120.0 can leak partial room state changes to users no longer in a room** + + Non-state events, like messages, are unaffected. + + Synapse instances can disable the Sliding Sync feature by setting `experimental_features.msc3575_enabled` to `false` in the configuration file. + + Fixed by [4daa533e82f345ce87b9495d31781af570ba3ead](https://github.com/element-hq/synapse/commit/4daa533e82f345ce87b9495d31781af570ba3ead). + +See the advisories for more details. If you have any questions, email [security at element.io](mailto:security@element.io). + +### Bugfixes + +- Fix release process to not create duplicate releases. ([\#17970](https://github.com/element-hq/synapse/issues/17970)) + + + +# Synapse 1.120.0 (2024-11-26) + +### Bugfixes + +- Fix a bug introduced in Synapse v1.120rc1 which would cause the newly-introduced `delete_old_otks` job to fail in worker-mode deployments. ([\#17960](https://github.com/element-hq/synapse/issues/17960)) + + + + +# Synapse 1.120.0rc1 (2024-11-20) + +This release enables the enforcement of authenticated media by default, with exemptions for media that is already present in the +homeserver's media store. + +Most homeservers operating in the public federation will not be impacted by this change, given that +the large homeserver `matrix.org` enabled this in September 2024 and therefore most clients and servers +will already have updated as a result. + +Some server administrators may still wish to disable this enforcement for the time being, in the interest of compatibility with older clients +and older federated homeservers. +See the [upgrade notes](https://element-hq.github.io/synapse/v1.120/upgrade.html#authenticated-media-is-now-enforced-by-default) for more information. + +### Features + +- Enforce authenticated media by default. Administrators can revert this by configuring `enable_authenticated_media` to `false`. In a future release of Synapse, this option will be removed and become always-on. ([\#17889](https://github.com/element-hq/synapse/issues/17889)) +- Add a one-off task to delete old One-Time Keys, to guard against us having old OTKs in the database that the client has long forgotten about. ([\#17934](https://github.com/element-hq/synapse/issues/17934)) + +### Improved Documentation + +- Clarify the semantics of the `enable_authenticated_media` configuration option. ([\#17913](https://github.com/element-hq/synapse/issues/17913)) +- Add documentation about backing up Synapse. ([\#17931](https://github.com/element-hq/synapse/issues/17931)) + +### Deprecations and Removals + +- Remove support for [MSC3886: Simple client rendezvous capability](https://github.com/matrix-org/matrix-spec-proposals/pull/3886), which has been superseded by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108) and therefore closed. ([\#17638](https://github.com/element-hq/synapse/issues/17638)) + +### Internal Changes + +- Addressed some typos in docs and returned error message for unknown MXC ID. ([\#17865](https://github.com/element-hq/synapse/issues/17865)) +- Unpin the upload release GHA action. ([\#17923](https://github.com/element-hq/synapse/issues/17923)) +- Bump macOS version used to build wheels during release, as current version used is end-of-life. ([\#17924](https://github.com/element-hq/synapse/issues/17924)) +- Move server event filtering logic to Rust. ([\#17928](https://github.com/element-hq/synapse/issues/17928)) +- Support new package name of PyPI package `python-multipart` 0.0.13 so that distro packagers do not need to work around name conflict with PyPI package `multipart`. ([\#17932](https://github.com/element-hq/synapse/issues/17932)) +- Speed up slow initial sliding syncs on large servers. ([\#17946](https://github.com/element-hq/synapse/issues/17946)) + +### Updates to locked dependencies + +* Bump anyhow from 1.0.92 to 1.0.93. ([\#17920](https://github.com/element-hq/synapse/issues/17920)) +* Bump bleach from 6.1.0 to 6.2.0. ([\#17918](https://github.com/element-hq/synapse/issues/17918)) +* Bump immutabledict from 4.2.0 to 4.2.1. ([\#17941](https://github.com/element-hq/synapse/issues/17941)) +* Bump packaging from 24.1 to 24.2. ([\#17940](https://github.com/element-hq/synapse/issues/17940)) +* Bump phonenumbers from 8.13.49 to 8.13.50. ([\#17942](https://github.com/element-hq/synapse/issues/17942)) +* Bump pygithub from 2.4.0 to 2.5.0. ([\#17917](https://github.com/element-hq/synapse/issues/17917)) +* Bump ruff from 0.7.2 to 0.7.3. ([\#17919](https://github.com/element-hq/synapse/issues/17919)) +* Bump serde from 1.0.214 to 1.0.215. ([\#17938](https://github.com/element-hq/synapse/issues/17938)) + +# Synapse 1.119.0 (2024-11-13) + +No significant changes since 1.119.0rc2. + +### Python 3.8 support dropped + +Python 3.8 is [end-of-life](https://devguide.python.org/versions/) and is no longer supported by Synapse. The minimum supported Python version is now 3.9. + +If you are running Synapse with Python 3.8, please upgrade to Python 3.9 (or greater) before upgrading Synapse. + + +# Synapse 1.119.0rc2 (2024-11-11) + +Note that due to packaging issues there was no v1.119.0rc1. + + +### Features + +- Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API. ([\#17374](https://github.com/element-hq/synapse/issues/17374)) +- Add experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2). ([\#17888](https://github.com/element-hq/synapse/issues/17888)) + +### Bugfixes + +- Fix bug with sliding sync where `$LAZY`-loading room members would not return `required_state` membership in incremental syncs. ([\#17809](https://github.com/element-hq/synapse/issues/17809)) +- Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi. ([\#17839](https://github.com/element-hq/synapse/issues/17839)) +- Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in + the config option `run_background_tasks_on`. ([\#17847](https://github.com/element-hq/synapse/issues/17847)) +- Fix bug where some presence and typing timeouts can expire early. ([\#17850](https://github.com/element-hq/synapse/issues/17850)) +- Fix detection when the built Rust library was outdated when using source installations. ([\#17861](https://github.com/element-hq/synapse/issues/17861)) +- Fix a long-standing bug in Synapse which could cause one-time keys to be issued in the incorrect order, causing message decryption failures. ([\#17903](https://github.com/element-hq/synapse/pull/17903)) +- Fix experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2) where we would return the full state on incremental syncs when using lazy loaded members and there were no new events in the timeline. ([\#17915](https://github.com/element-hq/synapse/pull/17915)) + +### Internal Changes + +- Remove support for python 3.8. ([\#17908](https://github.com/element-hq/synapse/issues/17908)) +- Add a test for downloading and thumbnailing a CMYK JPEG. ([\#17786](https://github.com/element-hq/synapse/issues/17786)) +- Refactor database calls to remove `Generator` usage. ([\#17813](https://github.com/element-hq/synapse/issues/17813), [\#17814](https://github.com/element-hq/synapse/issues/17814), [\#17815](https://github.com/element-hq/synapse/issues/17815), [\#17816](https://github.com/element-hq/synapse/issues/17816), [\#17817](https://github.com/element-hq/synapse/issues/17817), [\#17818](https://github.com/element-hq/synapse/issues/17818), [\#17890](https://github.com/element-hq/synapse/issues/17890)) +- Include the destination in the error of 'Destination mismatch' on federation requests. ([\#17830](https://github.com/element-hq/synapse/issues/17830)) +- The nix flake inside the repository no longer tracks nixpkgs/master to not catch the latest bugs from a PR merged 5 minutes ago. ([\#17852](https://github.com/element-hq/synapse/issues/17852)) +- Minor speed-up of sliding sync by computing extensions results in parallel. ([\#17884](https://github.com/element-hq/synapse/issues/17884)) +- Bump the default Python version in the Synapse Dockerfile from 3.11 -> 3.12. ([\#17887](https://github.com/element-hq/synapse/issues/17887)) +- Remove usage of internal header encoding API. ([\#17894](https://github.com/element-hq/synapse/issues/17894)) +- Use unique name for each os.arch variant when uploading Wheel artifacts. ([\#17905](https://github.com/element-hq/synapse/issues/17905)) +- Fix tests to run with latest Twisted. ([\#17906](https://github.com/element-hq/synapse/pull/17906), [\#17907](https://github.com/element-hq/synapse/pull/17907), [\#17911](https://github.com/element-hq/synapse/pull/17911)) +- Update version constraint to allow the latest poetry-core 1.9.1. ([\#17902](https://github.com/element-hq/synapse/pull/17902)) +- Update the portdb CI to use Python 3.13 and Postgres 17 as latest dependencies. ([\#17909](https://github.com/element-hq/synapse/pull/17909)) +- Add an index to `current_state_delta_stream` table. ([\#17912](https://github.com/element-hq/synapse/issues/17912)) +- Fix building and attaching release artifacts during the release process. ([\#17921](https://github.com/element-hq/synapse/issues/17921)) + +### Updates to locked dependencies + +* Bump actions/download-artifact & actions/upload-artifact from 3 to 4 in /.github/workflows. ([\#17657](https://github.com/element-hq/synapse/issues/17657)) +* Bump anyhow from 1.0.89 to 1.0.92. ([\#17858](https://github.com/element-hq/synapse/issues/17858), [\#17876](https://github.com/element-hq/synapse/issues/17876), [\#17901](https://github.com/element-hq/synapse/issues/17901)) +* Bump bytes from 1.7.2 to 1.8.0. ([\#17877](https://github.com/element-hq/synapse/issues/17877)) +* Bump cryptography from 43.0.1 to 43.0.3. ([\#17853](https://github.com/element-hq/synapse/issues/17853)) +* Bump mypy-zope from 1.0.7 to 1.0.8. ([\#17898](https://github.com/element-hq/synapse/issues/17898)) +* Bump phonenumbers from 8.13.47 to 8.13.49. ([\#17880](https://github.com/element-hq/synapse/issues/17880), [\#17899](https://github.com/element-hq/synapse/issues/17899)) +* Bump python-multipart from 0.0.12 to 0.0.16. ([\#17879](https://github.com/element-hq/synapse/issues/17879)) +* Bump regex from 1.11.0 to 1.11.1. ([\#17874](https://github.com/element-hq/synapse/issues/17874)) +* Bump ruff from 0.6.9 to 0.7.2. ([\#17868](https://github.com/element-hq/synapse/issues/17868), [\#17897](https://github.com/element-hq/synapse/issues/17897)) +* Bump serde from 1.0.210 to 1.0.214. ([\#17875](https://github.com/element-hq/synapse/issues/17875), [\#17900](https://github.com/element-hq/synapse/issues/17900)) +* Bump serde_json from 1.0.128 to 1.0.132. ([\#17857](https://github.com/element-hq/synapse/issues/17857)) +* Bump types-psycopg2 from 2.9.21.20240819 to 2.9.21.20241019. ([\#17855](https://github.com/element-hq/synapse/issues/17855)) +* Bump types-setuptools from 75.1.0.20241014 to 75.2.0.20241019. ([\#17856](https://github.com/element-hq/synapse/issues/17856)) + +# Synapse 1.118.0 (2024-10-29) + +No significant changes since 1.118.0rc1. + +### Python 3.8 support will be dropped in the next release + +Python 3.8 is now [end-of-life](https://devguide.python.org/versions/). As per our [Deprecation Policy for Platform Dependencies](https://element-hq.github.io/synapse/latest/deprecation_policy.html#policy), Synapse will be dropping support for Python 3.8 in the next release; Synapse 1.119.0. + +Synapse 1.118.x will be the final release to support Python 3.8. If you are running Synapse with Python 3.8, please upgrade before the 1.119.0 release, due in less than one month. + +### Python 3.13 and PostgreSQL 17 support + +On the other end of the spectrum, Synapse 1.118.0 is the first release to support [Python 3.13](https://www.python.org/downloads/release/python-3130/)! [PostgreSQL 17](https://www.postgresql.org/about/news/postgresql-17-released-2936/) is also supported as of this release. + + +# Synapse 1.118.0rc1 (2024-10-22) + +### Features + +- Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload. ([\#17708](https://github.com/element-hq/synapse/issues/17708)) +- Implement [MSC4210](https://github.com/matrix-org/matrix-spec-proposals/pull/4210): Remove legacy mentions. Contributed by @tulir @ Beeper. ([\#17783](https://github.com/element-hq/synapse/issues/17783)) + +### Bugfixes + +- Fix saving of PNG thumbnails, when the original image is in the CMYK color space. ([\#17736](https://github.com/element-hq/synapse/issues/17736)) +- Fix bug with sliding sync where the server would not return state that was added to the `required_state` config. ([\#17785](https://github.com/element-hq/synapse/issues/17785), [\#17805](https://github.com/element-hq/synapse/issues/17805)) +- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync that would cause rooms to stay forgotten and hidden even after rejoining. ([\#17835](https://github.com/element-hq/synapse/issues/17835)) + +### Improved Documentation + +- Clarify when the `user_may_invite` and `user_may_send_3pid_invite` module callbacks are called. ([\#17627](https://github.com/element-hq/synapse/issues/17627)) +- Correct documentation to refer to the `--config-path` argument instead of `--config-file`. ([\#17802](https://github.com/element-hq/synapse/issues/17802)) +- Fix typo in `target_cache_memory_usage` docs. ([\#17825](https://github.com/element-hq/synapse/issues/17825)) + +### Internal Changes + +- Slight optimization when fetching state/events for Sliding Sync. ([\#17718](https://github.com/element-hq/synapse/issues/17718)) +- Add Python 3.13 and Postgres 17 to the test matrix. ([\#17752](https://github.com/element-hq/synapse/issues/17752)) +- Test github token before running release script steps. ([\#17803](https://github.com/element-hq/synapse/issues/17803)) +- Build debian packages for new Ubuntu versions, and stop building for no longer supported versions. ([\#17824](https://github.com/element-hq/synapse/issues/17824)) +- Enable the `.org.matrix.msc4028.encrypted_event` push rule by default in accordance with [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028). Note that the corresponding experimental feature must still be switched on for this push rule to have any effect. ([\#17826](https://github.com/element-hq/synapse/issues/17826)) +- Fix some typing issues uncovered by upgrading mypy to 1.11.x. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) + + + +### Updates to locked dependencies + +* Bump mypy from 1.10.1 to 1.11.2. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) +* Bump mypy-zope from 1.0.5 to 1.0.7. ([\#17827](https://github.com/element-hq/synapse/issues/17827)) +* Bump phonenumbers from 8.13.46 to 8.13.47. ([\#17797](https://github.com/element-hq/synapse/issues/17797)) +* Bump psycopg2 from 2.9.9 to 2.9.10. ([\#17843](https://github.com/element-hq/synapse/issues/17843)) +* Bump ruff from 0.6.8 to 0.6.9. ([\#17794](https://github.com/element-hq/synapse/issues/17794)) +* Bump sentry-sdk from 2.14.0 to 2.15.0. ([\#17795](https://github.com/element-hq/synapse/issues/17795)) +* Bump sentry-sdk from 2.15.0 to 2.16.0. ([\#17829](https://github.com/element-hq/synapse/issues/17829)) +* Bump sentry-sdk from 2.16.0 to 2.17.0. ([\#17844](https://github.com/element-hq/synapse/issues/17844)) +* Bump sigstore/cosign-installer from 3.6.0 to 3.7.0. ([\#17798](https://github.com/element-hq/synapse/issues/17798)) +* Bump tomli from 2.0.1 to 2.0.2. ([\#17796](https://github.com/element-hq/synapse/issues/17796)) +* Bump types-requests from 2.32.0.20240914 to 2.32.0.20241016. ([\#17841](https://github.com/element-hq/synapse/issues/17841)) +* Bump types-setuptools from 75.1.0.20240917 to 75.1.0.20241014. ([\#17828](https://github.com/element-hq/synapse/issues/17828)) + +# Synapse 1.117.0 (2024-10-15) + +No significant changes since 1.117.0rc1. + + + + +# Synapse 1.117.0rc1 (2024-10-08) + +### Features + +- Add config option `redis.password_path`. ([\#17717](https://github.com/element-hq/synapse/issues/17717)) + +### Bugfixes + +- Fix a rare bug introduced in v1.29.0 where invalidating a user's access token from a worker could raise an error. ([\#17779](https://github.com/element-hq/synapse/issues/17779)) +- In the response to `GET /_matrix/client/versions`, set the `unstable_features` flag for [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140) to `false` when server configuration disables support for delayed events. ([\#17780](https://github.com/element-hq/synapse/issues/17780)) +- Improve input validation and room membership checks in admin redaction API. ([\#17792](https://github.com/element-hq/synapse/issues/17792)) + +### Improved Documentation + +- Clarify the docstring of `test_forget_when_not_left`. ([\#17628](https://github.com/element-hq/synapse/issues/17628)) +- Add documentation note about PYTHONMALLOC for accurate jemalloc memory tracking. Contributed by @hensg. ([\#17709](https://github.com/element-hq/synapse/issues/17709)) +- Remove spurious "TODO UPDATE ALL THIS" note in the Debian installation docs. ([\#17749](https://github.com/element-hq/synapse/issues/17749)) +- Explain how load balancing works for `federation_sender_instances`. ([\#17776](https://github.com/element-hq/synapse/issues/17776)) + +### Internal Changes + +- Minor performance increase for large accounts using sliding sync. ([\#17751](https://github.com/element-hq/synapse/issues/17751)) +- Increase performance of the notifier when there are many syncing users. ([\#17765](https://github.com/element-hq/synapse/issues/17765), [\#17766](https://github.com/element-hq/synapse/issues/17766)) +- Fix performance of streams that don't change often. ([\#17767](https://github.com/element-hq/synapse/issues/17767)) +- Improve performance of sliding sync connections that do not ask for any rooms. ([\#17768](https://github.com/element-hq/synapse/issues/17768)) +- Reduce overhead of sliding sync E2EE loops. ([\#17771](https://github.com/element-hq/synapse/issues/17771)) +- Sliding sync minor performance speed up using new table. ([\#17787](https://github.com/element-hq/synapse/issues/17787)) +- Sliding sync minor performance improvement by omitting unchanged data from incremental responses. ([\#17788](https://github.com/element-hq/synapse/issues/17788)) +- Speed up sliding sync when there are many active subscriptions. ([\#17789](https://github.com/element-hq/synapse/issues/17789)) +- Add missing license headers on new source files. ([\#17799](https://github.com/element-hq/synapse/issues/17799)) + + + +### Updates to locked dependencies + +* Bump phonenumbers from 8.13.45 to 8.13.46. ([\#17773](https://github.com/element-hq/synapse/issues/17773)) +* Bump python-multipart from 0.0.10 to 0.0.12. ([\#17772](https://github.com/element-hq/synapse/issues/17772)) +* Bump regex from 1.10.6 to 1.11.0. ([\#17770](https://github.com/element-hq/synapse/issues/17770)) +* Bump ruff from 0.6.7 to 0.6.8. ([\#17774](https://github.com/element-hq/synapse/issues/17774)) + +# Synapse 1.116.0 (2024-10-01) + +No significant changes since 1.116.0rc2. + + + + +# Synapse 1.116.0rc2 (2024-09-26) + +### Features + +- Add implementation of restricting who can overwrite a state event as proposed by [MSC3757](https://github.com/matrix-org/matrix-spec-proposals/pull/3757). ([\#17513](https://github.com/element-hq/synapse/issues/17513)) + + + + +# Synapse 1.116.0rc1 (2024-09-25) + +### Features + +- Add initial implementation of delayed events as proposed by [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). ([\#17326](https://github.com/element-hq/synapse/issues/17326)) +- Add an asynchronous Admin API endpoint [to redact all a user's events](https://element-hq.github.io/synapse/v1.116/admin_api/user_admin_api.html#redact-all-the-events-of-a-user), + and [an endpoint to check on the status of that redaction task](https://element-hq.github.io/synapse/v1.116/admin_api/user_admin_api.html#check-the-status-of-a-redaction-process). ([\#17506](https://github.com/element-hq/synapse/issues/17506)) +- Add support for the `tags` and `not_tags` filters for [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17662](https://github.com/element-hq/synapse/issues/17662)) +- Guests can use the new media endpoints to download media, as described by [MSC4189](https://github.com/matrix-org/matrix-spec-proposals/pull/4189). ([\#17675](https://github.com/element-hq/synapse/issues/17675)) +- Add config option `turn_shared_secret_path`. ([\#17690](https://github.com/element-hq/synapse/issues/17690)) +- Return room tags in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync account data extension. ([\#17707](https://github.com/element-hq/synapse/issues/17707)) + +### Bugfixes + +- Make sure we get up-to-date state information when using the new [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync tables to derive room membership. ([\#17692](https://github.com/element-hq/synapse/issues/17692)) +- Fix bug where room account data would not correctly be sent down [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync for old rooms. ([\#17695](https://github.com/element-hq/synapse/issues/17695)) +- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync which could prevent /sync from working for certain user accounts. ([\#17727](https://github.com/element-hq/synapse/issues/17727), [\#17733](https://github.com/element-hq/synapse/issues/17733)) +- Ignore invites from ignored users in Sliding Sync. ([\#17729](https://github.com/element-hq/synapse/issues/17729)) +- Fix bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync where the server would incorrectly return a negative bump stamp, which caused Element X apps to stop syncing. ([\#17748](https://github.com/element-hq/synapse/issues/17748)) + +### Internal Changes + +- Import pydantic objects from the `_pydantic_compat` module. + This allows `check_pydantic_models.py` to mock those pydantic objects + only in the synapse module, and not interfere with pydantic objects in + external dependencies. ([\#17667](https://github.com/element-hq/synapse/issues/17667)) +- Use [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync tables as a bulk shortcut for getting the max `event_stream_ordering` of rooms. ([\#17693](https://github.com/element-hq/synapse/issues/17693)) +- Speed up [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync requests a bit where there are many room changes. ([\#17696](https://github.com/element-hq/synapse/issues/17696)) +- Refactor [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync filter unit tests so the sliding sync API has better test coverage. ([\#17703](https://github.com/element-hq/synapse/issues/17703)) +- Fetch `bump_stamp`s more efficiently in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17723](https://github.com/element-hq/synapse/issues/17723)) +- Shortcut for checking if certain background updates have completed (utilized in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync). ([\#17724](https://github.com/element-hq/synapse/issues/17724)) +- More efficiently fetch rooms for [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17725](https://github.com/element-hq/synapse/issues/17725)) +- Fix `_bulk_get_max_event_pos` being inefficient. ([\#17728](https://github.com/element-hq/synapse/issues/17728)) +- Add cache to `get_tags_for_room(...)`. ([\#17730](https://github.com/element-hq/synapse/issues/17730)) +- Small performance improvement in speeding up [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17731](https://github.com/element-hq/synapse/issues/17731)) +- Minor speed up of initial [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync requests. ([\#17734](https://github.com/element-hq/synapse/issues/17734)) +- Remove usage of the deprecated `cgi` module, deprecated in Python 3.11 and removed in Python 3.13. ([\#17741](https://github.com/element-hq/synapse/issues/17741)) +- Fix typing of a variable that is not `Unknown` anymore after updating `treq`. ([\#17744](https://github.com/element-hq/synapse/issues/17744)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.86 to 1.0.89. ([\#17685](https://github.com/element-hq/synapse/issues/17685), [\#17716](https://github.com/element-hq/synapse/issues/17716)) +* Bump bytes from 1.7.1 to 1.7.2. ([\#17743](https://github.com/element-hq/synapse/issues/17743)) +* Bump cryptography from 43.0.0 to 43.0.1. ([\#17689](https://github.com/element-hq/synapse/issues/17689)) +* Bump idna from 3.8 to 3.10. ([\#17758](https://github.com/element-hq/synapse/issues/17758)) +* Bump msgpack from 1.0.8 to 1.1.0. ([\#17759](https://github.com/element-hq/synapse/issues/17759)) +* Bump phonenumbers from 8.13.44 to 8.13.45. ([\#17762](https://github.com/element-hq/synapse/issues/17762)) +* Bump prometheus-client from 0.20.0 to 0.21.0. ([\#17746](https://github.com/element-hq/synapse/issues/17746)) +* Bump pyasn1 from 0.6.0 to 0.6.1. ([\#17714](https://github.com/element-hq/synapse/issues/17714)) +* Bump pyasn1-modules from 0.4.0 to 0.4.1. ([\#17747](https://github.com/element-hq/synapse/issues/17747)) +* Bump pydantic from 2.8.2 to 2.9.2. ([\#17756](https://github.com/element-hq/synapse/issues/17756)) +* Bump python-multipart from 0.0.9 to 0.0.10. ([\#17745](https://github.com/element-hq/synapse/issues/17745)) +* Bump ruff from 0.6.4 to 0.6.7. ([\#17715](https://github.com/element-hq/synapse/issues/17715), [\#17760](https://github.com/element-hq/synapse/issues/17760)) +* Bump sentry-sdk from 2.13.0 to 2.14.0. ([\#17712](https://github.com/element-hq/synapse/issues/17712)) +* Bump serde from 1.0.209 to 1.0.210. ([\#17686](https://github.com/element-hq/synapse/issues/17686)) +* Bump serde_json from 1.0.127 to 1.0.128. ([\#17687](https://github.com/element-hq/synapse/issues/17687)) +* Bump treq from 23.11.0 to 24.9.1. ([\#17744](https://github.com/element-hq/synapse/issues/17744)) +* Bump types-pyyaml from 6.0.12.20240808 to 6.0.12.20240917. ([\#17755](https://github.com/element-hq/synapse/issues/17755)) +* Bump types-requests from 2.32.0.20240712 to 2.32.0.20240914. ([\#17713](https://github.com/element-hq/synapse/issues/17713)) +* Bump types-setuptools from 74.1.0.20240907 to 75.1.0.20240917. ([\#17757](https://github.com/element-hq/synapse/issues/17757)) + +# Synapse 1.115.0 (2024-09-17) + +No significant changes since 1.115.0rc2. + + + + +# Synapse 1.115.0rc2 (2024-09-12) + +### Internal Changes + +- Pre-populate room data used in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint for quick filtering/sorting. ([\#17652](https://github.com/element-hq/synapse/issues/17652)) +- Speed up sliding sync by reducing amount of data pulled out of the database for large rooms. ([\#17683](https://github.com/element-hq/synapse/issues/17683)) + + + + +# Synapse 1.115.0rc1 (2024-09-10) + +### Features + +- Improve cross-signing upload when using [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) to use a custom UIA flow stage, with web fallback support. ([\#17509](https://github.com/element-hq/synapse/issues/17509)) + +### Bugfixes + +- Return `400 M_BAD_JSON` upon attempting to complete various room actions with a non-local user ID and unknown room ID, rather than an internal server error. ([\#17607](https://github.com/element-hq/synapse/issues/17607)) +- Fix authenticated media responses using a wrong limit when following redirects over federation. ([\#17626](https://github.com/element-hq/synapse/issues/17626)) +- Fix bug where we returned the wrong `bump_stamp` for invites in sliding sync response, causing incorrect ordering of invites in the room list. ([\#17674](https://github.com/element-hq/synapse/issues/17674)) + +### Improved Documentation + +- Clarify that the admin api resource is only loaded on the main process and not workers. ([\#17590](https://github.com/element-hq/synapse/issues/17590)) +- Fixed typo in `saml2_config` config [example](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#saml2_config). ([\#17594](https://github.com/element-hq/synapse/issues/17594)) + +### Deprecations and Removals + +- Stabilise [MSC4156](https://github.com/matrix-org/matrix-spec-proposals/pull/4156) by removing the `msc4156_enabled` config setting and defaulting it to `true`. ([\#17650](https://github.com/element-hq/synapse/issues/17650)) + +### Internal Changes + +- Update [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation: load the issuer and account management URLs from OIDC discovery. ([\#17407](https://github.com/element-hq/synapse/issues/17407)) +- Pre-populate room data used in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint for quick filtering/sorting. ([\#17512](https://github.com/element-hq/synapse/issues/17512), [\#17632](https://github.com/element-hq/synapse/issues/17632), [\#17633](https://github.com/element-hq/synapse/issues/17633), [\#17634](https://github.com/element-hq/synapse/issues/17634), [\#17635](https://github.com/element-hq/synapse/issues/17635), [\#17636](https://github.com/element-hq/synapse/issues/17636), [\#17641](https://github.com/element-hq/synapse/issues/17641), [\#17654](https://github.com/element-hq/synapse/issues/17654), [\#17673](https://github.com/element-hq/synapse/issues/17673)) +- Store sliding sync per-connection state in the database. ([\#17599](https://github.com/element-hq/synapse/issues/17599), [\#17631](https://github.com/element-hq/synapse/issues/17631)) +- Make the sliding sync `PerConnectionState` class immutable. ([\#17600](https://github.com/element-hq/synapse/issues/17600)) +- Replace `isort` and `black` with `ruff`. ([\#17620](https://github.com/element-hq/synapse/issues/17620), [\#17643](https://github.com/element-hq/synapse/issues/17643)) +- Sliding Sync: Split up `get_room_membership_for_user_at_to_token`. ([\#17629](https://github.com/element-hq/synapse/issues/17629)) +- Use new database tables for sliding sync. ([\#17630](https://github.com/element-hq/synapse/issues/17630), [\#17649](https://github.com/element-hq/synapse/issues/17649)) +- Prevent duplicate tags being added to Sliding Sync traces. ([\#17655](https://github.com/element-hq/synapse/issues/17655)) +- Get `bump_stamp` from [new sliding sync tables](https://github.com/element-hq/synapse/pull/17512) which should be faster. ([\#17658](https://github.com/element-hq/synapse/issues/17658)) +- Speed up incremental Sliding Sync requests by avoiding extra work. ([\#17665](https://github.com/element-hq/synapse/issues/17665)) +- Small performance improvement in speeding up sliding sync. ([\#17666](https://github.com/element-hq/synapse/issues/17666), [\#17670](https://github.com/element-hq/synapse/issues/17670), [\#17672](https://github.com/element-hq/synapse/issues/17672)) +- Speed up sliding sync by reducing number of database calls. ([\#17684](https://github.com/element-hq/synapse/issues/17684)) +- Speed up sync by pulling out fewer events from the database. ([\#17688](https://github.com/element-hq/synapse/issues/17688)) + + + +### Updates to locked dependencies + +* Bump authlib from 1.3.1 to 1.3.2. ([\#17679](https://github.com/element-hq/synapse/issues/17679)) +* Bump idna from 3.7 to 3.8. ([\#17682](https://github.com/element-hq/synapse/issues/17682)) +* Bump ruff from 0.6.2 to 0.6.4. ([\#17680](https://github.com/element-hq/synapse/issues/17680)) +* Bump towncrier from 24.7.1 to 24.8.0. ([\#17645](https://github.com/element-hq/synapse/issues/17645)) +* Bump twisted from 24.7.0rc1 to 24.7.0. ([\#17647](https://github.com/element-hq/synapse/issues/17647)) +* Bump types-pillow from 10.2.0.20240520 to 10.2.0.20240822. ([\#17644](https://github.com/element-hq/synapse/issues/17644)) +* Bump types-psycopg2 from 2.9.21.20240417 to 2.9.21.20240819. ([\#17646](https://github.com/element-hq/synapse/issues/17646)) +* Bump types-setuptools from 71.1.0.20240818 to 74.1.0.20240907. ([\#17681](https://github.com/element-hq/synapse/issues/17681)) + +# Synapse 1.114.0 (2024-09-02) + +This release enables support for +[MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) — +Simplified Sliding Sync. This allows using the upcoming releases of the Element +X mobile apps without having to run a Sliding Sync Proxy. + + +### Features + +- Enable native sliding sync support ([MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) and [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186)) by default. ([\#17648](https://github.com/element-hq/synapse/issues/17648)) + + + + +# Synapse 1.114.0rc3 (2024-08-30) + +### Bugfixes + +- Fix regression in v1.114.0rc2 that caused workers to fail to start. ([\#17626](https://github.com/element-hq/synapse/issues/17626)) + + + + +# Synapse 1.114.0rc2 (2024-08-30) + +### Features + +- Improve cross-signing upload when using [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) to use a custom UIA flow stage, with web fallback support. ([\#17509](https://github.com/element-hq/synapse/issues/17509)) +- Make `hash_password` script accept password input from stdin. ([\#17608](https://github.com/element-hq/synapse/issues/17608)) + +### Bugfixes + +- Fix hierarchy returning 403 when room is accessible through federation. Contributed by Krishan (@kfiven). ([\#17194](https://github.com/element-hq/synapse/issues/17194)) +- Fix content-length on federation `/thumbnail` responses. ([\#17532](https://github.com/element-hq/synapse/issues/17532)) +- Fix authenticated media responses using a wrong limit when following redirects over federation. ([\#17543](https://github.com/element-hq/synapse/issues/17543)) + +### Internal Changes + +- MSC3861: load the issuer and account management URLs from OIDC discovery. ([\#17407](https://github.com/element-hq/synapse/issues/17407)) +- Refactor sliding sync class into multiple files. ([\#17595](https://github.com/element-hq/synapse/issues/17595)) +- Store sliding sync per-connection state in the database. ([\#17599](https://github.com/element-hq/synapse/issues/17599)) +- Make the sliding sync `PerConnectionState` class immutable. ([\#17600](https://github.com/element-hq/synapse/issues/17600)) +- Add support to `@tag_args` for standalone functions. ([\#17604](https://github.com/element-hq/synapse/issues/17604)) +- Speed up incremental syncs in sliding sync by adding some more caching. ([\#17606](https://github.com/element-hq/synapse/issues/17606)) +- Always return the user's own read receipts in sliding sync. ([\#17617](https://github.com/element-hq/synapse/issues/17617)) +- Replace `isort` and `black` with `ruff`. ([\#17620](https://github.com/element-hq/synapse/issues/17620)) +- Refactor sliding sync code to move room list logic out into a separate class. ([\#17622](https://github.com/element-hq/synapse/issues/17622)) + + + +### Updates to locked dependencies + +* Bump attrs from 23.2.0 to 24.2.0. ([\#17609](https://github.com/element-hq/synapse/issues/17609)) +* Bump cryptography from 42.0.8 to 43.0.0. ([\#17584](https://github.com/element-hq/synapse/issues/17584)) +* Bump phonenumbers from 8.13.43 to 8.13.44. ([\#17610](https://github.com/element-hq/synapse/issues/17610)) +* Bump pygithub from 2.3.0 to 2.4.0. ([\#17612](https://github.com/element-hq/synapse/issues/17612)) +* Bump pyyaml from 6.0.1 to 6.0.2. ([\#17611](https://github.com/element-hq/synapse/issues/17611)) +* Bump sentry-sdk from 2.12.0 to 2.13.0. ([\#17585](https://github.com/element-hq/synapse/issues/17585)) +* Bump serde from 1.0.206 to 1.0.208. ([\#17581](https://github.com/element-hq/synapse/issues/17581)) +* Bump serde from 1.0.208 to 1.0.209. ([\#17613](https://github.com/element-hq/synapse/issues/17613)) +* Bump serde_json from 1.0.124 to 1.0.125. ([\#17582](https://github.com/element-hq/synapse/issues/17582)) +* Bump serde_json from 1.0.125 to 1.0.127. ([\#17614](https://github.com/element-hq/synapse/issues/17614)) +* Bump types-jsonschema from 4.23.0.20240712 to 4.23.0.20240813. ([\#17583](https://github.com/element-hq/synapse/issues/17583)) +* Bump types-setuptools from 71.1.0.20240726 to 71.1.0.20240818. ([\#17586](https://github.com/element-hq/synapse/issues/17586)) + +# Synapse 1.114.0rc1 (2024-08-20) + +### Features + +- Add a flag to `/versions`, `org.matrix.simplified_msc3575`, to indicate whether experimental sliding sync support has been enabled. ([\#17571](https://github.com/element-hq/synapse/issues/17571)) +- Handle changes in `timeline_limit` in experimental sliding sync. ([\#17579](https://github.com/element-hq/synapse/issues/17579)) +- Correctly track read receipts that should be sent down in experimental sliding sync. ([\#17575](https://github.com/element-hq/synapse/issues/17575), [\#17589](https://github.com/element-hq/synapse/issues/17589), [\#17592](https://github.com/element-hq/synapse/issues/17592)) + +### Bugfixes + +- Start handlers for new media endpoints when media resource configured. ([\#17483](https://github.com/element-hq/synapse/issues/17483)) +- Fix timeline ordering (using `stream_ordering` instead of topological ordering) in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17510](https://github.com/element-hq/synapse/issues/17510)) +- Fix experimental sliding sync implementation to remember any updates in rooms that were not sent down immediately. ([\#17535](https://github.com/element-hq/synapse/issues/17535)) +- Better exclude partially stated rooms if we must await full state in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17538](https://github.com/element-hq/synapse/issues/17538)) +- Handle lower-case http headers in `_Mulitpart_Parser_Protocol`. ([\#17545](https://github.com/element-hq/synapse/issues/17545)) +- Fix fetching federation signing keys from servers that omit `old_verify_keys`. Contributed by @tulir @ Beeper. ([\#17568](https://github.com/element-hq/synapse/issues/17568)) +- Fix bug where we would respond with an error when a remote server asked for media that had a length of 0, using the new multipart federation media endpoint. ([\#17570](https://github.com/element-hq/synapse/issues/17570)) + +### Improved Documentation + +- Clarify default behaviour of the + [`auto_accept_invites.worker_to_run_on`](https://element-hq.github.io/synapse/develop/usage/configuration/config_documentation.html#auto-accept-invites) + option. ([\#17515](https://github.com/element-hq/synapse/issues/17515)) +- Improve docstrings for profile methods. ([\#17559](https://github.com/element-hq/synapse/issues/17559)) + +### Internal Changes + +- Add more tracing to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17514](https://github.com/element-hq/synapse/issues/17514)) +- Fixup comment in sliding sync implementation. ([\#17531](https://github.com/element-hq/synapse/issues/17531)) +- Replace override of deprecated method `HTTPAdapter.get_connection` with `get_connection_with_tls_context`. ([\#17536](https://github.com/element-hq/synapse/issues/17536)) +- Fix performance of device lists in `/key/changes` and sliding sync. ([\#17537](https://github.com/element-hq/synapse/issues/17537), [\#17548](https://github.com/element-hq/synapse/issues/17548)) +- Bump setuptools from 67.6.0 to 72.1.0. ([\#17542](https://github.com/element-hq/synapse/issues/17542)) +- Add a utility function for generating random event IDs. ([\#17557](https://github.com/element-hq/synapse/issues/17557)) +- Speed up responding to media requests. ([\#17558](https://github.com/element-hq/synapse/issues/17558), [\#17561](https://github.com/element-hq/synapse/issues/17561), [\#17564](https://github.com/element-hq/synapse/issues/17564), [\#17566](https://github.com/element-hq/synapse/issues/17566), [\#17567](https://github.com/element-hq/synapse/issues/17567), [\#17569](https://github.com/element-hq/synapse/issues/17569)) +- Test github token before running release script steps. ([\#17562](https://github.com/element-hq/synapse/issues/17562)) +- Reduce log spam of multipart files. ([\#17563](https://github.com/element-hq/synapse/issues/17563)) +- Refactor per-connection state in experimental sliding sync handler. ([\#17574](https://github.com/element-hq/synapse/issues/17574)) +- Add histogram metrics for sliding sync processing time. ([\#17593](https://github.com/element-hq/synapse/issues/17593)) + + + +### Updates to locked dependencies + +* Bump bytes from 1.6.1 to 1.7.1. ([\#17526](https://github.com/element-hq/synapse/issues/17526)) +* Bump lxml from 5.2.2 to 5.3.0. ([\#17550](https://github.com/element-hq/synapse/issues/17550)) +* Bump phonenumbers from 8.13.42 to 8.13.43. ([\#17551](https://github.com/element-hq/synapse/issues/17551)) +* Bump regex from 1.10.5 to 1.10.6. ([\#17527](https://github.com/element-hq/synapse/issues/17527)) +* Bump sentry-sdk from 2.10.0 to 2.12.0. ([\#17553](https://github.com/element-hq/synapse/issues/17553)) +* Bump serde from 1.0.204 to 1.0.206. ([\#17556](https://github.com/element-hq/synapse/issues/17556)) +* Bump serde_json from 1.0.122 to 1.0.124. ([\#17555](https://github.com/element-hq/synapse/issues/17555)) +* Bump sigstore/cosign-installer from 3.5.0 to 3.6.0. ([\#17549](https://github.com/element-hq/synapse/issues/17549)) +* Bump types-pyyaml from 6.0.12.20240311 to 6.0.12.20240808. ([\#17552](https://github.com/element-hq/synapse/issues/17552)) +* Bump types-requests from 2.31.0.20240406 to 2.32.0.20240712. ([\#17524](https://github.com/element-hq/synapse/issues/17524)) + +# Synapse 1.113.0 (2024-08-13) + +No significant changes since 1.113.0rc1. + + + + +# Synapse 1.113.0rc1 (2024-08-06) + +### Features + +- Track which rooms have been sent to clients in the experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17447](https://github.com/element-hq/synapse/issues/17447)) +- Add Account Data extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17477](https://github.com/element-hq/synapse/issues/17477)) +- Add receipts extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17489](https://github.com/element-hq/synapse/issues/17489)) +- Add typing notification extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17505](https://github.com/element-hq/synapse/issues/17505)) + +### Bugfixes + +- Update experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint to handle invite/knock rooms when filtering. ([\#17450](https://github.com/element-hq/synapse/issues/17450)) +- Fix a bug introduced in v1.110.0 which caused `/keys/query` to return incomplete results, leading to high network activity and CPU usage on Matrix clients. ([\#17499](https://github.com/element-hq/synapse/issues/17499)) + +### Improved Documentation + +- Update the [`allowed_local_3pids`](https://element-hq.github.io/synapse/v1.112/usage/configuration/config_documentation.html#allowed_local_3pids) config option's msisdn address to a working example. ([\#17476](https://github.com/element-hq/synapse/issues/17476)) + +### Internal Changes + +- Change sliding sync to use their own token format in preparation for storing per-connection state. ([\#17452](https://github.com/element-hq/synapse/issues/17452)) +- Ensure we don't send down negative `bump_stamp` in experimental sliding sync endpoint. ([\#17478](https://github.com/element-hq/synapse/issues/17478)) +- Do not send down empty room entries down experimental sliding sync endpoint. ([\#17479](https://github.com/element-hq/synapse/issues/17479)) +- Refactor Sliding Sync tests to better utilize the `SlidingSyncBase`. ([\#17481](https://github.com/element-hq/synapse/issues/17481), [\#17482](https://github.com/element-hq/synapse/issues/17482)) +- Add some opentracing tags and logging to the experimental sliding sync implementation. ([\#17501](https://github.com/element-hq/synapse/issues/17501)) +- Split and move Sliding Sync tests so we have some more sane test file sizes. ([\#17504](https://github.com/element-hq/synapse/issues/17504)) +- Update the `limited` field description in the Sliding Sync response to accurately describe what it actually represents. ([\#17507](https://github.com/element-hq/synapse/issues/17507)) +- Easier to understand `timeline` assertions in Sliding Sync tests. ([\#17511](https://github.com/element-hq/synapse/issues/17511)) +- Reset the sliding sync connection if we don't recognize the per-connection state position. ([\#17529](https://github.com/element-hq/synapse/issues/17529)) + + + +### Updates to locked dependencies + +* Bump bcrypt from 4.1.3 to 4.2.0. ([\#17495](https://github.com/element-hq/synapse/issues/17495)) +* Bump black from 24.4.2 to 24.8.0. ([\#17522](https://github.com/element-hq/synapse/issues/17522)) +* Bump phonenumbers from 8.13.39 to 8.13.42. ([\#17521](https://github.com/element-hq/synapse/issues/17521)) +* Bump ruff from 0.5.4 to 0.5.5. ([\#17494](https://github.com/element-hq/synapse/issues/17494)) +* Bump serde_json from 1.0.120 to 1.0.121. ([\#17493](https://github.com/element-hq/synapse/issues/17493)) +* Bump serde_json from 1.0.121 to 1.0.122. ([\#17525](https://github.com/element-hq/synapse/issues/17525)) +* Bump towncrier from 23.11.0 to 24.7.1. ([\#17523](https://github.com/element-hq/synapse/issues/17523)) +* Bump types-pyopenssl from 24.1.0.20240425 to 24.1.0.20240722. ([\#17496](https://github.com/element-hq/synapse/issues/17496)) +* Bump types-setuptools from 70.1.0.20240627 to 71.1.0.20240726. ([\#17497](https://github.com/element-hq/synapse/issues/17497)) + +# Synapse 1.112.0 (2024-07-30) + +This security release is to update our locked dependency on Twisted to 24.7.0rc1, which includes a security fix for [CVE-2024-41671 / GHSA-c8m8-j448-xjx7: Disordered HTTP pipeline response in twisted.web, again](https://github.com/twisted/twisted/security/advisories/GHSA-c8m8-j448-xjx7). + +Note that this security fix is also available as **Synapse 1.111.1**, which does not include the rest of the changes in Synapse 1.112.0. + +This issue means that, if multiple HTTP requests are pipelined in the same TCP connection, Synapse can send responses to the wrong HTTP request. +If a reverse proxy was configured to use HTTP pipelining, this could result in responses being sent to the wrong user, severely harming confidentiality. + +With that said, despite being a high severity issue, **we consider it unlikely that Synapse installations will be affected**. +The use of HTTP pipelining in this fashion would cause worse performance for clients (request-response latencies would be increased as users' responses would be artificially blocked behind other users' slow requests). Further, Nginx and Haproxy, two common reverse proxies, do not appear to support configuring their upstreams to use HTTP pipelining and thus would not be affected. For both of these reasons, we consider it unlikely that a Synapse deployment would be set up in such a configuration. + +Despite that, we cannot rule out that some installations may exist with this unusual setup and so we are releasing this security update today. + +**pip users:** Note that by default, upgrading Synapse using pip will not automatically upgrade Twisted. **Please manually install the new version of Twisted** using `pip install Twisted==24.7.0rc1`. Note also that even the `--upgrade-strategy=eager` flag to `pip install -U matrix-synapse` will not upgrade Twisted to a patched version because it is only a release candidate at this time. + +### Internal Changes + +- Upgrade locked dependency on Twisted to 24.7.0rc1. ([\#17502](https://github.com/element-hq/synapse/issues/17502)) + + +# Synapse 1.112.0rc1 (2024-07-23) + +Please note that this release candidate does not include the security dependency update +included in version 1.111.1 as this version was released before 1.111.1. +The same security fix can be found in the full release of 1.112.0. + +### Features + +- Add to-device extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17416](https://github.com/element-hq/synapse/issues/17416)) +- Populate `name`/`avatar` fields in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17418](https://github.com/element-hq/synapse/issues/17418)) +- Populate `heroes` and room summary fields (`joined_count`, `invited_count`) in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17419](https://github.com/element-hq/synapse/issues/17419)) +- Populate `is_dm` room field in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17429](https://github.com/element-hq/synapse/issues/17429)) +- Add room subscriptions to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17432](https://github.com/element-hq/synapse/issues/17432)) +- Prepare for authenticated media freeze. ([\#17433](https://github.com/element-hq/synapse/issues/17433)) +- Add E2EE extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17454](https://github.com/element-hq/synapse/issues/17454)) + +### Bugfixes + +- Add configurable option to always include offline users in presence sync results. Contributed by @Michael-Hollister. ([\#17231](https://github.com/element-hq/synapse/issues/17231)) +- Fix bug in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint when using room type filters and the user has one or more remote invites. ([\#17434](https://github.com/element-hq/synapse/issues/17434)) +- Order `heroes` by `stream_ordering` as the Matrix specification states (applies to `/sync`). ([\#17435](https://github.com/element-hq/synapse/issues/17435)) +- Fix rare bug where `/sync` would break for a user when using workers with multiple stream writers. ([\#17438](https://github.com/element-hq/synapse/issues/17438)) + +### Improved Documentation + +- Update the readme image to have a white background, so that it is readable in dark mode. ([\#17387](https://github.com/element-hq/synapse/issues/17387)) +- Add Red Hat Enterprise Linux and Rocky Linux 8 and 9 installation instructions. ([\#17423](https://github.com/element-hq/synapse/issues/17423)) +- Improve documentation for the [`default_power_level_content_override`](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#default_power_level_content_override) config option. ([\#17451](https://github.com/element-hq/synapse/issues/17451)) + +### Internal Changes + +- Make sure we always use the right logic for enabling the media repo. ([\#17424](https://github.com/element-hq/synapse/issues/17424)) +- Fix argument documentation for method `RateLimiter.record_action`. ([\#17426](https://github.com/element-hq/synapse/issues/17426)) +- Reduce volume of 'Waiting for current token' logs, which were introduced in v1.109.0. ([\#17428](https://github.com/element-hq/synapse/issues/17428)) +- Limit concurrent remote downloads to 6 per IP address, and decrement remote downloads without a content-length from the ratelimiter after the download is complete. ([\#17439](https://github.com/element-hq/synapse/issues/17439)) +- Remove unnecessary call to resume producing in fake channel. ([\#17449](https://github.com/element-hq/synapse/issues/17449)) +- Update experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint to bump room when it is created. ([\#17453](https://github.com/element-hq/synapse/issues/17453)) +- Speed up generating sliding sync responses. ([\#17458](https://github.com/element-hq/synapse/issues/17458)) +- Add cache to `get_rooms_for_local_user_where_membership_is` to speed up sliding sync. ([\#17460](https://github.com/element-hq/synapse/issues/17460)) +- Speed up fetching room keys from backup. ([\#17461](https://github.com/element-hq/synapse/issues/17461)) +- Speed up sorting of the room list in sliding sync. ([\#17468](https://github.com/element-hq/synapse/issues/17468)) +- Implement handling of `$ME` as a state key in sliding sync. ([\#17469](https://github.com/element-hq/synapse/issues/17469)) + + + +### Updates to locked dependencies + +* Bump bytes from 1.6.0 to 1.6.1. ([\#17441](https://github.com/element-hq/synapse/issues/17441)) +* Bump hiredis from 2.3.2 to 3.0.0. ([\#17464](https://github.com/element-hq/synapse/issues/17464)) +* Bump jsonschema from 4.22.0 to 4.23.0. ([\#17444](https://github.com/element-hq/synapse/issues/17444)) +* Bump matrix-org/done-action from 2 to 3. ([\#17440](https://github.com/element-hq/synapse/issues/17440)) +* Bump mypy from 1.9.0 to 1.10.1. ([\#17445](https://github.com/element-hq/synapse/issues/17445)) +* Bump pyopenssl from 24.1.0 to 24.2.1. ([\#17465](https://github.com/element-hq/synapse/issues/17465)) +* Bump ruff from 0.5.0 to 0.5.4. ([\#17466](https://github.com/element-hq/synapse/issues/17466)) +* Bump sentry-sdk from 2.6.0 to 2.8.0. ([\#17456](https://github.com/element-hq/synapse/issues/17456)) +* Bump sentry-sdk from 2.8.0 to 2.10.0. ([\#17467](https://github.com/element-hq/synapse/issues/17467)) +* Bump setuptools from 67.6.0 to 70.0.0. ([\#17448](https://github.com/element-hq/synapse/issues/17448)) +* Bump twine from 5.1.0 to 5.1.1. ([\#17443](https://github.com/element-hq/synapse/issues/17443)) +* Bump types-jsonschema from 4.22.0.20240610 to 4.23.0.20240712. ([\#17446](https://github.com/element-hq/synapse/issues/17446)) +* Bump ulid from 1.1.2 to 1.1.3. ([\#17442](https://github.com/element-hq/synapse/issues/17442)) +* Bump zipp from 3.15.0 to 3.19.1. ([\#17427](https://github.com/element-hq/synapse/issues/17427)) + + +# Synapse 1.111.1 (2024-07-30) + +This security release is to update our locked dependency on Twisted to 24.7.0rc1, which includes a security fix for [CVE-2024-41671 / GHSA-c8m8-j448-xjx7: Disordered HTTP pipeline response in twisted.web, again](https://github.com/twisted/twisted/security/advisories/GHSA-c8m8-j448-xjx7). + +This issue means that, if multiple HTTP requests are pipelined in the same TCP connection, Synapse can send responses to the wrong HTTP request. +If a reverse proxy was configured to use HTTP pipelining, this could result in responses being sent to the wrong user, severely harming confidentiality. + +With that said, despite being a high severity issue, **we consider it unlikely that Synapse installations will be affected**. +The use of HTTP pipelining in this fashion would cause worse performance for clients (request-response latencies would be increased as users' responses would be artificially blocked behind other users' slow requests). Further, Nginx and Haproxy, two common reverse proxies, do not appear to support configuring their upstreams to use HTTP pipelining and thus would not be affected. For both of these reasons, we consider it unlikely that a Synapse deployment would be set up in such a configuration. + +Despite that, we cannot rule out that some installations may exist with this unusual setup and so we are releasing this security update today. + +**pip users:** Note that by default, upgrading Synapse using pip will not automatically upgrade Twisted. **Please manually install the new version of Twisted** using `pip install Twisted==24.7.0rc1`. Note also that even the `--upgrade-strategy=eager` flag to `pip install -U matrix-synapse` will not upgrade Twisted to a patched version because it is only a release candidate at this time. + + +### Internal Changes + +- Upgrade locked dependency on Twisted to 24.7.0rc1. ([\#17502](https://github.com/element-hq/synapse/issues/17502)) + + +# Synapse 1.111.0 (2024-07-16) + +No significant changes since 1.111.0rc2. + + + + +# Synapse 1.111.0rc2 (2024-07-10) + +### Bugfixes + +- Fix bug where using `synapse.app.media_repository` worker configuration would break the new media endpoints. ([\#17420](https://github.com/element-hq/synapse/issues/17420)) + +### Improved Documentation + +- Document the new federation media worker endpoints in the [upgrade notes](https://element-hq.github.io/synapse/v1.111/upgrade.html) and [worker docs](https://element-hq.github.io/synapse/v1.111/workers.html). ([\#17421](https://github.com/element-hq/synapse/issues/17421)) + +### Internal Changes + +- Route authenticated federation media requests to media repository workers in Complement tests. ([\#17422](https://github.com/element-hq/synapse/issues/17422)) + + + + +# Synapse 1.111.0rc1 (2024-07-09) + +### Features + +- Add `rooms` data to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17320](https://github.com/element-hq/synapse/issues/17320)) +- Add `room_types`/`not_room_types` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17337](https://github.com/element-hq/synapse/issues/17337)) +- Return "required state" in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17342](https://github.com/element-hq/synapse/issues/17342)) +- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3916-authentication-for-media.md) by adding [`_matrix/client/v1/media/download`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediadownloadservernamemediaid) endpoint. ([\#17365](https://github.com/element-hq/synapse/issues/17365)) +- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/rav/authentication-for-media/proposals/3916-authentication-for-media.md) + by adding [`_matrix/client/v1/media/thumbnail`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediathumbnailservernamemediaid), [`_matrix/federation/v1/media/thumbnail`](https://spec.matrix.org/v1.11/server-server-api/#get_matrixfederationv1mediathumbnailmediaid) endpoints and stabilizing the + remaining [`_matrix/client/v1/media`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediaconfig) endpoints. ([\#17388](https://github.com/element-hq/synapse/issues/17388)) +- Add `rooms.bump_stamp` for easier client-side sorting in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17395](https://github.com/element-hq/synapse/issues/17395)) +- Forget all of a user's rooms upon deactivation, preventing local room purges from being blocked on deactivated users. ([\#17400](https://github.com/element-hq/synapse/issues/17400)) +- Declare support for [Matrix 1.11](https://matrix.org/blog/2024/06/20/matrix-v1.11-release/). ([\#17403](https://github.com/element-hq/synapse/issues/17403)) +- [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861): allow overriding the introspection endpoint. ([\#17406](https://github.com/element-hq/synapse/issues/17406)) + +### Bugfixes + +- Fix rare race which caused no new to-device messages to be received from remote server. ([\#17362](https://github.com/element-hq/synapse/issues/17362)) +- Fix bug in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint when using an old database. ([\#17398](https://github.com/element-hq/synapse/issues/17398)) + +### Improved Documentation + +- Clarify that `url_preview_url_blacklist` is a usability feature. ([\#17356](https://github.com/element-hq/synapse/issues/17356)) +- Fix broken links in README. ([\#17379](https://github.com/element-hq/synapse/issues/17379)) +- Clarify that changelog content *and file extension* need to match in order for entries to merge. ([\#17399](https://github.com/element-hq/synapse/issues/17399)) + +### Internal Changes + +- Make the release script create a release branch for Complement as well. ([\#17318](https://github.com/element-hq/synapse/issues/17318)) +- Fix uploading packages to PyPi. ([\#17363](https://github.com/element-hq/synapse/issues/17363)) +- Add CI check for the README. ([\#17367](https://github.com/element-hq/synapse/issues/17367)) +- Fix linting errors from new `ruff` version. ([\#17381](https://github.com/element-hq/synapse/issues/17381), [\#17411](https://github.com/element-hq/synapse/issues/17411)) +- Fix building debian packages on non-clean checkouts. ([\#17390](https://github.com/element-hq/synapse/issues/17390)) +- Finish up work to allow per-user feature flags. ([\#17392](https://github.com/element-hq/synapse/issues/17392), [\#17410](https://github.com/element-hq/synapse/issues/17410)) +- Allow enabling sliding sync per-user. ([\#17393](https://github.com/element-hq/synapse/issues/17393)) + + + +### Updates to locked dependencies + +* Bump certifi from 2023.7.22 to 2024.7.4. ([\#17404](https://github.com/element-hq/synapse/issues/17404)) +* Bump cryptography from 42.0.7 to 42.0.8. ([\#17382](https://github.com/element-hq/synapse/issues/17382)) +* Bump ijson from 3.2.3 to 3.3.0. ([\#17413](https://github.com/element-hq/synapse/issues/17413)) +* Bump log from 0.4.21 to 0.4.22. ([\#17384](https://github.com/element-hq/synapse/issues/17384)) +* Bump mypy-zope from 1.0.4 to 1.0.5. ([\#17414](https://github.com/element-hq/synapse/issues/17414)) +* Bump pillow from 10.3.0 to 10.4.0. ([\#17412](https://github.com/element-hq/synapse/issues/17412)) +* Bump pydantic from 2.7.1 to 2.8.2. ([\#17415](https://github.com/element-hq/synapse/issues/17415)) +* Bump ruff from 0.3.7 to 0.5.0. ([\#17381](https://github.com/element-hq/synapse/issues/17381)) +* Bump serde from 1.0.203 to 1.0.204. ([\#17409](https://github.com/element-hq/synapse/issues/17409)) +* Bump serde_json from 1.0.117 to 1.0.120. ([\#17385](https://github.com/element-hq/synapse/issues/17385), [\#17408](https://github.com/element-hq/synapse/issues/17408)) +* Bump types-setuptools from 69.5.0.20240423 to 70.1.0.20240627. ([\#17380](https://github.com/element-hq/synapse/issues/17380)) + +# Synapse 1.110.0 (2024-07-03) + +No significant changes since 1.110.0rc3. + + + + +# Synapse 1.110.0rc3 (2024-07-02) + +### Bugfixes + +- Fix bug where `/sync` requests could get blocked indefinitely after an upgrade from Synapse versions before v1.109.0. ([\#17386](https://github.com/element-hq/synapse/issues/17386), [\#17391](https://github.com/element-hq/synapse/issues/17391)) + +### Internal Changes + +- Limit size of presence EDUs to 50 entries. ([\#17371](https://github.com/element-hq/synapse/issues/17371)) +- Fix building debian package for debian sid. ([\#17389](https://github.com/element-hq/synapse/issues/17389)) + + + + +# Synapse 1.110.0rc2 (2024-06-26) + +### Internal Changes + +- Fix uploading packages to PyPi. ([\#17363](https://github.com/element-hq/synapse/issues/17363)) + + + + +# Synapse 1.110.0rc1 (2024-06-26) + +### Features + +- Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17187](https://github.com/element-hq/synapse/issues/17187)) +- Add experimental support for [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823) - Account suspension. ([\#17255](https://github.com/element-hq/synapse/issues/17255)) +- Improve ratelimiting in Synapse. ([\#17256](https://github.com/element-hq/synapse/issues/17256)) +- Add support for the unstable [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151) report room API. ([\#17270](https://github.com/element-hq/synapse/issues/17270), [\#17296](https://github.com/element-hq/synapse/issues/17296)) +- Filter for public and empty rooms added to Admin-API [List Room API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#list-room-api). ([\#17276](https://github.com/element-hq/synapse/issues/17276)) +- Add `is_dm` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17277](https://github.com/element-hq/synapse/issues/17277)) +- Add `is_encrypted` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17281](https://github.com/element-hq/synapse/issues/17281)) +- Include user membership in events served to clients, per [MSC4115](https://github.com/matrix-org/matrix-spec-proposals/pull/4115). ([\#17282](https://github.com/element-hq/synapse/issues/17282)) +- Do not require user-interactive authentication for uploading cross-signing keys for the first time, per [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967). ([\#17284](https://github.com/element-hq/synapse/issues/17284)) +- Add `stream_ordering` sort to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17293](https://github.com/element-hq/synapse/issues/17293)) +- `register_new_matrix_user` now supports a --password-file flag, which + is useful for scripting. ([\#17294](https://github.com/element-hq/synapse/issues/17294)) +- `register_new_matrix_user` now supports a --exists-ok flag to allow registration of users that already exist in the database. + This is useful for scripts that bootstrap user accounts with initial passwords. ([\#17304](https://github.com/element-hq/synapse/issues/17304)) +- Add support for via query parameter from [MSC4156](https://github.com/matrix-org/matrix-spec-proposals/pull/4156). ([\#17322](https://github.com/element-hq/synapse/issues/17322)) +- Add `is_invite` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17335](https://github.com/element-hq/synapse/issues/17335)) +- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3916-authentication-for-media.md) by adding a federation /download endpoint. ([\#17350](https://github.com/element-hq/synapse/issues/17350)) + +### Bugfixes + +- Fix searching for users with their exact localpart whose ID includes a hyphen. ([\#17254](https://github.com/element-hq/synapse/issues/17254)) +- Fix wrong retention policy being used when filtering events. ([\#17272](https://github.com/element-hq/synapse/issues/17272)) +- Fix bug where OTKs were not always included in `/sync` response when using workers. ([\#17275](https://github.com/element-hq/synapse/issues/17275)) +- Fix a long-standing bug where an invalid 'from' parameter to [`/notifications`](https://spec.matrix.org/v1.10/client-server-api/#get_matrixclientv3notifications) would result in an Internal Server Error. ([\#17283](https://github.com/element-hq/synapse/issues/17283)) +- Fix edge case in `/sync` returning the wrong the state when using sharded event persisters. ([\#17295](https://github.com/element-hq/synapse/issues/17295)) +- Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17301](https://github.com/element-hq/synapse/issues/17301)) +- Fix email notification subject when invited to a space. ([\#17336](https://github.com/element-hq/synapse/issues/17336)) + +### Improved Documentation + +- Add missing quotes for example for `exclude_rooms_from_sync`. ([\#17308](https://github.com/element-hq/synapse/issues/17308)) +- Update header in the README to visually fix the the auto-generated table of contents. ([\#17329](https://github.com/element-hq/synapse/issues/17329)) +- Fix stale references to the Foundation's Security Disclosure Policy. ([\#17341](https://github.com/element-hq/synapse/issues/17341)) +- Add default values for `rc_invites.per_issuer` to docs. ([\#17347](https://github.com/element-hq/synapse/issues/17347)) +- Fix an error in the docs for `search_all_users` parameter under `user_directory`. ([\#17348](https://github.com/element-hq/synapse/issues/17348)) + +### Internal Changes + +- Remove unused `expire_access_token` option in the Synapse Docker config file. Contributed by @AaronDewes. ([\#17198](https://github.com/element-hq/synapse/issues/17198)) +- Use fully-qualified `PersistedEventPosition` when returning `RoomsForUser` to facilitate proper comparisons and `RoomStreamToken` generation. ([\#17265](https://github.com/element-hq/synapse/issues/17265)) +- Add debug logging for when room keys are uploaded, including whether they are replacing other room keys. ([\#17266](https://github.com/element-hq/synapse/issues/17266)) +- Handle OTK uploads off master. ([\#17271](https://github.com/element-hq/synapse/issues/17271)) +- Don't try and resync devices for remote users whose servers are marked as down. ([\#17273](https://github.com/element-hq/synapse/issues/17273)) +- Re-organize Pydantic models and types used in handlers. ([\#17279](https://github.com/element-hq/synapse/issues/17279)) +- Expose the worker instance that persisted the event on `event.internal_metadata.instance_name`. ([\#17300](https://github.com/element-hq/synapse/issues/17300)) +- Update the README with Element branding, improve headers and fix the #synapse:matrix.org support room link rendering. ([\#17324](https://github.com/element-hq/synapse/issues/17324)) +- Change path of the experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync implementation to `/org.matrix.simplified_msc3575/sync` since our simplified API is slightly incompatible with what's in the current MSC. ([\#17331](https://github.com/element-hq/synapse/issues/17331)) +- Handle device lists notifications for large accounts more efficiently in worker mode. ([\#17333](https://github.com/element-hq/synapse/issues/17333), [\#17358](https://github.com/element-hq/synapse/issues/17358)) +- Do not block event sending/receiving while calculating large event auth chains. ([\#17338](https://github.com/element-hq/synapse/issues/17338)) +- Tidy up `parse_integer` docs and call sites to reflect the fact that they require non-negative integers by default, and bring `parse_integer_from_args` default in alignment. Contributed by Denis Kasak (@dkasak). ([\#17339](https://github.com/element-hq/synapse/issues/17339)) + + + +### Updates to locked dependencies + +* Bump authlib from 1.3.0 to 1.3.1. ([\#17343](https://github.com/element-hq/synapse/issues/17343)) +* Bump dawidd6/action-download-artifact from 3.1.4 to 5. ([\#17289](https://github.com/element-hq/synapse/issues/17289)) +* Bump dawidd6/action-download-artifact from 5 to 6. ([\#17313](https://github.com/element-hq/synapse/issues/17313)) +* Bump docker/build-push-action from 5 to 6. ([\#17312](https://github.com/element-hq/synapse/issues/17312)) +* Bump jinja2 from 3.1.3 to 3.1.4. ([\#17287](https://github.com/element-hq/synapse/issues/17287)) +* Bump lazy_static from 1.4.0 to 1.5.0. ([\#17355](https://github.com/element-hq/synapse/issues/17355)) +* Bump msgpack from 1.0.7 to 1.0.8. ([\#17317](https://github.com/element-hq/synapse/issues/17317)) +* Bump netaddr from 1.2.1 to 1.3.0. ([\#17353](https://github.com/element-hq/synapse/issues/17353)) +* Bump packaging from 24.0 to 24.1. ([\#17352](https://github.com/element-hq/synapse/issues/17352)) +* Bump phonenumbers from 8.13.37 to 8.13.39. ([\#17315](https://github.com/element-hq/synapse/issues/17315)) +* Bump regex from 1.10.4 to 1.10.5. ([\#17290](https://github.com/element-hq/synapse/issues/17290)) +* Bump requests from 2.31.0 to 2.32.2. ([\#17345](https://github.com/element-hq/synapse/issues/17345)) +* Bump sentry-sdk from 2.1.1 to 2.3.1. ([\#17263](https://github.com/element-hq/synapse/issues/17263)) +* Bump sentry-sdk from 2.3.1 to 2.6.0. ([\#17351](https://github.com/element-hq/synapse/issues/17351)) +* Bump tornado from 6.4 to 6.4.1. ([\#17344](https://github.com/element-hq/synapse/issues/17344)) +* Bump mypy from 1.8.0 to 1.9.0. ([\#17297](https://github.com/element-hq/synapse/issues/17297)) +* Bump types-jsonschema from 4.21.0.20240311 to 4.22.0.20240610. ([\#17288](https://github.com/element-hq/synapse/issues/17288)) +* Bump types-netaddr from 1.2.0.20240219 to 1.3.0.20240530. ([\#17314](https://github.com/element-hq/synapse/issues/17314)) +* Bump types-pillow from 10.2.0.20240423 to 10.2.0.20240520. ([\#17285](https://github.com/element-hq/synapse/issues/17285)) +* Bump types-pyyaml from 6.0.12.12 to 6.0.12.20240311. ([\#17316](https://github.com/element-hq/synapse/issues/17316)) +* Bump typing-extensions from 4.11.0 to 4.12.2. ([\#17354](https://github.com/element-hq/synapse/issues/17354)) +* Bump urllib3 from 2.0.7 to 2.2.2. ([\#17346](https://github.com/element-hq/synapse/issues/17346)) + +# Synapse 1.109.0 (2024-06-18) + +### Internal Changes + +- Fix the building of binary wheels for macOS by switching to macOS 12 CI runners. ([\#17319](https://github.com/element-hq/synapse/issues/17319)) + + + + +# Synapse 1.109.0rc3 (2024-06-17) + +### Bugfixes + +- When rolling back to a previous Synapse version and then forwards again to this release, don't require server operators to manually run SQL. ([\#17305](https://github.com/element-hq/synapse/issues/17305), [\#17309](https://github.com/element-hq/synapse/issues/17309)) + +### Internal Changes + +- Use the release branch for sytest in release-branch PRs. ([\#17306](https://github.com/element-hq/synapse/issues/17306)) + + + + +# Synapse 1.109.0rc2 (2024-06-11) + +### Bugfixes + +- Fix bug where one-time-keys were not always included in `/sync` response when using workers. Introduced in v1.109.0rc1. ([\#17275](https://github.com/element-hq/synapse/issues/17275)) +- Fix bug where `/sync` could get stuck due to edge case in device lists handling. Introduced in v1.109.0rc1. ([\#17292](https://github.com/element-hq/synapse/issues/17292)) + + + + +# Synapse 1.109.0rc1 (2024-06-04) + +### Features + +- Add the ability to auto-accept invites on the behalf of users. See the [`auto_accept_invites`](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#auto-accept-invites) config option for details. ([\#17147](https://github.com/element-hq/synapse/issues/17147)) +- Add experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync/e2ee` endpoint for to-device messages and device encryption info. ([\#17167](https://github.com/element-hq/synapse/issues/17167)) +- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/issues/3916) by adding unstable media endpoints to `/_matrix/client`. ([\#17213](https://github.com/element-hq/synapse/issues/17213)) +- Add logging to tasks managed by the task scheduler, showing CPU and database usage. ([\#17219](https://github.com/element-hq/synapse/issues/17219)) + +### Bugfixes + +- Fix deduplicating of membership events to not create unused state groups. ([\#17164](https://github.com/element-hq/synapse/issues/17164)) +- Fix bug where duplicate events could be sent down sync when using workers that are overloaded. ([\#17215](https://github.com/element-hq/synapse/issues/17215)) +- Ignore attempts to send to-device messages to bad users, to avoid log spam when we try to connect to the bad server. ([\#17240](https://github.com/element-hq/synapse/issues/17240)) +- Fix handling of duplicate concurrent uploading of device one-time-keys. ([\#17241](https://github.com/element-hq/synapse/issues/17241)) +- Fix reporting of default tags to Sentry, such as worker name. Broke in v1.108.0. ([\#17251](https://github.com/element-hq/synapse/issues/17251)) +- Fix bug where typing updates would not be sent when using workers after a restart. ([\#17252](https://github.com/element-hq/synapse/issues/17252)) + +### Improved Documentation + +- Update the LemonLDAP documentation to say that claims should be explicitly included in the returned `id_token`, as Synapse won't request them. ([\#17204](https://github.com/element-hq/synapse/issues/17204)) + +### Internal Changes + +- Improve DB usage when fetching related events. ([\#17083](https://github.com/element-hq/synapse/issues/17083)) +- Log exceptions when failing to auto-join new user according to the `auto_join_rooms` option. ([\#17176](https://github.com/element-hq/synapse/issues/17176)) +- Reduce work of calculating outbound device lists updates. ([\#17211](https://github.com/element-hq/synapse/issues/17211)) +- Improve performance of calculating device lists changes in `/sync`. ([\#17216](https://github.com/element-hq/synapse/issues/17216)) +- Move towards using `MultiWriterIdGenerator` everywhere. ([\#17226](https://github.com/element-hq/synapse/issues/17226)) +- Replaces all usages of `StreamIdGenerator` with `MultiWriterIdGenerator`. ([\#17229](https://github.com/element-hq/synapse/issues/17229)) +- Change the `allow_unsafe_locale` config option to also apply when setting up new databases. ([\#17238](https://github.com/element-hq/synapse/issues/17238)) +- Fix errors in logs about closing incorrect logging contexts when media gets rejected by a module. ([\#17239](https://github.com/element-hq/synapse/issues/17239), [\#17246](https://github.com/element-hq/synapse/issues/17246)) +- Clean out invalid destinations from `device_federation_outbox` table. ([\#17242](https://github.com/element-hq/synapse/issues/17242)) +- Stop logging errors when receiving invalid User IDs in key querys requests. ([\#17250](https://github.com/element-hq/synapse/issues/17250)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.83 to 1.0.86. ([\#17220](https://github.com/element-hq/synapse/issues/17220)) +* Bump bcrypt from 4.1.2 to 4.1.3. ([\#17224](https://github.com/element-hq/synapse/issues/17224)) +* Bump lxml from 5.2.1 to 5.2.2. ([\#17261](https://github.com/element-hq/synapse/issues/17261)) +* Bump mypy-zope from 1.0.3 to 1.0.4. ([\#17262](https://github.com/element-hq/synapse/issues/17262)) +* Bump phonenumbers from 8.13.35 to 8.13.37. ([\#17235](https://github.com/element-hq/synapse/issues/17235)) +* Bump prometheus-client from 0.19.0 to 0.20.0. ([\#17233](https://github.com/element-hq/synapse/issues/17233)) +* Bump pyasn1 from 0.5.1 to 0.6.0. ([\#17223](https://github.com/element-hq/synapse/issues/17223)) +* Bump pyicu from 2.13 to 2.13.1. ([\#17236](https://github.com/element-hq/synapse/issues/17236)) +* Bump pyopenssl from 24.0.0 to 24.1.0. ([\#17234](https://github.com/element-hq/synapse/issues/17234)) +* Bump serde from 1.0.201 to 1.0.202. ([\#17221](https://github.com/element-hq/synapse/issues/17221)) +* Bump serde from 1.0.202 to 1.0.203. ([\#17232](https://github.com/element-hq/synapse/issues/17232)) +* Bump twine from 5.0.0 to 5.1.0. ([\#17225](https://github.com/element-hq/synapse/issues/17225)) +* Bump types-psycopg2 from 2.9.21.20240311 to 2.9.21.20240417. ([\#17222](https://github.com/element-hq/synapse/issues/17222)) +* Bump types-pyopenssl from 24.0.0.20240311 to 24.1.0.20240425. ([\#17260](https://github.com/element-hq/synapse/issues/17260)) + +# Synapse 1.108.0 (2024-05-28) + +No significant changes since 1.108.0rc1. + + + + +# Synapse 1.108.0rc1 (2024-05-21) + +### Features + +- Add a feature that allows clients to query the configured federation whitelist. Disabled by default. ([\#16848](https://github.com/element-hq/synapse/issues/16848), [\#17199](https://github.com/element-hq/synapse/issues/17199)) +- Add the ability to allow numeric user IDs with a specific prefix when in the CAS flow. Contributed by Aurélien Grimpard. ([\#17098](https://github.com/element-hq/synapse/issues/17098)) + +### Bugfixes + +- Fix bug where push rules would be empty in `/sync` for some accounts. Introduced in v1.93.0. ([\#17142](https://github.com/element-hq/synapse/issues/17142)) +- Add support for optional whitespace around the Federation API's `Authorization` header's parameter commas. ([\#17145](https://github.com/element-hq/synapse/issues/17145)) +- Fix bug where disabling room publication prevented public rooms being created on workers. ([\#17177](https://github.com/element-hq/synapse/issues/17177), [\#17184](https://github.com/element-hq/synapse/issues/17184)) + +### Improved Documentation + +- Document [`/v1/make_knock`](https://spec.matrix.org/v1.10/server-server-api/#get_matrixfederationv1make_knockroomiduserid) and [`/v1/send_knock/`](https://spec.matrix.org/v1.10/server-server-api/#put_matrixfederationv1send_knockroomideventid) federation endpoints as worker-compatible. ([\#17058](https://github.com/element-hq/synapse/issues/17058)) +- Update User Admin API with note about prefixing OIDC external_id providers. ([\#17139](https://github.com/element-hq/synapse/issues/17139)) +- Clarify the state of the created room when using the `autocreate_auto_join_room_preset` config option. ([\#17150](https://github.com/element-hq/synapse/issues/17150)) +- Update the Admin FAQ with the current libjemalloc version for latest Debian stable. Additionally update the name of the "push_rules" stream in the Workers documentation. ([\#17171](https://github.com/element-hq/synapse/issues/17171)) + +### Internal Changes + +- Add note to reflect that [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) is closed but will remain supported for some time. ([\#17151](https://github.com/element-hq/synapse/issues/17151)) +- Update dependency PyO3 to 0.21. ([\#17162](https://github.com/element-hq/synapse/issues/17162)) +- Fixes linter errors found in PR #17147. ([\#17166](https://github.com/element-hq/synapse/issues/17166)) +- Bump black from 24.2.0 to 24.4.2. ([\#17170](https://github.com/element-hq/synapse/issues/17170)) +- Cache literal sync filter validation for performance. ([\#17186](https://github.com/element-hq/synapse/issues/17186)) +- Improve performance by fixing a reactor pause. ([\#17192](https://github.com/element-hq/synapse/issues/17192)) +- Route `/make_knock` and `/send_knock` federation APIs to the federation reader worker in Complement test runs. ([\#17195](https://github.com/element-hq/synapse/issues/17195)) +- Prepare sync handler to be able to return different sync responses (`SyncVersion`). ([\#17200](https://github.com/element-hq/synapse/issues/17200)) +- Organize the sync cache key parameter outside of the sync config (separate concerns). ([\#17201](https://github.com/element-hq/synapse/issues/17201)) +- Refactor `SyncResultBuilder` assembly to its own function. ([\#17202](https://github.com/element-hq/synapse/issues/17202)) +- Rename to be obvious: `joined_rooms` -> `joined_room_ids`. ([\#17203](https://github.com/element-hq/synapse/issues/17203), [\#17208](https://github.com/element-hq/synapse/issues/17208)) +- Add a short pause when rate-limiting a request. ([\#17210](https://github.com/element-hq/synapse/issues/17210)) + + + +### Updates to locked dependencies + +* Bump cryptography from 42.0.5 to 42.0.7. ([\#17180](https://github.com/element-hq/synapse/issues/17180)) +* Bump gitpython from 3.1.41 to 3.1.43. ([\#17181](https://github.com/element-hq/synapse/issues/17181)) +* Bump immutabledict from 4.1.0 to 4.2.0. ([\#17179](https://github.com/element-hq/synapse/issues/17179)) +* Bump sentry-sdk from 1.40.3 to 2.1.1. ([\#17178](https://github.com/element-hq/synapse/issues/17178)) +* Bump serde from 1.0.200 to 1.0.201. ([\#17183](https://github.com/element-hq/synapse/issues/17183)) +* Bump serde_json from 1.0.116 to 1.0.117. ([\#17182](https://github.com/element-hq/synapse/issues/17182)) + +Synapse 1.107.0 (2024-05-14) +============================ + +No significant changes since 1.107.0rc1. + + +# Synapse 1.107.0rc1 (2024-05-07) + +### Features + +- Add preliminary support for [MSC3823: Account Suspension](https://github.com/matrix-org/matrix-spec-proposals/pull/3823). ([\#17051](https://github.com/element-hq/synapse/issues/17051)) +- Declare support for [Matrix v1.10](https://matrix.org/blog/2024/03/22/matrix-v1.10-release/). Contributed by @clokep. ([\#17082](https://github.com/element-hq/synapse/issues/17082)) +- Add support for [MSC4115: membership metadata on events](https://github.com/matrix-org/matrix-spec-proposals/pull/4115). ([\#17104](https://github.com/element-hq/synapse/issues/17104), [\#17137](https://github.com/element-hq/synapse/issues/17137)) + +### Bugfixes + +- Fixed search feature of Element Android on homesevers using SQLite by returning search terms as search highlights. ([\#17000](https://github.com/element-hq/synapse/issues/17000)) +- Fixes a bug introduced in v1.52.0 where the `destination` query parameter for the [Destination Rooms Admin API](https://element-hq.github.io/synapse/v1.105/usage/administration/admin_api/federation.html#destination-rooms) failed to actually filter returned rooms. ([\#17077](https://github.com/element-hq/synapse/issues/17077)) +- For MSC3266 room summaries, support queries at the recommended endpoint of `/_matrix/client/unstable/im.nheko.summary/summary/{roomIdOrAlias}`. The existing endpoint of `/_matrix/client/unstable/im.nheko.summary/rooms/{roomIdOrAlias}/summary` is deprecated. ([\#17078](https://github.com/element-hq/synapse/issues/17078)) +- Apply user email & picture during OIDC registration if present & selected. ([\#17120](https://github.com/element-hq/synapse/issues/17120)) +- Improve error message for cross signing reset with [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) enabled. ([\#17121](https://github.com/element-hq/synapse/issues/17121)) +- Fix a bug which meant that to-device messages received over federation could be dropped when the server was under load or networking problems caused problems between Synapse processes or the database. ([\#17127](https://github.com/element-hq/synapse/issues/17127)) +- Fix bug where `StreamChangeCache` would not respect configured cache factors. ([\#17152](https://github.com/element-hq/synapse/issues/17152)) + +### Updates to the Docker image + +- Correct licensing metadata on Docker image. ([\#17141](https://github.com/element-hq/synapse/issues/17141)) + +### Improved Documentation + +- Update the `event_cache_size` and `global_factor` configuration options' documentation. ([\#17071](https://github.com/element-hq/synapse/issues/17071)) +- Remove broken sphinx docs. ([\#17073](https://github.com/element-hq/synapse/issues/17073), [\#17148](https://github.com/element-hq/synapse/issues/17148)) +- Add RuntimeDirectory to example matrix-synapse.service systemd unit. ([\#17084](https://github.com/element-hq/synapse/issues/17084)) +- Fix various small typos throughout the docs. ([\#17114](https://github.com/element-hq/synapse/issues/17114)) +- Update enable_notifs configuration documentation. ([\#17116](https://github.com/element-hq/synapse/issues/17116)) +- Update the Upgrade Notes with the latest minimum supported Rust version of 1.66.0. Contributed by @jahway603. ([\#17140](https://github.com/element-hq/synapse/issues/17140)) + +### Internal Changes + +- Enable [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) by default in the Synapse Complement image. ([\#17105](https://github.com/element-hq/synapse/issues/17105)) +- Add optimisation to `StreamChangeCache.get_entities_changed(..)`. ([\#17130](https://github.com/element-hq/synapse/issues/17130)) + + + +### Updates to locked dependencies + +* Bump furo from 2024.1.29 to 2024.4.27. ([\#17133](https://github.com/element-hq/synapse/issues/17133)) +* Bump idna from 3.6 to 3.7. ([\#17136](https://github.com/element-hq/synapse/issues/17136)) +* Bump jsonschema from 4.21.1 to 4.22.0. ([\#17157](https://github.com/element-hq/synapse/issues/17157)) +* Bump lxml from 5.1.0 to 5.2.1. ([\#17158](https://github.com/element-hq/synapse/issues/17158)) +* Bump phonenumbers from 8.13.29 to 8.13.35. ([\#17106](https://github.com/element-hq/synapse/issues/17106)) +- Bump pillow from 10.2.0 to 10.3.0. ([\#17146](https://github.com/element-hq/synapse/issues/17146)) +* Bump pydantic from 2.6.4 to 2.7.0. ([\#17107](https://github.com/element-hq/synapse/issues/17107)) +* Bump pydantic from 2.7.0 to 2.7.1. ([\#17160](https://github.com/element-hq/synapse/issues/17160)) +* Bump pyicu from 2.12 to 2.13. ([\#17109](https://github.com/element-hq/synapse/issues/17109)) +* Bump serde from 1.0.197 to 1.0.198. ([\#17111](https://github.com/element-hq/synapse/issues/17111)) +* Bump serde from 1.0.198 to 1.0.199. ([\#17132](https://github.com/element-hq/synapse/issues/17132)) +* Bump serde from 1.0.199 to 1.0.200. ([\#17161](https://github.com/element-hq/synapse/issues/17161)) +* Bump serde_json from 1.0.115 to 1.0.116. ([\#17112](https://github.com/element-hq/synapse/issues/17112)) +- Update `tornado` Python dependency from 6.2 to 6.4. ([\#17131](https://github.com/element-hq/synapse/issues/17131)) +* Bump twisted from 23.10.0 to 24.3.0. ([\#17135](https://github.com/element-hq/synapse/issues/17135)) +* Bump types-bleach from 6.1.0.1 to 6.1.0.20240331. ([\#17110](https://github.com/element-hq/synapse/issues/17110)) +* Bump types-pillow from 10.2.0.20240415 to 10.2.0.20240423. ([\#17159](https://github.com/element-hq/synapse/issues/17159)) +* Bump types-setuptools from 69.0.0.20240125 to 69.5.0.20240423. ([\#17134](https://github.com/element-hq/synapse/issues/17134)) + +# Synapse 1.106.0 (2024-04-30) + +No significant changes since 1.106.0rc1. + + + + +# Synapse 1.106.0rc1 (2024-04-25) + +### Features + +- Send an email if the address is already bound to an user account. ([\#16819](https://github.com/element-hq/synapse/issues/16819)) +- Implement the rendezvous mechanism described by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/issues/4108). ([\#17056](https://github.com/element-hq/synapse/issues/17056)) +- Support delegating the rendezvous mechanism described [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/issues/4108) to an external implementation. ([\#17086](https://github.com/element-hq/synapse/issues/17086)) + +### Bugfixes + +- Add validation to ensure that the `limit` parameter on `/publicRooms` is non-negative. ([\#16920](https://github.com/element-hq/synapse/issues/16920)) +- Return `400 M_NOT_JSON` upon receiving invalid JSON in query parameters across various client and admin endpoints, rather than an internal server error. ([\#16923](https://github.com/element-hq/synapse/issues/16923)) +- Make the CSAPI endpoint `/keys/device_signing/upload` idempotent. ([\#16943](https://github.com/element-hq/synapse/issues/16943)) +- Redact membership events if the user requested erasure upon deactivating. ([\#17076](https://github.com/element-hq/synapse/issues/17076)) + +### Improved Documentation + +- Add a prompt in the contributing guide to manually configure icu4c. ([\#17069](https://github.com/element-hq/synapse/issues/17069)) +- Clarify what part of message retention is still experimental. ([\#17099](https://github.com/element-hq/synapse/issues/17099)) + +### Internal Changes + +- Use new receipts column to optimise receipt and push action SQL queries. Contributed by Nick @ Beeper (@fizzadar). ([\#17032](https://github.com/element-hq/synapse/issues/17032), [\#17096](https://github.com/element-hq/synapse/issues/17096)) +- Fix mypy with latest Twisted release. ([\#17036](https://github.com/element-hq/synapse/issues/17036)) +- Bump minimum supported Rust version to 1.66.0. ([\#17079](https://github.com/element-hq/synapse/issues/17079)) +- Add helpers to transform Twisted requests to Rust http Requests/Responses. ([\#17081](https://github.com/element-hq/synapse/issues/17081)) +- Fix type annotation for `visited_chains` after `mypy` upgrade. ([\#17125](https://github.com/element-hq/synapse/issues/17125)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.81 to 1.0.82. ([\#17095](https://github.com/element-hq/synapse/issues/17095)) +* Bump peaceiris/actions-gh-pages from 3.9.3 to 4.0.0. ([\#17087](https://github.com/element-hq/synapse/issues/17087)) +* Bump peaceiris/actions-mdbook from 1.2.0 to 2.0.0. ([\#17089](https://github.com/element-hq/synapse/issues/17089)) +* Bump pyasn1-modules from 0.3.0 to 0.4.0. ([\#17093](https://github.com/element-hq/synapse/issues/17093)) +* Bump pygithub from 2.2.0 to 2.3.0. ([\#17092](https://github.com/element-hq/synapse/issues/17092)) +* Bump ruff from 0.3.5 to 0.3.7. ([\#17094](https://github.com/element-hq/synapse/issues/17094)) +* Bump sigstore/cosign-installer from 3.4.0 to 3.5.0. ([\#17088](https://github.com/element-hq/synapse/issues/17088)) +* Bump twine from 4.0.2 to 5.0.0. ([\#17091](https://github.com/element-hq/synapse/issues/17091)) +* Bump types-pillow from 10.2.0.20240406 to 10.2.0.20240415. ([\#17090](https://github.com/element-hq/synapse/issues/17090)) + +# Synapse 1.105.1 (2024-04-23) + +## Security advisory + +The following issues are fixed in 1.105.1. + +- [GHSA-3h7q-rfh9-xm4v](https://github.com/element-hq/synapse/security/advisories/GHSA-3h7q-rfh9-xm4v) / [CVE-2024-31208](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-31208) — High Severity + + Weakness in auth chain indexing allows DoS from remote room members through disk fill and high CPU usage. + +See the advisories for more details. If you have any questions, email security@element.io. + + + +# Synapse 1.105.0 (2024-04-16) + +No significant changes since 1.105.0rc1. + + + + +# Synapse 1.105.0rc1 (2024-04-11) + +### Features + +- Stabilize support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which clarifies the interaction of push rules and account data. Contributed by @clokep. ([\#17022](https://github.com/element-hq/synapse/issues/17022)) +- Stabilize support for [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981): `/relations` recursion. Contributed by @clokep. ([\#17023](https://github.com/element-hq/synapse/issues/17023)) +- Add support for moving `/pushrules` off of main process. ([\#17037](https://github.com/element-hq/synapse/issues/17037), [\#17038](https://github.com/element-hq/synapse/issues/17038)) + +### Bugfixes + +- Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16930](https://github.com/element-hq/synapse/issues/16930), [\#16932](https://github.com/element-hq/synapse/issues/16932), [\#16942](https://github.com/element-hq/synapse/issues/16942), [\#17064](https://github.com/element-hq/synapse/issues/17064), [\#17065](https://github.com/element-hq/synapse/issues/17065), [\#17066](https://github.com/element-hq/synapse/issues/17066)) +- Fix server notice rooms not always being created as unencrypted rooms, even when `encryption_enabled_by_default_for_room_type` is in use (server notices are always unencrypted). ([\#17033](https://github.com/element-hq/synapse/issues/17033)) +- Fix the `.m.rule.encrypted_room_one_to_one` and `.m.rule.room_one_to_one` default underride push rules being in the wrong order. Contributed by @Sumpy1. ([\#17043](https://github.com/element-hq/synapse/issues/17043)) + +### Internal Changes + +- Refactor auth chain fetching to reduce duplication. ([\#17044](https://github.com/element-hq/synapse/issues/17044)) +- Improve database performance by adding a missing index to `access_tokens.refresh_token_id`. ([\#17045](https://github.com/element-hq/synapse/issues/17045), [\#17054](https://github.com/element-hq/synapse/issues/17054)) +- Improve database performance by reducing number of receipts fetched when sending push notifications. ([\#17049](https://github.com/element-hq/synapse/issues/17049)) + + + +### Updates to locked dependencies + +* Bump packaging from 23.2 to 24.0. ([\#17027](https://github.com/element-hq/synapse/issues/17027)) +* Bump regex from 1.10.3 to 1.10.4. ([\#17028](https://github.com/element-hq/synapse/issues/17028)) +* Bump ruff from 0.3.2 to 0.3.5. ([\#17060](https://github.com/element-hq/synapse/issues/17060)) +* Bump serde_json from 1.0.114 to 1.0.115. ([\#17041](https://github.com/element-hq/synapse/issues/17041)) +* Bump types-pillow from 10.2.0.20240125 to 10.2.0.20240406. ([\#17061](https://github.com/element-hq/synapse/issues/17061)) +* Bump types-requests from 2.31.0.20240125 to 2.31.0.20240406. ([\#17063](https://github.com/element-hq/synapse/issues/17063)) +* Bump typing-extensions from 4.9.0 to 4.11.0. ([\#17062](https://github.com/element-hq/synapse/issues/17062)) + +# Synapse 1.104.0 (2024-04-02) + +### Bugfixes + +- Fix regression when using OIDC provider. Introduced in v1.104.0rc1. ([\#17031](https://github.com/element-hq/synapse/issues/17031)) + + +# Synapse 1.104.0rc1 (2024-03-26) + +### Features + +- Add an OIDC config to specify extra parameters for the authorization grant URL. IT can be useful to pass an ACR value for example. ([\#16971](https://github.com/element-hq/synapse/issues/16971)) +- Add support for OIDC provider returning JWT. ([\#16972](https://github.com/element-hq/synapse/issues/16972), [\#17031](https://github.com/element-hq/synapse/issues/17031)) + +### Bugfixes + +- Fix a bug which meant that, under certain circumstances, we might never retry sending events or to-device messages over federation after a failure. ([\#16925](https://github.com/element-hq/synapse/issues/16925)) +- Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16949](https://github.com/element-hq/synapse/issues/16949)) +- Fix case in which `m.fully_read` marker would not get updated. Contributed by @SpiritCroc. ([\#16990](https://github.com/element-hq/synapse/issues/16990)) +- Fix bug which did not retract a user's pending knocks at rooms when their account was deactivated. Contributed by @hanadi92. ([\#17010](https://github.com/element-hq/synapse/issues/17010)) + +### Updates to the Docker image + +- Updated `start.py` to generate config using the correct user ID when running as root (fixes [\#16824](https://github.com/element-hq/synapse/issues/16824), [\#15202](https://github.com/element-hq/synapse/issues/15202)). ([\#16978](https://github.com/element-hq/synapse/issues/16978)) + +### Improved Documentation + +- Add a query to force a refresh of a remote user's device list to the "Useful SQL for Admins" documentation page. ([\#16892](https://github.com/element-hq/synapse/issues/16892)) +- Minor grammatical corrections to the upgrade documentation. ([\#16965](https://github.com/element-hq/synapse/issues/16965)) +- Fix the sort order for the documentation version picker, so that newer releases appear above older ones. ([\#16966](https://github.com/element-hq/synapse/issues/16966)) +- Remove recommendation for a specific poetry version from contributing guide. ([\#17002](https://github.com/element-hq/synapse/issues/17002)) + +### Internal Changes + +- Improve lock performance when a lot of locks are all waiting for a single lock to be released. ([\#16840](https://github.com/element-hq/synapse/issues/16840)) +- Update power level default for public rooms. ([\#16907](https://github.com/element-hq/synapse/issues/16907)) +- Improve event validation. ([\#16908](https://github.com/element-hq/synapse/issues/16908)) +- Multi-worker-docker-container: disable log buffering. ([\#16919](https://github.com/element-hq/synapse/issues/16919)) +- Refactor state delta calculation in `/sync` handler. ([\#16929](https://github.com/element-hq/synapse/issues/16929)) +- Clarify docs for some room state functions. ([\#16950](https://github.com/element-hq/synapse/issues/16950)) +- Specify IP subnets in canonical form. ([\#16953](https://github.com/element-hq/synapse/issues/16953)) +- As done for SAML mapping provider, let's pass the module API to the OIDC one so the mapper can do more logic in its code. ([\#16974](https://github.com/element-hq/synapse/issues/16974)) +- Allow containers building on top of Synapse's Complement container is use the included PostgreSQL cluster. ([\#16985](https://github.com/element-hq/synapse/issues/16985)) +- Raise poetry-core version cap to 1.9.0. ([\#16986](https://github.com/element-hq/synapse/issues/16986)) +- Patch the db conn pool sooner in tests. ([\#17017](https://github.com/element-hq/synapse/issues/17017)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.80 to 1.0.81. ([\#17009](https://github.com/element-hq/synapse/issues/17009)) +* Bump black from 23.10.1 to 24.2.0. ([\#16936](https://github.com/element-hq/synapse/issues/16936)) +* Bump cryptography from 41.0.7 to 42.0.5. ([\#16958](https://github.com/element-hq/synapse/issues/16958)) +* Bump dawidd6/action-download-artifact from 3.1.1 to 3.1.2. ([\#16960](https://github.com/element-hq/synapse/issues/16960)) +* Bump dawidd6/action-download-artifact from 3.1.2 to 3.1.4. ([\#17008](https://github.com/element-hq/synapse/issues/17008)) +* Bump jinja2 from 3.1.2 to 3.1.3. ([\#17005](https://github.com/element-hq/synapse/issues/17005)) +* Bump log from 0.4.20 to 0.4.21. ([\#16977](https://github.com/element-hq/synapse/issues/16977)) +* Bump mypy from 1.5.1 to 1.8.0. ([\#16901](https://github.com/element-hq/synapse/issues/16901)) +* Bump netaddr from 0.9.0 to 1.2.1. ([\#17006](https://github.com/element-hq/synapse/issues/17006)) +* Bump pydantic from 2.6.0 to 2.6.4. ([\#17004](https://github.com/element-hq/synapse/issues/17004)) +* Bump pyo3 from 0.20.2 to 0.20.3. ([\#16962](https://github.com/element-hq/synapse/issues/16962)) +* Bump ruff from 0.1.14 to 0.3.2. ([\#16994](https://github.com/element-hq/synapse/issues/16994)) +* Bump serde from 1.0.196 to 1.0.197. ([\#16963](https://github.com/element-hq/synapse/issues/16963)) +* Bump serde_json from 1.0.113 to 1.0.114. ([\#16961](https://github.com/element-hq/synapse/issues/16961)) +* Bump types-jsonschema from 4.21.0.20240118 to 4.21.0.20240311. ([\#17007](https://github.com/element-hq/synapse/issues/17007)) +* Bump types-psycopg2 from 2.9.21.16 to 2.9.21.20240311. ([\#16995](https://github.com/element-hq/synapse/issues/16995)) +* Bump types-pyopenssl from 23.3.0.0 to 24.0.0.20240311. ([\#17003](https://github.com/element-hq/synapse/issues/17003)) + +# Synapse 1.103.0 (2024-03-19) + +No significant changes since 1.103.0rc1. + + + + +# Synapse 1.103.0rc1 (2024-03-12) + +### Features + +- Add a new [List Accounts v3](https://element-hq.github.io/synapse/v1.103/admin_api/user_admin_api.html#list-accounts-v3) Admin API with improved deactivated user filtering capabilities. ([\#16874](https://github.com/element-hq/synapse/issues/16874)) +- Include `Retry-After` header by default per [MSC4041](https://github.com/matrix-org/matrix-spec-proposals/pull/4041). Contributed by @clokep. ([\#16947](https://github.com/element-hq/synapse/issues/16947)) + +### Bugfixes + +- Fix joining remote rooms when a module uses the `on_new_event` callback. This callback may now pass partial state events instead of the full state for remote rooms. Introduced in v1.76.0. ([\#16973](https://github.com/element-hq/synapse/issues/16973)) +- Fix performance issue when joining very large rooms that can cause the server to lock up. Introduced in v1.100.0. Contributed by @ggogel. ([\#16968](https://github.com/element-hq/synapse/issues/16968)) + +### Improved Documentation + +- Add HAProxy example for single port operation to reverse proxy documentation. Contributed by Georg Pfuetzenreuter (@tacerus). ([\#16768](https://github.com/element-hq/synapse/issues/16768)) +- Improve the documentation around running Complement tests with new configuration parameters. ([\#16946](https://github.com/element-hq/synapse/issues/16946)) +- Add docs on upgrading from a very old version. ([\#16951](https://github.com/element-hq/synapse/issues/16951)) + + +### Updates to locked dependencies + +* Bump JasonEtco/create-an-issue from 2.9.1 to 2.9.2. ([\#16934](https://github.com/element-hq/synapse/issues/16934)) +* Bump anyhow from 1.0.79 to 1.0.80. ([\#16935](https://github.com/element-hq/synapse/issues/16935)) +* Bump dawidd6/action-download-artifact from 3.0.0 to 3.1.1. ([\#16933](https://github.com/element-hq/synapse/issues/16933)) +* Bump furo from 2023.9.10 to 2024.1.29. ([\#16939](https://github.com/element-hq/synapse/issues/16939)) +* Bump pyopenssl from 23.3.0 to 24.0.0. ([\#16937](https://github.com/element-hq/synapse/issues/16937)) +* Bump types-netaddr from 0.10.0.20240106 to 1.2.0.20240219. ([\#16938](https://github.com/element-hq/synapse/issues/16938)) + + +# Synapse 1.102.0 (2024-03-05) + +### Bugfixes + +- Revert https://github.com/element-hq/synapse/pull/16756, which caused incorrect notification counts on mobile clients since v1.100.0. ([\#16979](https://github.com/element-hq/synapse/issues/16979)) + + +# Synapse 1.102.0rc1 (2024-02-20) + +### Features + +- A metric was added for emails sent by Synapse, broken down by type: `synapse_emails_sent_total`. Contributed by Remi Rampin. ([\#16881](https://github.com/element-hq/synapse/issues/16881)) + +### Bugfixes + +- Do not send multiple concurrent requests for keys for the same server. ([\#16894](https://github.com/element-hq/synapse/issues/16894)) +- Fix performance issue when joining very large rooms that can cause the server to lock up. Introduced in v1.100.0. ([\#16903](https://github.com/element-hq/synapse/issues/16903)) +- Always prefer unthreaded receipt when >1 exist ([MSC4102](https://github.com/matrix-org/matrix-spec-proposals/pull/4102)). ([\#16927](https://github.com/element-hq/synapse/issues/16927)) + +### Improved Documentation + +- Fix a small typo in the Rooms section of the Admin API documentation. Contributed by @RainerZufall187. ([\#16857](https://github.com/element-hq/synapse/issues/16857)) + +### Internal Changes + +- Don't invalidate the entire event cache when we purge history. ([\#16905](https://github.com/element-hq/synapse/issues/16905)) +- Add experimental config option to not send device list updates for specific users. ([\#16909](https://github.com/element-hq/synapse/issues/16909)) +- Fix incorrect docker hub link in release script. ([\#16910](https://github.com/element-hq/synapse/issues/16910)) + + + +### Updates to locked dependencies + +* Bump attrs from 23.1.0 to 23.2.0. ([\#16899](https://github.com/element-hq/synapse/issues/16899)) +* Bump bcrypt from 4.0.1 to 4.1.2. ([\#16900](https://github.com/element-hq/synapse/issues/16900)) +* Bump pygithub from 2.1.1 to 2.2.0. ([\#16902](https://github.com/element-hq/synapse/issues/16902)) +* Bump sentry-sdk from 1.40.0 to 1.40.3. ([\#16898](https://github.com/element-hq/synapse/issues/16898)) + +# Synapse 1.101.0 (2024-02-13) + +### Bugfixes + +- Fix performance regression when fetching auth chains from the DB. Introduced in v1.100.0. ([\#16893](https://github.com/element-hq/synapse/issues/16893)) + + + + +# Synapse 1.101.0rc1 (2024-02-06) + +### Improved Documentation + +- Fix broken links in the documentation. ([\#16853](https://github.com/element-hq/synapse/issues/16853)) +- Update MacOS installation instructions to mention that libicu is optional. ([\#16854](https://github.com/element-hq/synapse/issues/16854)) +- The version picker now correctly lists versions after `v1.98.0`. ([\#16880](https://github.com/element-hq/synapse/issues/16880)) + +### Internal Changes + +- Add support for stabilised [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981) that adds a `recurse` parameter on the `/relations` API. ([\#16842](https://github.com/element-hq/synapse/issues/16842)) + + + +### Updates to locked dependencies + +* Bump dorny/paths-filter from 2 to 3. ([\#16869](https://github.com/element-hq/synapse/issues/16869)) +* Bump gitpython from 3.1.40 to 3.1.41. ([\#16850](https://github.com/element-hq/synapse/issues/16850)) +* Bump hiredis from 2.2.3 to 2.3.2. ([\#16862](https://github.com/element-hq/synapse/issues/16862)) +* Bump jsonschema from 4.20.0 to 4.21.1. ([\#16887](https://github.com/element-hq/synapse/issues/16887)) +* Bump lxml-stubs from 0.4.0 to 0.5.1. ([\#16885](https://github.com/element-hq/synapse/issues/16885)) +* Bump mypy-zope from 1.0.1 to 1.0.3. ([\#16865](https://github.com/element-hq/synapse/issues/16865)) +* Bump phonenumbers from 8.13.26 to 8.13.29. ([\#16868](https://github.com/element-hq/synapse/issues/16868)) +* Bump pydantic from 2.5.3 to 2.6.0. ([\#16888](https://github.com/element-hq/synapse/issues/16888)) +* Bump sentry-sdk from 1.39.1 to 1.40.0. ([\#16889](https://github.com/element-hq/synapse/issues/16889)) +* Bump serde from 1.0.195 to 1.0.196. ([\#16867](https://github.com/element-hq/synapse/issues/16867)) +* Bump serde_json from 1.0.111 to 1.0.113. ([\#16866](https://github.com/element-hq/synapse/issues/16866)) +* Bump sigstore/cosign-installer from 3.3.0 to 3.4.0. ([\#16890](https://github.com/element-hq/synapse/issues/16890)) +* Bump types-pillow from 10.1.0.2 to 10.2.0.20240125. ([\#16864](https://github.com/element-hq/synapse/issues/16864)) +* Bump types-requests from 2.31.0.10 to 2.31.0.20240125. ([\#16886](https://github.com/element-hq/synapse/issues/16886)) +* Bump types-setuptools from 69.0.0.0 to 69.0.0.20240125. ([\#16863](https://github.com/element-hq/synapse/issues/16863)) + +# Synapse 1.100.0 (2024-01-30) + +No significant changes since 1.100.0rc3. + + + + +# Synapse 1.100.0rc3 (2024-01-24) + +### Bugfixes + +- Fix database performance regression due to changing Postgres table statistics. Introduced in v1.100.0rc1. ([\#16849](https://github.com/element-hq/synapse/issues/16849)) + + + + +# Synapse 1.100.0rc2 (2024-01-24) + +This version is the same as 1.100.0rc1 but with fixes to the release process. + +### Internal Changes + +- Downgrade the `download-artifact` and `upload-artifact` actions to v3 due to breaking changes. ([\#16847](https://github.com/element-hq/synapse/issues/16847)) + + +# Synapse 1.100.0rc1 (2024-01-23) + +*This version was never released to PyPI or the Debian repository due to failures in the automatic part of the release process.* + +### Features + +- Advertise experimental support for [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028) through `/_matrix/clients/versions` if enabled. Contributed by @hanadi92. ([\#16787](https://github.com/element-hq/synapse/issues/16787)) + +### Bugfixes + +- Handle wildcard type filters properly for room messages endpoint. Contributed by Mo Balaa. ([\#14984](https://github.com/element-hq/synapse/issues/14984)) + +### Improved Documentation + +- Add a link to the "Request log format" explainer on the "Logging sample config" documentation page. ([\#16778](https://github.com/element-hq/synapse/issues/16778)) +- Fix broken links in issue templates and documentation. ([\#16810](https://github.com/element-hq/synapse/issues/16810)) +- NGINX listen http2 deprecation in documentation template for reverse proxy. ([\#16831](https://github.com/element-hq/synapse/issues/16831)) + +### Internal Changes + +- Faster partial join to room with complex auth graph. ([\#7](https://github.com/element-hq/synapse/issues/7)) +- Improve DB performance of calculating badge counts for push. ([\#16756](https://github.com/element-hq/synapse/issues/16756)) +- Split up deleting devices into batches. ([\#16766](https://github.com/element-hq/synapse/issues/16766)) +- Remove CI check for sign-off as we require a CLA signature instead. ([\#16776](https://github.com/element-hq/synapse/issues/16776)) +- Ensure CI fails when linting fails to make sure auto-merge does the correct thing. ([\#16781](https://github.com/element-hq/synapse/issues/16781)) +- Faster load recents for sync by reducing amount of state pulled out. ([\#16783](https://github.com/element-hq/synapse/issues/16783)) +- Reduce amount of state pulled out when querying federation hierachy. ([\#16785](https://github.com/element-hq/synapse/issues/16785)) +- Pull less state out of the DB when we retry fetching old events during backfill. ([\#16788](https://github.com/element-hq/synapse/issues/16788)) +- Optimize query for fetching to-device messages in `/sync`. ([\#16805](https://github.com/element-hq/synapse/issues/16805)) +- Reject OIDC config when `client_secret` isn't specified, but the auth method requires one. ([\#16806](https://github.com/element-hq/synapse/issues/16806)) +- Allow room creation but not publishing to continue if room publication rules are violated when creating + a new room. ([\#16811](https://github.com/element-hq/synapse/issues/16811)) +- Bump minimum supported Rust version to 1.65.0. ([\#16818](https://github.com/element-hq/synapse/issues/16818)) +- Fixup copyright lines in file headers after the licensing change. ([\#16820](https://github.com/element-hq/synapse/issues/16820)) +- Add a `--generate-only` option to the internal configuration/launch script for Complement. ([\#16828](https://github.com/element-hq/synapse/issues/16828)) +- Preparatory work for tweaking performance of auth chain lookups. ([\#16833](https://github.com/element-hq/synapse/issues/16833)) +- Speed up e2e device keys queries for bot accounts. ([\#16841](https://github.com/element-hq/synapse/issues/16841)) + +### Updates to locked dependencies + +* Bump actions/cache from 3 to 4. ([\#16832](https://github.com/element-hq/synapse/issues/16832)) +* Bump actions/download-artifact from 3 to 4. ([\#16795](https://github.com/element-hq/synapse/issues/16795)) +* Bump actions/upload-artifact from 3 to 4. ([\#16796](https://github.com/element-hq/synapse/issues/16796)) +* Bump anyhow from 1.0.75 to 1.0.79. ([\#16789](https://github.com/element-hq/synapse/issues/16789)) +* Bump authlib from 1.2.1 to 1.3.0. ([\#16801](https://github.com/element-hq/synapse/issues/16801)) +* Bump dawidd6/action-download-artifact from 2.28.0 to 3.0.0. ([\#16794](https://github.com/element-hq/synapse/issues/16794)) +* Bump immutabledict from 4.0.0 to 4.1.0. ([\#16812](https://github.com/element-hq/synapse/issues/16812)) +* Bump isort from 5.13.1 to 5.13.2. ([\#16835](https://github.com/element-hq/synapse/issues/16835)) +* Bump lxml from 4.9.3 to 5.1.0. ([\#16813](https://github.com/element-hq/synapse/issues/16813)) +* Bump pillow from 10.1.0 to 10.2.0. ([\#16802](https://github.com/element-hq/synapse/issues/16802)) +* Bump pydantic from 2.5.2 to 2.5.3. ([\#16836](https://github.com/element-hq/synapse/issues/16836)) +* Bump pyo3 from 0.20.0 to 0.20.2. ([\#16791](https://github.com/element-hq/synapse/issues/16791)) +* Bump regex from 1.9.6 to 1.10.3. ([\#16837](https://github.com/element-hq/synapse/issues/16837)) +* Bump ruff from 0.1.13 to 0.1.14. ([\#16838](https://github.com/element-hq/synapse/issues/16838)) +* Bump ruff from 0.1.7 to 0.1.13. ([\#16814](https://github.com/element-hq/synapse/issues/16814)) +* Bump sentry-sdk from 1.35.0 to 1.39.1. ([\#16799](https://github.com/element-hq/synapse/issues/16799)) +* Bump serde_json from 1.0.108 to 1.0.111. ([\#16792](https://github.com/element-hq/synapse/issues/16792)) +* Bump service-identity from 23.1.0 to 24.1.0. ([\#16816](https://github.com/element-hq/synapse/issues/16816)) +* Bump types-commonmark from 0.9.2.4 to 0.9.2.20240106. ([\#16797](https://github.com/element-hq/synapse/issues/16797)) +* Bump types-jsonschema from 4.20.0.0 to 4.20.0.20240105. ([\#16800](https://github.com/element-hq/synapse/issues/16800)) +* Bump types-jsonschema from 4.20.0.20240105 to 4.21.0.20240118. ([\#16834](https://github.com/element-hq/synapse/issues/16834)) +* Bump types-netaddr from 0.9.0.1 to 0.10.0.20240106. ([\#16839](https://github.com/element-hq/synapse/issues/16839)) +* Bump typing-extensions from 4.8.0 to 4.9.0. ([\#16815](https://github.com/element-hq/synapse/issues/16815)) + + +# Synapse 1.99.0 (2024-01-16) + +Synapse 1.99.0 is the first Synapse release under an AGPLv3.0 licence (with CLA to enable Element to sell AGPL +exceptions). You can read more about this here: + + - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ + - https://element.io/blog/element-to-adopt-agplv3/ + - https://element.io/blog/synapse-now-lives-at-github-com-element-hq-synapse/ + +No significant changes since 1.99.0rc1. + + +# Synapse 1.99.0rc1 (2024-01-09) + +### Features + +- Add [config options](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#server_notices) to set the avatar and the topic of the server notices room, as well as the avatar of the server notices user. ([\#16679](https://github.com/matrix-org/synapse/issues/16679)) +- Add config option [`email.notif_delay_before_mail`](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#email) to tweak the delay before an email is sent following a notification. ([\#16696](https://github.com/matrix-org/synapse/issues/16696)) +- Add new configuration option [`sentry.environment`](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#sentry) for improved system monitoring. Contributed by @zeeshanrafiqrana. ([\#16738](https://github.com/matrix-org/synapse/issues/16738)) +- Filter out rooms from the room directory being served to other homeservers when those rooms block that homeserver by their Access Control Lists. ([\#16759](https://github.com/element-hq/synapse/issues/16759)) + +### Bugfixes + +- Fix a long-standing bug where the signing keys generated by Synapse were world-readable. Contributed by Fabian Klemp. ([\#16740](https://github.com/matrix-org/synapse/issues/16740)) +- Fix email verification redirection. Contributed by Fadhlan Ridhwanallah. ([\#16761](https://github.com/element-hq/synapse/issues/16761)) +- Fixed a bug that prevented users from being queried by display name if it contains non-ASCII characters. ([\#16767](https://github.com/element-hq/synapse/issues/16767)) +- Allow reactivate user without password with Admin API in some edge cases. ([\#16770](https://github.com/element-hq/synapse/issues/16770)) +- Adds the `recursion_depth` parameter to the response of the /relations endpoint if MSC3981 recursion is being performed. ([\#16775](https://github.com/element-hq/synapse/issues/16775)) + +### Improved Documentation + +- Added version picker for Synapse documentation. Contributed by @Dmytro27Ind. ([\#16533](https://github.com/matrix-org/synapse/issues/16533)) +- Clarify that `password_config.enabled: "only_for_reauth"` does not allow new logins to be created using password auth. ([\#16737](https://github.com/matrix-org/synapse/issues/16737)) +- Remove value from header in configuration documentation for `refresh_token_lifetime`. ([\#16763](https://github.com/element-hq/synapse/issues/16763)) +- Add another custom statistics collection server to the documentation. Contributed by @loelkes. ([\#16769](https://github.com/element-hq/synapse/issues/16769)) + +### Internal Changes + +- Remove run-once workflow after adding the version picker to the documentation. ([\#9453](https://github.com/element-hq/synapse/issues/9453)) +- Update the implementation of [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965) (OIDC Provider discovery). ([\#16726](https://github.com/matrix-org/synapse/issues/16726)) +- Move the rust stubs inline for better IDE integration. ([\#16757](https://github.com/element-hq/synapse/issues/16757)) +- Fix sample config doc CI. ([\#16758](https://github.com/element-hq/synapse/issues/16758)) +- Simplify event internal metadata class. ([\#16762](https://github.com/element-hq/synapse/issues/16762), [\#16780](https://github.com/element-hq/synapse/issues/16780)) +- Sign the published docker image using [cosign](https://docs.sigstore.dev/). ([\#16774](https://github.com/element-hq/synapse/issues/16774)) +- Port `EventInternalMetadata` class to Rust. ([\#16782](https://github.com/element-hq/synapse/issues/16782)) + + + +### Updates to locked dependencies + +* Bump actions/setup-go from 4 to 5. ([\#16749](https://github.com/matrix-org/synapse/issues/16749)) +* Bump actions/setup-python from 4 to 5. ([\#16748](https://github.com/matrix-org/synapse/issues/16748)) +* Bump immutabledict from 3.0.0 to 4.0.0. ([\#16743](https://github.com/matrix-org/synapse/issues/16743)) +* Bump isort from 5.12.0 to 5.13.0. ([\#16745](https://github.com/matrix-org/synapse/issues/16745)) +* Bump isort from 5.13.0 to 5.13.1. ([\#16752](https://github.com/matrix-org/synapse/issues/16752)) +* Bump pydantic from 2.5.1 to 2.5.2. ([\#16747](https://github.com/matrix-org/synapse/issues/16747)) +* Bump ruff from 0.1.6 to 0.1.7. ([\#16746](https://github.com/matrix-org/synapse/issues/16746)) +* Bump types-setuptools from 68.2.0.2 to 69.0.0.0. ([\#16744](https://github.com/matrix-org/synapse/issues/16744)) diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index 8403664850..2f3a09723e 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -1,13 +1,11 @@ -Deprecation Policy for Platform Dependencies -============================================ +# Deprecation Policy -Synapse has a number of platform dependencies, including Python, Rust, -PostgreSQL and SQLite. This document outlines the policy towards which versions -we support, and when we drop support for versions in the future. +Synapse has a number of **platform dependencies** (Python, Rust, PostgreSQL, and SQLite) +and **application dependencies** (Python and Rust packages). This document outlines the +policy towards which versions we support, and when we drop support for versions in the +future. - -Policy ------- +## Platform Dependencies Synapse follows the upstream support life cycles for Python and PostgreSQL, i.e. when a version reaches End of Life Synapse will withdraw support for that @@ -26,8 +24,8 @@ The oldest supported version of SQLite is the version [provided](https://packages.debian.org/bullseye/libsqlite3-0) by [Debian oldstable](https://wiki.debian.org/DebianOldStable). -Context -------- + +### Context It is important for system admins to have a clear understanding of the platform requirements of Synapse and its deprecation policies so that they can @@ -50,4 +48,42 @@ the ecosystem. On a similar note, SQLite does not generally have a concept of "supported release"; bugfixes are published for the latest minor release only. We chose to track Debian's oldstable as this is relatively conservative, predictably updated -and is consistent with the `.deb` packages released by Matrix.org. \ No newline at end of file +and is consistent with the `.deb` packages released by Matrix.org. + + +## Application dependencies + +For application-level Python dependencies, we often specify loose version constraints +(ex. `>=X.Y.Z`) to be forwards compatible with any new versions. Upper bounds (` Optional[JsonDict] +``` + +** +Caution: This callback is currently experimental . The method signature or behaviour +may change without notice. +** + +Called when processing a request from a client for the +[media config endpoint](https://spec.matrix.org/latest/client-server-api/#get_matrixclientv1mediaconfig). + +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`) making the request. + +If the callback returns a dictionary then it will be used as the body of the response to the +client. + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `None`, Synapse falls through to the next one. The value of the first +callback that does not return `None` will be used. If this happens, Synapse will not call +any of the subsequent implementations of this callback. + +If no module returns a non-`None` value then the default media config will be returned. + +### `is_user_allowed_to_upload_media_of_size` + +_First introduced in Synapse v1.132.0_ + +```python +async def is_user_allowed_to_upload_media_of_size(user_id: str, size: int) -> bool +``` + +** +Caution: This callback is currently experimental . The method signature or behaviour +may change without notice. +** + +Called before media is accepted for upload from a user, in case the module needs to +enforce a different limit for the particular user. + +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`) making the request. +* `size`: The size in bytes of media that is being requested to upload. + +If the module returns `False`, the current request will be denied with the error code +`M_TOO_LARGE` and the HTTP status code 413. + +If multiple modules implement this callback, they will be considered in order. If a callback +returns `True`, Synapse falls through to the next one. The value of the first callback that +returns `False` will be used. If this happens, Synapse will not call any of the subsequent +implementations of this callback. + +### `get_media_upload_limits_for_user` + +_First introduced in Synapse v1.139.0_ + +```python +async def get_media_upload_limits_for_user(user_id: str, size: int) -> Optional[List[synapse.module_api.MediaUploadLimit]] +``` + +** +Caution: This callback is currently experimental. The method signature or behaviour +may change without notice. +** + +Called when processing a request to store content in the media repository. This can be used to dynamically override +the [media upload limits configuration](../usage/configuration/config_documentation.html#media_upload_limits). + +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`) making the request. + +If the callback returns a list then it will be used as the limits instead of those in the configuration (if any). + +If an empty list is returned then no limits are applied (**warning:** users will be able +to upload as much data as they desire). + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `None`, Synapse falls through to the next one. The value of the first +callback that does not return `None` will be used. If this happens, Synapse will not call +any of the subsequent implementations of this callback. + +If there are no registered modules, or if all modules return `None`, then +the default +[media upload limits configuration](../usage/configuration/config_documentation.html#media_upload_limits) +will be used. + +### `on_media_upload_limit_exceeded` + +_First introduced in Synapse v1.139.0_ + +```python +async def on_media_upload_limit_exceeded(user_id: str, limit: synapse.module_api.MediaUploadLimit, sent_bytes: int, attempted_bytes: int) -> None +``` + +** +Caution: This callback is currently experimental. The method signature or behaviour +may change without notice. +** + +Called when a user attempts to upload media that would exceed a +[configured media upload limit](../usage/configuration/config_documentation.html#media_upload_limits). + +This callback will only be called on workers which handle +[POST /_matrix/media/v3/upload](https://spec.matrix.org/v1.15/client-server-api/#post_matrixmediav3upload) +requests. + +This could be used to inform the user that they have reached a media upload limit through +some external method. + +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`) making the request. +* `limit`: The `synapse.module_api.MediaUploadLimit` representing the limit that was reached. +* `sent_bytes`: The number of bytes already sent during the period of the limit. +* `attempted_bytes`: The number of bytes that the user attempted to send. diff --git a/docs/modules/ratelimit_callbacks.md b/docs/modules/ratelimit_callbacks.md new file mode 100644 index 0000000000..30d94024fa --- /dev/null +++ b/docs/modules/ratelimit_callbacks.md @@ -0,0 +1,43 @@ +# Ratelimit callbacks + +Ratelimit callbacks allow module developers to override ratelimit settings dynamically whilst +Synapse is running. Ratelimit callbacks can be registered using the module API's +`register_ratelimit_callbacks` method. + +The available ratelimit callbacks are: + +### `get_ratelimit_override_for_user` + +_First introduced in Synapse v1.132.0_ + +```python +async def get_ratelimit_override_for_user(user: str, limiter_name: str) -> Optional[synapse.module_api.RatelimitOverride] +``` + +** +Caution: This callback is currently experimental . The method signature or behaviour +may change without notice. +** + +Called when constructing a ratelimiter of a particular type for a user. The module can +return a `messages_per_second` and `burst_count` to be used, or `None` if +the default settings are adequate. The user is represented by their Matrix user ID +(e.g. `@alice:example.com`). The limiter name is usually taken from the `RatelimitSettings` key +value. + +The limiters that are currently supported are: + +- `rc_invites.per_room` +- `rc_invites.per_user` +- `rc_invites.per_issuer` + +The `RatelimitOverride` return type has the following fields: + +- `per_second: float`. The number of actions that can be performed in a second. `0.0` means that ratelimiting is disabled. +- `burst_count: int`. The number of actions that can be performed before being limited. + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `None`, Synapse falls through to the next one. The value of the first +callback that does not return `None` will be used. If this happens, Synapse will not call +any of the subsequent implementations of this callback. If no module returns a non-`None` value +then the default settings will be used. diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index ec306d81ab..49b7e06bb3 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -80,6 +80,8 @@ Called when processing an invitation, both when one is created locally or when receiving an invite over federation. Both inviter and invitee are represented by their Matrix user ID (e.g. `@alice:example.com`). +Note that federated invites will call `federated_user_may_invite` before this callback. + The callback must return one of: - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still @@ -97,6 +99,34 @@ be used. If this happens, Synapse will not call any of the subsequent implementa this callback. +### `federated_user_may_invite` + +_First introduced in Synapse v1.133.0_ + +```python +async def federated_user_may_invite(event: "synapse.events.EventBase") -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] +``` + +Called when processing an invitation received over federation. Unlike `user_may_invite`, +this callback receives the entire event, including any stripped state in the `unsigned` +section, not just the room and user IDs. + +The callback must return one of: + - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. + - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. +The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will +be used. If this happens, Synapse will not call any of the subsequent implementations of +this callback. + +If all of the callbacks return `synapse.module_api.NOT_SPAM`, Synapse will also fall +through to the `user_may_invite` callback before approving the invite. + + ### `user_may_send_3pid_invite` _First introduced in Synapse v1.45.0_ @@ -159,12 +189,19 @@ _First introduced in Synapse v1.37.0_ _Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ +_Changed in Synapse v1.132.0: Added the `room_config` argument. Callbacks that only expect a single `user_id` argument are still supported._ + ```python -async def user_may_create_room(user_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] +async def user_may_create_room(user_id: str, room_config: synapse.module_api.JsonDict) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] ``` Called when processing a room creation request. +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`). +* `room_config`: The contents of the body of a [/createRoom request](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom) as a dictionary. + The callback must return one of: - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still decide to reject it. @@ -239,13 +276,48 @@ be used. If this happens, Synapse will not call any of the subsequent implementa this callback. +### `user_may_send_state_event` + +_First introduced in Synapse v1.132.0_ + +```python +async def user_may_send_state_event(user_id: str, room_id: str, event_type: str, state_key: str, content: JsonDict) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"] +``` + +** +Caution: This callback is currently experimental . The method signature or behaviour +may change without notice. +** + +Called when processing a request to [send state events](https://spec.matrix.org/latest/client-server-api/#put_matrixclientv3roomsroomidstateeventtypestatekey) to a room. + +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`) sending the state event. +* `room_id`: The ID of the room that the requested state event is being sent to. +* `event_type`: The requested type of event. +* `state_key`: The requested state key. +* `content`: The requested event contents. + +The callback must return one of: + - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. + - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. +The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will +be used. If this happens, Synapse will not call any of the subsequent implementations of +this callback. + ### `check_username_for_spam` _First introduced in Synapse v1.37.0_ ```python -async def check_username_for_spam(user_profile: synapse.module_api.UserProfile) -> bool +async def check_username_for_spam(user_profile: synapse.module_api.UserProfile, requester_id: str) -> bool ``` Called when computing search results in the user directory. The module must return a @@ -264,6 +336,8 @@ The profile is represented as a dictionary with the following keys: The module is given a copy of the original dictionary, so modifying it from within the module cannot modify a user's profile when included in user directory search results. +The requester_id parameter is the ID of the user that called the user directory API. + If multiple modules implement this callback, they will be considered in order. If a callback returns `False`, Synapse falls through to the next one. The value of the first callback that does not return `False` will be used. If this happens, Synapse will not call @@ -351,6 +425,8 @@ callback returns `False`, Synapse falls through to the next one. The value of th callback that does not return `False` will be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. +Note that this check is applied to federation invites as of Synapse v1.130.0. + ### `check_login_for_spam` diff --git a/docs/openid.md b/docs/openid.md index 5a3d7e9fba..819f754390 100644 --- a/docs/openid.md +++ b/docs/openid.md @@ -23,6 +23,7 @@ such as [Github][github-idp]. [auth0]: https://auth0.com/ [authentik]: https://goauthentik.io/ [lemonldap]: https://lemonldap-ng.org/ +[pocket-id]: https://pocket-id.org/ [okta]: https://www.okta.com/ [dex-idp]: https://github.com/dexidp/dex [keycloak-idp]: https://www.keycloak.org/docs/latest/server_admin/#sso-protocols @@ -185,6 +186,7 @@ oidc_providers: 4. Note the slug of your application, Client ID and Client Secret. Note: RSA keys must be used for signing for Authentik, ECC keys do not work. +Note: The provider must have a signing key set and must not use an encryption key. Synapse config: ```yaml @@ -203,6 +205,12 @@ oidc_providers: config: localpart_template: "{{ user.preferred_username }}" display_name_template: "{{ user.preferred_username|capitalize }}" # TO BE FILLED: If your users have names in Authentik and you want those in Synapse, this should be replaced with user.name|capitalize. +[...] +jwt_config: + enabled: true + secret: "your client secret" # TO BE FILLED (same as `client_secret` above) + algorithm: "RS256" + # (...other fields) ``` ### Dex @@ -624,6 +632,32 @@ oidc_providers: Note that the fields `client_id` and `client_secret` are taken from the CURL response above. +### Pocket ID + +[Pocket ID][pocket-id] is a simple OIDC provider that allows users to authenticate with their passkeys. +1. Go to `OIDC Clients` +2. Click on `Add OIDC Client` +3. Add a name, for example `Synapse` +4. Add `"https://auth.example.org/_synapse/client/oidc/callback` to `Callback URLs` # Replace `auth.example.org` with your domain +5. Click on `Save` +6. Note down your `Client ID` and `Client secret`, these will be used later + +Synapse config: + +```yaml +oidc_providers: + - idp_id: pocket_id + idp_name: Pocket ID + issuer: "https://auth.example.org/" # Replace with your domain + client_id: "your-client-id" # Replace with the "Client ID" you noted down before + client_secret: "your-client-secret" # Replace with the "Client secret" you noted down before + scopes: ["openid", "profile"] + user_mapping_provider: + config: + localpart_template: "{{ user.preferred_username }}" + display_name_template: "{{ user.name }}" +``` + ### Shibboleth with OIDC Plugin [Shibboleth](https://www.shibboleth.net/) is an open Standard IdP solution widely used by Universities. diff --git a/docs/postgres.md b/docs/postgres.md index 51670667e8..d51f54c722 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -100,6 +100,14 @@ database: keepalives_count: 3 ``` +## Postgresql major version upgrades + +Postgres uses separate directories for database locations between major versions (typically `/var/lib/postgresql//main`). + +Therefore, it is recommended to stop Synapse and other services (MAS, etc) before upgrading Postgres major versions. + +It is also strongly recommended to [back up](./usage/administration/backups.md#database) your database beforehand to ensure no data loss arising from a failed upgrade. + ## Backups Don't forget to [back up](./usage/administration/backups.md#database) your database! diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 7128af114e..f871a39939 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -5,10 +5,10 @@ It is recommended to put a reverse proxy such as [Apache](https://httpd.apache.org/docs/current/mod/mod_proxy_http.html), [Caddy](https://caddyserver.com/docs/quick-starts/reverse-proxy), [HAProxy](https://www.haproxy.org/) or -[relayd](https://man.openbsd.org/relayd.8) in front of Synapse. One advantage -of doing so is that it means that you can expose the default https port -(443) to Matrix clients without needing to run Synapse with root -privileges. +[relayd](https://man.openbsd.org/relayd.8) in front of Synapse. +This has the advantage of being able to expose the default HTTPS port (443) to Matrix +clients without requiring Synapse to bind to a privileged port (port numbers less than +1024), avoiding the need for `CAP_NET_BIND_SERVICE` or running as root. You should configure your reverse proxy to forward requests to `/_matrix` or `/_synapse/client` to Synapse, and have it set the `X-Forwarded-For` and @@ -74,7 +74,7 @@ server { proxy_pass http://localhost:8008; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $host; + proxy_set_header Host $host:$server_port; # Nginx by default only allows file uploads up to 1M in size # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml diff --git a/docs/setup/forward_proxy.md b/docs/setup/forward_proxy.md index f02c7b5fc5..eab8bb9951 100644 --- a/docs/setup/forward_proxy.md +++ b/docs/setup/forward_proxy.md @@ -7,8 +7,23 @@ proxy is supported, not SOCKS proxy or anything else. ## Configure -The `http_proxy`, `https_proxy`, `no_proxy` environment variables are used to -specify proxy settings. The environment variable is not case sensitive. +The proxy settings can be configured in the homeserver configuration file via +[`http_proxy`](../usage/configuration/config_documentation.md#http_proxy), +[`https_proxy`](../usage/configuration/config_documentation.md#https_proxy), and +[`no_proxy_hosts`](../usage/configuration/config_documentation.md#no_proxy_hosts). + +`homeserver.yaml` example: +```yaml +http_proxy: http://USERNAME:PASSWORD@10.0.1.1:8080/ +https_proxy: http://USERNAME:PASSWORD@proxy.example.com:8080/ +no_proxy_hosts: + - master.hostname.example.com + - 10.1.0.0/16 + - 172.30.0.0/16 +``` + +The proxy settings can also be configured via the `http_proxy`, `https_proxy`, +`no_proxy` environment variables. The environment variable is not case sensitive. - `http_proxy`: Proxy server to use for HTTP requests. - `https_proxy`: Proxy server to use for HTTPS requests. - `no_proxy`: Comma-separated list of hosts, IP addresses, or IP ranges in CIDR @@ -44,7 +59,7 @@ The proxy will be **used** for: - phone-home stats - recaptcha validation - CAS auth validation -- OpenID Connect +- OpenID Connect (OIDC) - Outbound federation - Federation (checking public key revocation) - Fetching public keys of other servers @@ -53,7 +68,7 @@ The proxy will be **used** for: It will **not be used** for: - Application Services -- Identity servers +- Matrix Identity servers - In worker configurations - connections between workers - connections from workers to Redis diff --git a/docs/setup/installation.md b/docs/setup/installation.md index bfeacab375..0840f532b0 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -157,7 +157,7 @@ sudo pip install py-bcrypt #### Alpine Linux -6543 maintains [Synapse packages for Alpine Linux](https://pkgs.alpinelinux.org/packages?name=synapse&branch=edge) in the community repository. Install with: +Jahway603 maintains [Synapse packages for Alpine Linux](https://pkgs.alpinelinux.org/packages?name=synapse&branch=edge) in the community repository. Install with: ```sh sudo apk add synapse @@ -286,7 +286,7 @@ Installing prerequisites on Ubuntu or Debian: ```sh sudo apt install build-essential python3-dev libffi-dev \ python3-pip python3-setuptools sqlite3 \ - libssl-dev virtualenv libjpeg-dev libxslt1-dev libicu-dev + libssl-dev virtualenv libjpeg-dev libxslt1-dev ``` ##### ArchLinux @@ -295,7 +295,7 @@ Installing prerequisites on ArchLinux: ```sh sudo pacman -S base-devel python python-pip \ - python-setuptools python-virtualenv sqlite3 icu + python-setuptools python-virtualenv sqlite3 ``` ##### CentOS/Fedora @@ -305,34 +305,22 @@ Installing prerequisites on CentOS or Fedora Linux: ```sh sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \ libwebp-devel libxml2-devel libxslt-devel libpq-devel \ - python3-virtualenv libffi-devel openssl-devel python3-devel \ - libicu-devel + python3-virtualenv libffi-devel openssl-devel python3-devel sudo dnf group install "Development Tools" ``` -##### Red Hat Enterprise Linux / Rocky Linux +##### Red Hat Enterprise Linux / Rocky Linux / Oracle Linux -*Note: The term "RHEL" below refers to both Red Hat Enterprise Linux and Rocky Linux. The distributions are 1:1 binary compatible.* +*Note: The term "RHEL" below refers to Red Hat Enterprise Linux, Oracle Linux and Rocky Linux. The distributions are 1:1 binary compatible.* It's recommended to use the latest Python versions. -RHEL 8 in particular ships with Python 3.6 by default which is EOL and therefore no longer supported by Synapse. RHEL 9 ship with Python 3.9 which is still supported by the Python core team as of this writing. However, newer Python versions provide significant performance improvements and they're available in official distributions' repositories. Therefore it's recommended to use them. +RHEL 8 in particular ships with Python 3.6 by default which is EOL and therefore no longer supported by Synapse. RHEL 9 ships with Python 3.9 which is still supported by the Python core team as of this writing. However, newer Python versions provide significant performance improvements and they're available in official distributions' repositories. Therefore it's recommended to use them. Python 3.11 and 3.12 are available for both RHEL 8 and 9. These commands should be run as root user. -RHEL 8 -```bash -# Enable PowerTools repository -dnf config-manager --set-enabled powertools -``` -RHEL 9 -```bash -# Enable CodeReady Linux Builder repository -crb enable -``` - Install new version of Python. You only need one of these: ```bash # Python 3.11 @@ -344,7 +332,7 @@ dnf install python3.12 python3.12-devel ``` Finally, install common prerequisites ```bash -dnf install libicu libicu-devel libpq5 libpq5-devel lz4 pkgconf +dnf install libpq5 libpq5-devel lz4 pkgconf dnf group install "Development Tools" ``` ###### Using venv module instead of virtualenv command @@ -376,20 +364,6 @@ xcode-select --install Some extra dependencies may be needed. You can use Homebrew (https://brew.sh) for them. -You may need to install icu, and make the icu binaries and libraries accessible. -Please follow [the official instructions of PyICU](https://pypi.org/project/PyICU/) to do so. - -If you're struggling to get icu discovered, and see: -``` - RuntimeError: - Please install pkg-config on your system or set the ICU_VERSION environment - variable to the version of ICU you have installed. -``` -despite it being installed and having your `PATH` updated, you can omit this dependency by -not specifying `--extras all` to `poetry`. If using postgres, you can install Synapse via -`poetry install --extras saml2 --extras oidc --extras postgres --extras opentracing --extras redis --extras sentry`. -ICU is not a hard dependency on getting a working installation. - On ARM-based Macs you may also need to install libjpeg and libpq: ```sh brew install jpeg libpq @@ -411,8 +385,7 @@ Installing prerequisites on openSUSE: ```sh sudo zypper in -t pattern devel_basis sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \ - python-devel libffi-devel libopenssl-devel libjpeg62-devel \ - libicu-devel + python-devel libffi-devel libopenssl-devel libjpeg62-devel ``` ##### OpenBSD diff --git a/docs/setup/turn/coturn.md b/docs/setup/turn/coturn.md index e5fbfa53f2..ae01a943e3 100644 --- a/docs/setup/turn/coturn.md +++ b/docs/setup/turn/coturn.md @@ -88,7 +88,8 @@ This will install and start a systemd service called `coturn`. denied-peer-ip=172.16.0.0-172.31.255.255 # recommended additional local peers to block, to mitigate external access to internal services. - # https://www.rtcsec.com/article/slack-webrtc-turn-compromise-and-bug-bounty/#how-to-fix-an-open-turn-relay-to-address-this-vulnerability + # https://www.enablesecurity.com/blog/slack-webrtc-turn-compromise-and-bug-bounty/#how-to-fix-an-open-turn-relay-to-address-this-vulnerability + # https://www.enablesecurity.com/blog/cve-2020-26262-bypass-of-coturns-access-control-protection/#further-concerns-what-else no-multicast-peers denied-peer-ip=0.0.0.0-0.255.255.255 denied-peer-ip=100.64.0.0-100.127.255.255 @@ -101,6 +102,14 @@ This will install and start a systemd service called `coturn`. denied-peer-ip=198.51.100.0-198.51.100.255 denied-peer-ip=203.0.113.0-203.0.113.255 denied-peer-ip=240.0.0.0-255.255.255.255 + denied-peer-ip=::1 + denied-peer-ip=64:ff9b::-64:ff9b::ffff:ffff + denied-peer-ip=::ffff:0.0.0.0-::ffff:255.255.255.255 + denied-peer-ip=100::-100::ffff:ffff:ffff:ffff + denied-peer-ip=2001::-2001:1ff:ffff:ffff:ffff:ffff:ffff:ffff + denied-peer-ip=2002::-2002:ffff:ffff:ffff:ffff:ffff:ffff:ffff + denied-peer-ip=fc00::-fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff + denied-peer-ip=fe80::-febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff # special case the turn server itself so that client->TURN->TURN->client flows work # this should be one of the turn server's listening IPs diff --git a/docs/spam_checker.md b/docs/spam_checker.md index 1b6d814937..ead0f03595 100644 --- a/docs/spam_checker.md +++ b/docs/spam_checker.md @@ -63,7 +63,7 @@ class ExampleSpamChecker: async def user_may_invite(self, inviter_userid, invitee_userid, room_id): return True # allow all invites - async def user_may_create_room(self, userid): + async def user_may_create_room(self, userid, room_config): return True # allow all room creations async def user_may_create_room_alias(self, userid, room_alias): @@ -72,8 +72,8 @@ class ExampleSpamChecker: async def user_may_publish_room(self, userid, room_id): return True # allow publishing of all rooms - async def check_username_for_spam(self, user_profile): - return False # allow all usernames + async def check_username_for_spam(self, user_profile, requester_id): + return False # allow all usernames regardless of requester async def check_registration_for_spam( self, diff --git a/docs/sso_mapping_providers.md b/docs/sso_mapping_providers.md index d6c4e860ae..4d33c8da75 100644 --- a/docs/sso_mapping_providers.md +++ b/docs/sso_mapping_providers.md @@ -10,7 +10,7 @@ As an example, a SSO service may return the email address to turn that into a displayname when creating a Matrix user for this individual. It may choose `John Smith`, or `Smith, John [Example.com]` or any number of variations. As each Synapse configuration may want something different, this is -where SAML mapping providers come into play. +where SSO mapping providers come into play. SSO mapping providers are currently supported for OpenID and SAML SSO configurations. Please see the details below for how to implement your own. diff --git a/docs/structured_logging.md b/docs/structured_logging.md index 002565b223..761d6466dd 100644 --- a/docs/structured_logging.md +++ b/docs/structured_logging.md @@ -35,7 +35,7 @@ handlers: loggers: synapse: level: INFO - handlers: [remote] + handlers: [file] synapse.storage.SQL: level: WARNING ``` diff --git a/docs/upgrade.md b/docs/upgrade.md index 45e63b0c5d..082d204b58 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,148 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.136.0 + +## Deprecate `run_as_background_process` exported as part of the module API interface in favor of `ModuleApi.run_as_background_process` + +The `run_as_background_process` function is now a method of the `ModuleApi` class. If +you were using the function directly from the module API, it will continue to work fine +but the background process metrics will not include an accurate `server_name` label. +This kind of metric labeling isn't relevant for many use cases and is used to +differentiate Synapse instances running in the same Python process (relevant to Synapse +Pro: Small Hosts). We recommend updating your usage to use the new +`ModuleApi.run_as_background_process` method to stay on top of future changes. + +
+Example run_as_background_process upgrade + +Before: +```python +class MyModule: + def __init__(self, module_api: ModuleApi) -> None: + run_as_background_process(__name__ + ":setup_database", self.setup_database) +``` + +After: +```python +class MyModule: + def __init__(self, module_api: ModuleApi) -> None: + module_api.run_as_background_process(__name__ + ":setup_database", self.setup_database) +``` + +
+ +## Metric labels have changed on `synapse_federation_last_received_pdu_time` and `synapse_federation_last_sent_pdu_time` + +Previously, the `synapse_federation_last_received_pdu_time` and +`synapse_federation_last_sent_pdu_time` metrics both used the `server_name` label to +differentiate between different servers that we send and receive events from. + +Since we're now using the `server_name` label to differentiate between different Synapse +homeserver instances running in the same process, these metrics have been changed as follows: + + - `synapse_federation_last_received_pdu_time` now uses the `origin_server_name` label + - `synapse_federation_last_sent_pdu_time` now uses the `destination_server_name` label + +The Grafana dashboard JSON in `contrib/grafana/synapse.json` has been updated to reflect +this change but you will need to manually update your own existing Grafana dashboards +using these metrics. + +## Stable integration with Matrix Authentication Service + +Support for [Matrix Authentication Service (MAS)](https://github.com/element-hq/matrix-authentication-service) is now stable, with a simplified configuration. +This stable integration requires MAS 0.20.0 or later. + +The existing `experimental_features.msc3861` configuration option is now deprecated and will be removed in Synapse v1.137.0. + +Synapse deployments already using MAS should now use the new configuration options: + +```yaml +matrix_authentication_service: + # Enable the MAS integration + enabled: true + # The base URL where Synapse will contact MAS + endpoint: http://localhost:8080 + # The shared secret used to authenticate MAS requests, must be the same as `matrix.secret` in the MAS configuration + # See https://element-hq.github.io/matrix-authentication-service/reference/configuration.html#matrix + secret: "asecurerandomsecretstring" +``` + +They must remove the `experimental_features.msc3861` configuration option from their configuration. + +They can also remove the client previously used by Synapse [in the MAS configuration](https://element-hq.github.io/matrix-authentication-service/reference/configuration.html#clients) as it is no longer in use. + +# Upgrading to v1.135.0 + +## `on_user_registration` module API callback may now run on any worker + +Previously, the `on_user_registration` callback would only run on the main +process. Modules relying on this callback must assume that they may now be +called from any worker, not just the main process. + +# Upgrading to v1.134.0 + +## ICU bundled with Synapse + +Synapse now uses the Rust `icu` library for improved user search. Installing the +native ICU library on your system is no longer required. + +# Upgrading to v1.130.0 + +## Documented endpoint which can be delegated to a federation worker + +The endpoint `^/_matrix/federation/v1/version$` can be delegated to a federation +worker. This is not new behaviour, but had not been documented yet. The +[list of delegatable endpoints](workers.md#synapseappgeneric_worker) has +been updated to include it. Make sure to check your reverse proxy rules if you +are using workers. + +# Upgrading to v1.126.0 + +## Room list publication rules change + +The default [`room_list_publication_rules`] setting was changed to disallow +anyone (except server admins) from publishing to the room list by default. + +This is in line with Synapse policy of locking down features by default that can +be abused without moderation. + +To keep the previous behavior of allowing publication by default, add the +following to the config: + +```yaml +room_list_publication_rules: + - "action": "allow" +``` + +[`room_list_publication_rules`]: usage/configuration/config_documentation.md#room_list_publication_rules + +## Change of signing key expiry date for the Debian/Ubuntu package repository + +Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, +please be aware that we have recently updated the expiry date on the repository's GPG signing key, +but this change must be imported into your keyring. + +If you have the `matrix-org-archive-keyring` package installed and it updates before the current key expires, this should +happen automatically. + +Otherwise, if you see an error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you +will need to get a fresh copy of the keys. You can do so with: + +```sh +sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg +``` + +The old version of the key will expire on `2025-03-15`. + +# Upgrading to v1.122.0 + +## Dropping support for PostgreSQL 11 and 12 + +In line with our [deprecation policy](deprecation_policy.md), we've dropped +support for PostgreSQL 11 and 12, as they are no longer supported upstream. +This release of Synapse requires PostgreSQL 13+. + # Upgrading to v1.120.0 ## Removal of experimental MSC3886 feature diff --git a/docs/usage/administration/admin_faq.md b/docs/usage/administration/admin_faq.md index 0dce3d3e37..20f8c6a157 100644 --- a/docs/usage/administration/admin_faq.md +++ b/docs/usage/administration/admin_faq.md @@ -160,7 +160,7 @@ Using the following curl command: ```console curl -H 'Authorization: Bearer ' -X DELETE https://matrix.org/_matrix/client/r0/directory/room/ ``` -`` - can be obtained in riot by looking in the riot settings, down the bottom is: +`` - can be obtained in element by looking in All settings, clicking Help & About and down the bottom is: Access Token:\ `` - the room alias, eg. #my_room:matrix.org this possibly needs to be URL encoded also, for example %23my_room%3Amatrix.org @@ -255,7 +255,7 @@ line to `/etc/default/matrix-synapse`: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -*Note*: You may need to set `PYTHONMALLOC=malloc` to ensure that `jemalloc` can accurately calculate memory usage. By default, Python uses its internal small-object allocator, which may interfere with jemalloc's ability to track memory consumption correctly. This could prevent the [cache_autotuning](../configuration/config_documentation.md#caches-and-associated-values) feature from functioning as expected, as the Python allocator may not reach the memory threshold set by `max_cache_memory_usage`, thus not triggering the cache eviction process. +*Note*: You may need to set `PYTHONMALLOC=malloc` to ensure that `jemalloc` can accurately calculate memory usage. By default, Python uses its internal small-object allocator, which may interfere with jemalloc's ability to track memory consumption correctly. This could prevent the [cache_autotuning](../configuration/config_documentation.md#caches) feature from functioning as expected, as the Python allocator may not reach the memory threshold set by `max_cache_memory_usage`, thus not triggering the cache eviction process. This made a significant difference on Python 2.7 - it's unclear how much of an improvement it provides on Python 3.x. diff --git a/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md b/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md index 4c0dbb5acd..a8a717e2a2 100644 --- a/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md +++ b/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md @@ -30,7 +30,7 @@ The following statistics are sent to the configured reporting endpoint: | `python_version` | string | The Python version number in use (e.g "3.7.1"). Taken from `sys.version_info`. | | `total_users` | int | The number of registered users on the homeserver. | | `total_nonbridged_users` | int | The number of users, excluding those created by an Application Service. | -| `daily_user_type_native` | int | The number of native users created in the last 24 hours. | +| `daily_user_type_native` | int | The number of native, non-guest users created in the last 24 hours. | | `daily_user_type_guest` | int | The number of guest users created in the last 24 hours. | | `daily_user_type_bridged` | int | The number of users created by Application Services in the last 24 hours. | | `total_room_count` | int | The total number of rooms present on the homeserver. | @@ -50,8 +50,8 @@ The following statistics are sent to the configured reporting endpoint: | `cache_factor` | int | The configured [`global factor`](../../configuration/config_documentation.md#caching) value for caching. | | `event_cache_size` | int | The configured [`event_cache_size`](../../configuration/config_documentation.md#caching) value for caching. | | `database_engine` | string | The database engine that is in use. Either "psycopg2" meaning PostgreSQL is in use, or "sqlite3" for SQLite3. | -| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system. | -| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc. | +| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system. | +| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc. | [^1]: Native matrix users and guests are always counted. If the diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 7a48d76bbb..3c401d569b 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1,3 +1,5 @@ + + # Configuring Synapse This is intended as a guide to the Synapse configuration. The behavior of a Synapse instance can be modified @@ -90,32 +92,32 @@ apply if you want your config file to be read properly. A few helpful things to the sub-options, if any, are identified and listed in the body of the section. In addition, each setting has an example of its usage, with the proper indentation shown. - +--- ## Modules Server admins can expand Synapse's functionality with external modules. -See [here](../../modules/index.md) for more -documentation on how to configure or create custom modules for Synapse. - +See [here](../../modules/index.md) for more documentation on how to configure or create custom modules for Synapse. --- ### `modules` -Use the `module` sub-option to add modules under this option to extend functionality. -The `module` setting then has a sub-option, `config`, which can be used to define some configuration -for the `module`. +*(array)* Use the `module` sub-option to add modules under this option to extend functionality. The `module` setting then has a sub-option, `config`, which can be used to define some configuration for the `module`. Defaults to `[]`. -Defaults to none. +Options for each entry include: + +* `module` (string): Path to the Python class of the module. + +* `config` (object): Configuration options for the module. Example configuration: ```yaml modules: - - module: my_super_module.MySuperClass - config: - do_thing: true - - module: my_other_super_module.SomeClass - config: {} +- module: my_super_module.MySuperClass + config: + do_thing: true +- module: my_other_super_module.SomeClass + config: {} ``` --- ## Server @@ -125,46 +127,83 @@ Define your homeserver name and other base options. --- ### `server_name` -This sets the public-facing domain of the server. +*(string)* This sets the public-facing domain of the server. -The `server_name` name will appear at the end of usernames and room addresses -created on your server. For example if the `server_name` was example.com, -usernames on your server would be in the format `@user:example.com` +The `server_name` name will appear at the end of usernames and room addresses created on your server. For example if the `server_name` was example.com, usernames on your server would be in the format `@user:example.com`. -In most cases you should avoid using a matrix specific subdomain such as -matrix.example.com or synapse.example.com as the `server_name` for the same -reasons you wouldn't use user@email.example.com as your email address. -See [here](../../delegate.md) -for information on how to host Synapse on a subdomain while preserving -a clean `server_name`. +In most cases you should avoid using a matrix specific subdomain such as matrix.example.com or synapse.example.com as the `server_name` for the same reasons you wouldn't use user@email.example.com as your email address. See [here](../../delegate.md) for information on how to host Synapse on a subdomain while preserving a clean `server_name`. -The `server_name` cannot be changed later so it is important to -configure this correctly before you start Synapse. It should be all -lowercase and may contain an explicit port. +The `server_name` cannot be changed later so it is important to configure this correctly before you start Synapse. It should be all lowercase and may contain an explicit port. There is no default for this option. -Example configuration #1: +Example configurations: ```yaml server_name: matrix.org ``` -Example configuration #2: + ```yaml server_name: localhost:8080 ``` --- ### `pid_file` -When running Synapse as a daemon, the file to store the pid in. Defaults to none. +*(string|null)* When running Synapse as a daemon, the file to store the pid in. Defaults to `null`. Example configuration: ```yaml pid_file: DATADIR/homeserver.pid ``` --- +### `daemonize` + +*(boolean)* Specifies whether Synapse should be started as a daemon process. If Synapse is being managed by [systemd](../../systemd-with-workers/), this option must be omitted or set to `false`. + +This can also be set by the `--daemonize` (`-D`) argument when starting Synapse. + +See `worker_daemonize` for more information on daemonizing workers. + +Defaults to `false`. + +Example configuration: +```yaml +daemonize: true +``` +--- +### `print_pidfile` + +*(boolean)* Print the path to the pidfile just before daemonizing. + +This can also be set by the `--print-pidfile` argument when starting Synapse. + +Defaults to `false`. + +Example configuration: +```yaml +print_pidfile: true +``` +--- +### `user_agent_suffix` + +*(string|null)* A suffix that is appended to the Synapse user-agent (ex. `Synapse/v1.123.0`). Defaults to `null`. + +Example configuration: +```yaml +user_agent_suffix: ' (I''m a teapot; Linux x86_64)' +``` +--- +### `use_frozen_dicts` + +*(boolean)* Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. signature dicts. However, freezing a dict is expensive. Defaults to `false`. + +Example configuration: +```yaml +use_frozen_dicts: true +``` +--- ### `web_client_location` -The absolute URL to the web client which `/` will redirect to. Defaults to none. +*(string|null)* The absolute URL to the web client which `/` will redirect to. Defaults to `null`. Example configuration: ```yaml @@ -173,14 +212,11 @@ web_client_location: https://riot.example.com/ --- ### `public_baseurl` -The public-facing base URL that clients use to access this Homeserver (not -including _matrix/...). This is the same URL a user might enter into the -'Custom Homeserver URL' field on their client. If you use Synapse with a -reverse proxy, this should be the URL to reach Synapse via the proxy. -Otherwise, it should be the URL to reach Synapse's client HTTP listener (see -['listeners'](#listeners) below). +*(string|null)* The public-facing base URL that clients use to access this Homeserver (not including _matrix/...). This is the same URL a user might enter into the "Custom Homeserver URL" field on their client. If you use Synapse with a reverse proxy, this should be the URL to reach Synapse via the proxy. Otherwise, it should be the URL to reach Synapse's client HTTP listener (see [`listeners`](#listeners) below). -Defaults to `https:///`. +If unset or null, `https:///` is used. + +Defaults to `null`. Example configuration: ```yaml @@ -189,46 +225,39 @@ public_baseurl: https://example.com/ --- ### `serve_server_wellknown` -By default, other servers will try to reach our server on port 8448, which can -be inconvenient in some environments. +*(boolean)* By default, other servers will try to reach our server on port 8448, which can be inconvenient in some environments. -Provided `https:///` on port 443 is routed to Synapse, this -option configures Synapse to serve a file at `https:///.well-known/matrix/server`. -This will tell other servers to send traffic to port 443 instead. +Provided `https:///` on port 443 is routed to Synapse, this option configures Synapse to serve a file at `https:///.well-known/matrix/server`. This will tell other servers to send traffic to port 443 instead. This option currently defaults to false. -See [Delegation of incoming federation traffic](../../delegate.md) for more -information. +See [Delegation of incoming federation traffic](../../delegate.md) for more information. + +Defaults to `false`. Example configuration: ```yaml serve_server_wellknown: true ``` --- -### `extra_well_known_client_content ` +### `extra_well_known_client_content` -This option allows server runners to add arbitrary key-value pairs to the [client-facing `.well-known` response](https://spec.matrix.org/latest/client-server-api/#well-known-uri). -Note that the `public_baseurl` config option must be provided for Synapse to serve a response to `/.well-known/matrix/client` at all. +*(object)* This option allows server runners to add arbitrary key-value pairs to the [client-facing `.well-known` response](https://spec.matrix.org/latest/client-server-api/#well-known-uri). Note that the `public_baseurl` config option must be provided for Synapse to serve a response to `/.well-known/matrix/client` at all. -If this option is provided, it parses the given yaml to json and -serves it on `/.well-known/matrix/client` endpoint -alongside the standard properties. +If this option is provided, it parses the given yaml to json and serves it on `/.well-known/matrix/client` endpoint alongside the standard properties. *Added in Synapse 1.62.0.* Example configuration: ```yaml -extra_well_known_client_content : +extra_well_known_client_content: option1: value1 option2: value2 ``` --- ### `soft_file_limit` -Set the soft limit on the number of file descriptors synapse can use. -Zero is used to indicate synapse should set the soft limit to the hard limit. -Defaults to 0. +*(integer)* Set the soft limit on the number of file descriptors synapse can use. Zero is used to indicate synapse should set the soft limit to the hard limit. Defaults to `0`. Example configuration: ```yaml @@ -237,10 +266,19 @@ soft_file_limit: 3 --- ### `presence` -Presence tracking allows users to see the state (e.g online/offline) -of other local and remote users. Set the `enabled` sub-option to false to -disable presence tracking on this homeserver. Defaults to true. -This option replaces the previous top-level 'use_presence' option. +*(object)* Presence tracking allows users to see the state (e.g online/offline) of other local and remote users. This option replaces the previous top-level `use_presence` option. + +This setting has the following sub-options: + +* `enabled` (boolean|string): Set to false to disable presence tracking on this homeserver. + + Can also be set to a special value of "untracked" which ignores updates received via clients and federation, while still accepting updates from the [module API](../../modules/index.md). + + *The "untracked" option was added in Synapse 1.96.0.* + + Defaults to `true`. + +* `include_offline_users_on_sync` (boolean): When clients perform an initial or `full_state` sync, presence results for offline users are not included by default. Setting `include_offline_users_on_sync` to `true` will always include offline users in the results. Defaults to `false`. Example configuration: ```yaml @@ -248,23 +286,10 @@ presence: enabled: false include_offline_users_on_sync: false ``` - -`enabled` can also be set to a special value of "untracked" which ignores updates -received via clients and federation, while still accepting updates from the -[module API](../../modules/index.md). - -*The "untracked" option was added in Synapse 1.96.0.* - -When clients perform an initial or `full_state` sync, presence results for offline users are -not included by default. Setting `include_offline_users_on_sync` to `true` will always include -offline users in the results. Defaults to false. - --- ### `require_auth_for_profile_requests` -Whether to require authentication to retrieve profile data (avatars, display names) of other -users through the client API. Defaults to false. Note that profile data is also available -via the federation API, unless `allow_profile_lookup_over_federation` is set to false. +*(boolean)* Whether to require authentication to retrieve profile data (avatars, display names) of other users through the client API. Note that profile data is also available via the federation API, unless `allow_profile_lookup_over_federation` is set to false. Defaults to `false`. Example configuration: ```yaml @@ -273,10 +298,7 @@ require_auth_for_profile_requests: true --- ### `limit_profile_requests_to_users_who_share_rooms` -Use this option to require a user to share a room with another user in order -to retrieve their profile information. Only checked on Client-Server -requests. Profile requests from other servers should be checked by the -requesting server. Defaults to false. +*(boolean)* Use this option to require a user to share a room with another user in order to retrieve their profile information. Only checked on Client-Server requests. Profile requests from other servers should be checked by the requesting server. Defaults to `false`. Example configuration: ```yaml @@ -285,11 +307,7 @@ limit_profile_requests_to_users_who_share_rooms: true --- ### `include_profile_data_on_invite` -Use this option to prevent a user's profile data from being retrieved and -displayed in a room until they have joined it. By default, a user's -profile data is included in an invite event, regardless of the values -of the above two settings, and whether or not the users share a server. -Defaults to true. +*(boolean)* Use this option to prevent a user's profile data from being retrieved and displayed in a room until they have joined it. By default, a user's profile data is included in an invite event, regardless of the values of the above two settings, and whether or not the users share a server. Defaults to `true`. Example configuration: ```yaml @@ -298,9 +316,7 @@ include_profile_data_on_invite: false --- ### `allow_public_rooms_without_auth` -If set to true, removes the need for authentication to access the server's -public rooms directory through the client API, meaning that anyone can -query the room directory. Defaults to false. +*(boolean)* If set to true, removes the need for authentication to access the server's public rooms directory through the client API, meaning that anyone can query the room directory. Defaults to `false`. Example configuration: ```yaml @@ -309,8 +325,7 @@ allow_public_rooms_without_auth: true --- ### `allow_public_rooms_over_federation` -If set to true, allows any other homeserver to fetch the server's public -rooms directory via federation. Defaults to false. +*(boolean)* If set to true, allows any other homeserver to fetch the server's public rooms directory via federation. Defaults to `false`. Example configuration: ```yaml @@ -319,50 +334,56 @@ allow_public_rooms_over_federation: true --- ### `default_room_version` -The default room version for newly created rooms on this server. +*(string)* The default room version for newly created rooms on this server. Known room versions are listed [here](https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions) -For example, for room version 1, `default_room_version` should be set -to "1". - -Currently defaults to ["10"](https://spec.matrix.org/v1.5/rooms/v10/). +For example, for room version 1, `default_room_version` should be set to "1". _Changed in Synapse 1.76:_ the default version room version was increased from [9](https://spec.matrix.org/v1.5/rooms/v9/) to [10](https://spec.matrix.org/v1.5/rooms/v10/). +Defaults to `"10"`. + Example configuration: ```yaml -default_room_version: "8" +default_room_version: '8' ``` --- ### `gc_thresholds` -The garbage collection threshold parameters to pass to `gc.set_threshold`, if defined. -Defaults to none. +*(array|null)* The garbage collection threshold parameters to pass to `gc.set_threshold`, if defined. Defaults to `null`. Example configuration: ```yaml -gc_thresholds: [700, 10, 10] +gc_thresholds: +- 700 +- 10 +- 10 ``` --- ### `gc_min_interval` -The minimum time in seconds between each GC for a generation, regardless of -the GC thresholds. This ensures that we don't do GC too frequently. A value of `[1s, 10s, 30s]` -indicates that a second must pass between consecutive generation 0 GCs, etc. +*(array)* The minimum time in seconds between each GC for a generation, regardless of the GC thresholds. This ensures that we don't do GC too frequently. A value of `[1s, 10s, 30s]` indicates that a second must pass between consecutive generation 0 GCs, etc. -Defaults to `[1s, 10s, 30s]`. +Default configuration: +```yaml +gc_min_interval: +- 1s +- 10s +- 30s +``` Example configuration: ```yaml -gc_min_interval: [0.5s, 30s, 1m] +gc_min_interval: +- 0.5s +- 30s +- 1m ``` --- ### `filter_timeline_limit` -Set the limit on the returned events in the timeline in the get -and sync operations. Defaults to 100. A value of -1 means no upper limit. - +*(integer)* Set the limit on the returned events in the timeline in the get and sync operations. A value of -1 means no upper limit. Defaults to `100`. Example configuration: ```yaml @@ -371,8 +392,7 @@ filter_timeline_limit: 5000 --- ### `block_non_admin_invites` -Whether room invites to users on this server should be blocked -(except those sent by local server admins). Defaults to false. +*(boolean)* Whether room invites to users on this server should be blocked (except those sent by local server admins). Defaults to `false`. Example configuration: ```yaml @@ -381,8 +401,7 @@ block_non_admin_invites: true --- ### `enable_search` -If set to false, new messages will not be indexed for searching and users -will receive errors when searching for messages. Defaults to true. +*(boolean)* If set to false, new messages will not be indexed for searching and users will receive errors when searching for messages. Defaults to `true`. Example configuration: ```yaml @@ -391,126 +410,61 @@ enable_search: false --- ### `ip_range_blacklist` -This option prevents outgoing requests from being sent to the specified blacklisted IP address -CIDR ranges. If this option is not specified then it defaults to private IP -address ranges (see the example below). +*(array)* This option prevents outgoing requests from being sent to the specified blacklisted IP address CIDR ranges. If this option is not specified then it defaults to private IP address ranges (see the example below). -The blacklist applies to the outbound requests for federation, identity servers, -push servers, and for checking key validity for third-party invite events. +The blacklist applies to the outbound requests for federation, identity servers, push servers, and for checking key validity for third-party invite events. -(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly -listed here, since they correspond to unroutable addresses.) +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly listed here, since they correspond to unroutable addresses.) This option replaces `federation_ip_range_blacklist` in Synapse v1.25.0. Note: The value is ignored when an HTTP proxy is in use. -Example configuration: +Default configuration: ```yaml ip_range_blacklist: - - '127.0.0.0/8' - - '10.0.0.0/8' - - '172.16.0.0/12' - - '192.168.0.0/16' - - '100.64.0.0/10' - - '192.0.0.0/24' - - '169.254.0.0/16' - - '192.88.99.0/24' - - '198.18.0.0/15' - - '192.0.2.0/24' - - '198.51.100.0/24' - - '203.0.113.0/24' - - '224.0.0.0/4' - - '::1/128' - - 'fe80::/10' - - 'fc00::/7' - - '2001:db8::/32' - - 'ff00::/8' - - 'fec0::/10' +- 127.0.0.0/8 +- 10.0.0.0/8 +- 172.16.0.0/12 +- 192.168.0.0/16 +- 100.64.0.0/10 +- 192.0.0.0/24 +- 169.254.0.0/16 +- 192.88.99.0/24 +- 198.18.0.0/15 +- 192.0.2.0/24 +- 198.51.100.0/24 +- 203.0.113.0/24 +- 224.0.0.0/4 +- ::1/128 +- fe80::/10 +- fc00::/7 +- 2001:db8::/32 +- ff00::/8 +- fec0::/10 ``` --- ### `ip_range_whitelist` -List of IP address CIDR ranges that should be allowed for federation, -identity servers, push servers, and for checking key validity for -third-party invite events. This is useful for specifying exceptions to -wide-ranging blacklisted target IP ranges - e.g. for communication with -a push server only visible in your network. +*(array)* List of IP address CIDR ranges that should be allowed for federation, identity servers, push servers, and for checking key validity for third-party invite events. This is useful for specifying exceptions to wide-ranging blacklisted target IP ranges – e.g. for communication with a push server only visible in your network. -This whitelist overrides `ip_range_blacklist` and defaults to an empty -list. +This whitelist overrides `ip_range_blacklist`. + +Defaults to `[]`. Example configuration: ```yaml ip_range_whitelist: - - '192.168.1.1' +- 192.168.1.1 ``` --- ### `listeners` -List of ports that Synapse should listen on, their purpose and their -configuration. - -Sub-options for each listener include: - -* `port`: the TCP port to bind to. - -* `tag`: An alias for the port in the logger name. If set the tag is logged instead -of the port. Default to `None`, is optional and only valid for listener with `type: http`. -See the docs [request log format](../administration/request_log.md). - -* `bind_addresses`: a list of local addresses to listen on. The default is - 'all local interfaces'. - -* `type`: the type of listener. Normally `http`, but other valid options are: - - * `manhole`: (see the docs [here](../../manhole.md)), - - * `metrics`: (see the docs [here](../../metrics-howto.md)), - -* `tls`: set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path. - -* `x_forwarded`: Only valid for an 'http' listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is - behind a [reverse-proxy](../../reverse_proxy.md). - -* `request_id_header`: The header extracted from each incoming request that is - used as the basis for the request ID. The request ID is used in - [logs](../administration/request_log.md#request-log-format) and tracing to - correlate and match up requests. When unset, Synapse will automatically - generate sequential request IDs. This option is useful when Synapse is behind - a [reverse-proxy](../../reverse_proxy.md). - - _Added in Synapse 1.68.0._ - -* `resources`: Only valid for an 'http' listener. A list of resources to host - on this port. Sub-options for each resource are: - - * `names`: a list of names of HTTP resources. See below for a list of valid resource names. - - * `compress`: set to true to enable gzip compression on HTTP bodies for this resource. This is currently only supported with the - `client`, `consent`, `metrics` and `federation` resources. - -* `additional_resources`: Only valid for an 'http' listener. A map of - additional endpoints which should be loaded via dynamic modules. - -Unix socket support (_Added in Synapse 1.89.0_): -* `path`: A path and filename for a Unix socket. Make sure it is located in a - directory with read and write permissions, and that it already exists (the directory - will not be created). Defaults to `None`. - * **Note**: The use of both `path` and `port` options for the same `listener` is not - compatible. - * The `x_forwarded` option defaults to true when using Unix sockets and can be omitted. - * Other options that would not make sense to use with a UNIX socket, such as - `bind_addresses` and `tls` will be ignored and can be removed. -* `mode`: The file permissions to set on the UNIX socket. Defaults to `666` -* **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`). - Also make sure that `metrics` is not included in `resources` -> `names` - +*(array)* List of ports that Synapse should listen on, their purpose and their configuration. Valid resource names are: -* `client`: the client-server API (/_matrix/client). Also implies `media` and `static`. - If configuring the main process, the Synapse Admin API (/_synapse/admin) is also implied. +* `client`: the client-server API (/_matrix/client). Also implies `media` and `static`. If configuring the main process, the Synapse Admin API (/_synapse/admin) is also implied. * `consent`: user consent forms (/_matrix/consent). See [here](../../consent_tracking.md) for more. @@ -526,85 +480,126 @@ Valid resource names are: * `replication`: the HTTP replication API (/_synapse/replication). See [here](../../workers.md). -* `static`: static resources under synapse/static (/_matrix/static). (Mostly useful for 'fallback authentication'.) +* `static`: static resources under synapse/static (/_matrix/static). (Mostly useful for "fallback authentication".) -* `health`: the [health check endpoint](../../reverse_proxy.md#health-check-endpoint). This endpoint - is by default active for all other resources and does not have to be activated separately. - This is only useful if you want to use the health endpoint explicitly on a dedicated port or - for [workers](../../workers.md) and containers without listener e.g. - [application services](../../workers.md#notifying-application-services). +* `health`: the [health check endpoint](../../reverse_proxy.md#health-check-endpoint). This endpoint is by default active for all other resources and does not have to be activated separately. This is only useful if you want to use the health endpoint explicitly on a dedicated port or for [workers](../../workers.md) and containers without listener e.g. [application services](../../workers.md#notifying-application-services). -Example configuration #1: +Defaults to `[]`. + +Options for each entry include: + +* `port` (integer): The TCP port to bind to. + +* `tag` (string|null): An alias for the port in the logger name. If set the tag is logged instead of the port. Default to `None`, is optional and only valid for listener with `type: http`. See the docs [request log format](../administration/request_log.md). + +* `bind_addresses` (array|null): A list of local addresses to listen on. The default is "all local interfaces". + +* `type` (string): The type of listener. Normally `http`, but other valid options are [`manhole`](../../manhole.md) and [`metrics`](../../metrics-howto.md). + +* `tls` (boolean): Set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path/tls_certificate_path. + +* `x_forwarded` (boolean): Only valid for an `http` listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is behind a [reverse-proxy](../../reverse_proxy.md). + +* `request_id_header` (string|null): The header extracted from each incoming request that is used as the basis for the request ID. The request ID is used in [logs](../administration/request_log.md#request-log-format) and tracing to correlate and match up requests. When unset, Synapse will automatically generate sequential request IDs. This option is useful when Synapse is behind a [reverse-proxy](../../reverse_proxy.md). + + _Added in Synapse 1.68.0._ + +* `resources` (array): Only valid for an `http` listener. A list of resources to host on this port. + + Options for each entry include: + + * `names` (array): A list of names of HTTP resources. See below for a list of valid resource names. + + * `compress` (boolean): Set to true to enable gzip compression on HTTP bodies for this resource. This is currently only supported with the `client`, `consent`, `metrics` and `federation` resources. + +* `additional_resources` (object): Only valid for an `http` listener. A map of additional endpoints which should be loaded via dynamic modules. + +* `path` (string): A path and filename for a Unix socket. Make sure it is located in a directory with read and write permissions, and that it already exists (the directory will not be created). Defaults to `None`. + * **Note**: The use of both `path` and `port` options for the same `listener` is not compatible. + * The `x_forwarded` option defaults to true when using Unix sockets and can be omitted. + * Other options that would not make sense to use with a UNIX socket, such as `bind_addresses` and `tls` will be ignored and can be removed. + + _Added in Synapse 1.89.0_: Unix socket support + +* `mode` (integer|null): The file permissions to set on the UNIX socket. Defaults to `666` if unset or null. + + **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`). Also make sure that `metrics` is not included in `resources` -> `names` + + _Added in Synapse 1.89.0_: Unix socket support + +Example configurations: ```yaml listeners: - # TLS-enabled listener: for when matrix traffic is sent directly to synapse. - # - # (Note that you will also need to give Synapse a TLS key and certificate: see the TLS section - # below.) - # - - port: 8448 - type: http - tls: true - resources: - - names: [client, federation] +- port: 8448 + type: http + tls: true + resources: + - names: + - client + - federation ``` -Example configuration #2: + ```yaml listeners: - # Insecure HTTP listener: for when matrix traffic passes through a reverse proxy - # that unwraps TLS. - # - # If you plan to use a reverse proxy, please see - # https://element-hq.github.io/synapse/latest/reverse_proxy.html. - # - - port: 8008 - tls: false - type: http - x_forwarded: true - bind_addresses: ['::1', '127.0.0.1'] - - resources: - - names: [client, federation] - compress: false - - # example additional_resources: - additional_resources: - "/_matrix/my/custom/endpoint": - module: my_module.CustomRequestHandler - config: {} - - # Turn on the twisted ssh manhole service on localhost on the given - # port. - - port: 9000 - bind_addresses: ['::1', '127.0.0.1'] - type: manhole +- port: 8008 + tls: false + type: http + x_forwarded: true + bind_addresses: + - ::1 + - 127.0.0.1 + resources: + - names: + - client + - federation + compress: false + additional_resources: + /_matrix/my/custom/endpoint: + module: my_module.CustomRequestHandler + config: {} +- port: 9000 + bind_addresses: + - ::1 + - 127.0.0.1 + type: manhole ``` -Example configuration #3: + ```yaml listeners: - # Unix socket listener: Ideal for Synapse deployments behind a reverse proxy, offering - # lightweight interprocess communication without TCP/IP overhead, avoid port - # conflicts, and providing enhanced security through system file permissions. - # - # Note that x_forwarded will default to true, when using a UNIX socket. Please see - # https://element-hq.github.io/synapse/latest/reverse_proxy.html. - # - - path: /run/synapse/main_public.sock - type: http - resources: - - names: [client, federation] +- path: /run/synapse/main_public.sock + type: http + resources: + - names: + - client + - federation ``` +--- +### `manhole` +*(integer|null)* Turn on the Twisted telnet manhole service on the given port. + +This can also be set by the `--manhole` argument when starting Synapse. + +Defaults to `null`. + +Example configuration: +```yaml +manhole: 1234 +``` --- ### `manhole_settings` -Connection settings for the manhole. You can find more information -on the manhole [here](../../manhole.md). Manhole sub-options include: -* `username` : the username for the manhole. This defaults to 'matrix'. -* `password`: The password for the manhole. This defaults to 'rabbithole'. -* `ssh_priv_key_path` and `ssh_pub_key_path`: The private and public SSH key pair used to encrypt the manhole traffic. - If these are left unset, then hardcoded and non-secret keys are used, - which could allow traffic to be intercepted if sent over a public network. +*(object)* Connection settings for the manhole. You can find more information on the manhole [here](../../manhole.md). + +This setting has the following sub-options: + +* `username` (string|null): The username for the manhole. This defaults to "matrix". + +* `password` (string|null): The password for the manhole. This defaults to "rabbithole". + +* `ssh_priv_key_path` (string|null): The private SSH key used to encrypt the manhole traffic. If left unset, then hardcoded and non-secret keys are used, which could allow traffic to be intercepted if sent over a public network. + +* `ssh_pub_key_path` (string|null): The public SSH key corresponsing to `ssh_priv_key_path`. If left unset, a hardcoded key is used. Example configuration: ```yaml @@ -615,17 +610,68 @@ manhole_settings: ssh_pub_key_path: CONFDIR/id_rsa.pub ``` --- +### `http_proxy` + +*(string|null)* Proxy server to use for HTTP requests. +For more details, see the [forward proxy documentation](../../setup/forward_proxy.md). There is no default for this option. + +Example configuration: +```yaml +http_proxy: http://USERNAME:PASSWORD@10.0.1.1:8080/ +``` +--- +### `https_proxy` + +*(string|null)* Proxy server to use for HTTPS requests. +For more details, see the [forward proxy documentation](../../setup/forward_proxy.md). There is no default for this option. + +Example configuration: +```yaml +https_proxy: http://USERNAME:PASSWORD@proxy.example.com:8080/ +``` +--- +### `no_proxy_hosts` + +*(array)* List of hosts, IP addresses, or IP ranges in CIDR format which should not use the proxy. Synapse will directly connect to these hosts. +For more details, see the [forward proxy documentation](../../setup/forward_proxy.md). There is no default for this option. + +Example configuration: +```yaml +no_proxy_hosts: +- master.hostname.example.com +- 10.1.0.0/16 +- 172.30.0.0/16 +``` +--- +### `matrix_authentication_service` + +*(object)* The `matrix_authentication_service` setting configures integration with [Matrix Authentication Service (MAS)](https://github.com/element-hq/matrix-authentication-service). + +This setting has the following sub-options: + +* `enabled` (boolean): Whether or not to enable the MAS integration. If this is set to `false`, Synapse will use its legacy internal authentication API. Defaults to `false`. + +* `endpoint` (string): The URL where Synapse can reach MAS. This *must* have the `discovery` and `oauth` resources mounted. Defaults to `"http://localhost:8080"`. + +* `secret` (string|null): A shared secret that will be used to authenticate requests from and to MAS. + +* `secret_path` (string|null): Alternative to `secret`, reading the shared secret from a file. The file should be a plain text file, containing only the secret. Synapse reads the secret from the given file once at startup. + +Example configuration: +```yaml +matrix_authentication_service: + enabled: true + secret: someverysecuresecret + endpoint: http://localhost:8080 +``` +--- ### `dummy_events_threshold` -Forward extremities can build up in a room due to networking delays between -homeservers. Once this happens in a large room, calculation of the state of -that room can become quite expensive. To mitigate this, once the number of -forward extremities reaches a given threshold, Synapse will send an -`org.matrix.dummy_event` event, which will reduce the forward extremities -in the room. +*(integer)* Forward extremities can build up in a room due to networking delays between homeservers. Once this happens in a large room, calculation of the state of that room can become quite expensive. To mitigate this, once the number of forward extremities reaches a given threshold, Synapse will send an `org.matrix.dummy_event` event, which will reduce the forward extremities in the room. This setting defines the threshold (i.e. number of forward extremities in the room) at which dummy events are sent. -The default value is 10. + +Defaults to `10`. Example configuration: ```yaml @@ -634,14 +680,13 @@ dummy_events_threshold: 5 --- ### `delete_stale_devices_after` -An optional duration. If set, Synapse will run a daily background task to log out and -delete any device that hasn't been accessed for more than the specified amount of time. +An optional duration. If set, Synapse will run a daily background task to log out and delete any device that hasn't been accessed for more than the specified amount of time. -Defaults to no duration, which means devices are never pruned. +A value of null means devices are never pruned. -**Note:** This task will always run on the main process, regardless of the value of -`run_background_tasks_on`. This is due to workers currently not having the ability to -delete devices. +**Note:** This task will always run on the main process, regardless of the value of `run_background_tasks_on`. This is due to workers currently not having the ability to delete devices. + +Defaults to `null`. Example configuration: ```yaml @@ -650,165 +695,187 @@ delete_stale_devices_after: 1y --- ### `email` -Configuration for sending emails from Synapse. +*(object)* Configuration for sending emails from Synapse. -Server admins can configure custom templates for email content. See -[here](../../templates.md) for more information. +Server admins can configure custom templates for email content. See [here](../../templates.md) for more information. This setting has the following sub-options: -* `smtp_host`: The hostname of the outgoing SMTP server to use. Defaults to 'localhost'. -* `smtp_port`: The port on the mail server for outgoing SMTP. Defaults to 465 if `force_tls` is true, else 25. + +* `smtp_host` (string): The hostname of the outgoing SMTP server to use. Defaults to `"localhost"`. + +* `smtp_port` (string|null): The port on the mail server for outgoing SMTP. If null or unset, 465 is used if `force_tls` is true, else 25. _Changed in Synapse 1.64.0:_ the default port is now aware of `force_tls`. -* `smtp_user` and `smtp_pass`: Username/password for authentication to the SMTP server. By default, no - authentication is attempted. -* `force_tls`: By default, Synapse connects over plain text and then optionally upgrades - to TLS via STARTTLS. If this option is set to true, TLS is used from the start (Implicit TLS), - and the option `require_transport_security` is ignored. - It is recommended to enable this if supported by your mail server. + + Defaults to `null`. + +* `smtp_user` (string|null): Username for authentication to the SMTP server. Defaults to `null`. + +* `smtp_pass` (string|null): Password for authentication to the SMTP server. Defaults to `null`. + +* `force_tls` (boolean): By default, Synapse connects over plain text and then optionally upgrades to TLS via STARTTLS. If this option is set to true, TLS is used from the start (Implicit TLS), and the option `require_transport_security` is ignored. It is recommended to enable this if supported by your mail server. _New in Synapse 1.64.0._ -* `require_transport_security`: Set to true to require TLS transport security for SMTP. - By default, Synapse will connect over plain text, and will then switch to - TLS via STARTTLS *if the SMTP server supports it*. If this option is set, - Synapse will refuse to connect unless the server supports STARTTLS. -* `enable_tls`: By default, if the server supports TLS, it will be used, and the server - must present a certificate that is valid for 'smtp_host'. If this option - is set to false, TLS will not be used. -* `notif_from`: defines the "From" address to use when sending emails. - It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name, - which is normally set in `app_name`, but may be overridden by the - Matrix client application. Note that the placeholder must be written '%(app)s', including the - trailing 's'. -* `app_name`: `app_name` defines the default value for '%(app)s' in `notif_from` and email - subjects. It defaults to 'Matrix'. -* `enable_notifs`: Set to true to allow users to receive e-mail notifications. If this is not set, - users can configure e-mail notifications but will not receive them. Disabled by default. -* `notif_for_new_users`: Set to false to disable automatic subscription to email - notifications for new users. Enabled by default. -* `notif_delay_before_mail`: The time to wait before emailing about a notification. - This gives the user a chance to view the message via push or an open client. - Defaults to 10 minutes. + + Defaults to `false`. + +* `require_transport_security` (boolean): Set to true to require TLS transport security for SMTP. By default, Synapse will connect over plain text, and will then switch to TLS via STARTTLS *if the SMTP server supports it*. If this option is set, Synapse will refuse to connect unless the server supports STARTTLS. Defaults to `false`. + +* `enable_tls` (boolean): By default, if the server supports TLS, it will be used, and the server must present a certificate that is valid for `tlsname`. If this option is set to false, TLS will not be used. Defaults to `true`. + +* `tlsname` (string): The domain name the SMTP server's TLS certificate must be valid for, defaulting to `smtp_host`. + +* `notif_from` (string|null): Defines the "From" address to use when sending emails. It must be set if email sending is enabled. The placeholder `%(app)s` will be replaced by the application name, which is normally set in `app_name`, but may be overridden by the Matrix client application. Note that the placeholder must be written `%(app)s`, including the trailing 's'. Defaults to `null`. + +* `app_name` (string): Defines the default value for `%(app)s` in `notif_from` and email subjects. Defaults to `"Matrix"`. + +* `enable_notifs` (boolean): Set to true to allow users to receive e-mail notifications. If this is not set, users can configure e-mail notifications but will not receive them. Defaults to `false`. + +* `notif_for_new_users` (boolean): Set to false to disable automatic subscription to email notifications for new users. Defaults to `true`. + +* `notif_delay_before_mail` (duration): The time to wait before emailing about a notification. This gives the user a chance to view the message via push or an open client. _New in Synapse 1.99.0._ -* `client_base_url`: Custom URL for client links within the email notifications. By default - links will be based on "https://matrix.to". (This setting used to be called `riot_base_url`; - the old name is still supported for backwards-compatibility but is now deprecated.) -* `validation_token_lifetime`: Configures the time that a validation email will expire after sending. - Defaults to 1h. -* `invite_client_location`: The web client location to direct users to during an invite. This is passed - to the identity server as the `org.matrix.web_client_location` key. Defaults - to unset, giving no guidance to the identity server. -* `subjects`: Subjects to use when sending emails from Synapse. The placeholder '%(app)s' will - be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application. - In addition, each subject can use the following placeholders: '%(person)s', which will be replaced by the displayname - of the user(s) that sent the message(s), e.g. "Alice and Bob", and '%(room)s', which will be replaced by the name of the room the - message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will - can use the '%(server_name)s' placeholder, which will be replaced by the value of the - `server_name` setting in your Synapse configuration. - Here is a list of subjects for notification emails that can be set: - * `message_from_person_in_room`: Subject to use to notify about one message from one or more user(s) in a - room which has a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." - * `message_from_person`: Subject to use to notify about one message from one or more user(s) in a - room which doesn't have a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s..." - * `messages_from_person`: Subject to use to notify about multiple messages from one or more users in - a room which doesn't have a name. Defaults to "[%(app)s] You have messages on %(app)s from %(person)s..." - * `messages_in_room`: Subject to use to notify about multiple messages in a room which has a - name. Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room..." - * `messages_in_room_and_others`: Subject to use to notify about multiple messages in multiple rooms. - Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." - * `messages_from_person_and_others`: Subject to use to notify about multiple messages from multiple persons in - multiple rooms. This is similar to the setting above except it's used when - the room in which the notification was triggered has no name. Defaults to - "[%(app)s] You have messages on %(app)s from %(person)s and others..." - * `invite_from_person_to_room`: Subject to use to notify about an invite to a room which has a name. - Defaults to "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." - * `invite_from_person`: Subject to use to notify about an invite to a room which doesn't have a - name. Defaults to "[%(app)s] %(person)s has invited you to chat on %(app)s..." - * `password_reset`: Subject to use when sending a password reset email. Defaults to "[%(server_name)s] Password reset" - * `email_validation`: Subject to use when sending a verification email to assert an address's - ownership. Defaults to "[%(server_name)s] Validate your email" + Defaults to `"10m"`. + +* `client_base_url` (string): Custom URL for client links within the email notifications. (This setting used to be called `riot_base_url`; the old name is still supported for backwards-compatibility but is now deprecated.) Defaults to `"https://matrix.to"`. + +* `validation_token_lifetime` (duration): Configures the time that a validation email will expire after sending. Defaults to `"1h"`. + +* `invite_client_location` (string|null): The web client location to direct users to during an invite. This is passed to the identity server as the `org.matrix.web_client_location` key. If null or unset no guidance is given to the identity server. Defaults to `null`. + +* `subjects` (object): Subjects to use when sending emails from Synapse. The placeholder `%(app)s` will be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application. In addition, each subject can use the following placeholders: `%(person)s`, which will be replaced by the displayname of the user(s) that sent the message(s), e.g. "Alice and Bob", and `%(room)s`, which will be replaced by the name of the room the message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will can use the `%(server_name)s` placeholder, which will be replaced by the value of the `server_name` setting in your Synapse configuration. + + This setting has the following sub-options: + + * `message_from_person_in_room` (string): Subject to use to notify about one message from one or more user(s) in a room which has a name. Defaults to `"[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..."`. + + * `message_from_person` (string): Subject to use to notify about one message from one or more user(s) in a room which doesn't have a name. Defaults to `"[%(app)s] You have a message on %(app)s from %(person)s..."`. + + * `messages_from_person` (string): Subject to use to notify about multiple messages from one or more users in a room which doesn't have a name. Defaults to `"[%(app)s] You have messages on %(app)s from %(person)s..."`. + + * `messages_in_room` (string): Subject to use to notify about multiple messages in a room which has a name. Defaults to `"[%(app)s] You have messages on %(app)s in the %(room)s room..."`. + + * `messages_in_room_and_others` (string): Subject to use to notify about multiple messages in multiple rooms. Defaults to `"[%(app)s] You have messages on %(app)s in the %(room)s room and others..."`. + + * `messages_from_person_and_others` (string): Subject to use to notify about multiple messages from multiple persons in multiple rooms. This is similar to the setting above except it's used when the room in which the notification was triggered has no name. Defaults to `"[%(app)s] You have messages on %(app)s from %(person)s and others..."`. + + * `invite_from_person_to_room` (string): Subject to use to notify about an invite to a room which has a name. Defaults to `"[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..."`. + + * `invite_from_person` (string): Subject to use to notify about an invite to a room which doesn't have a name. Defaults to `"[%(app)s] %(person)s has invited you to chat on %(app)s..."`. + + * `password_reset` (string): Subject to use when sending a password reset email. Defaults to `"[%(server_name)s] Password reset"`. + + * `email_validation` (string): Subject to use when sending a verification email to assert an address's ownership. Defaults to `"[%(server_name)s] Validate your email"`. Example configuration: - ```yaml email: smtp_host: mail.server smtp_port: 587 - smtp_user: "exampleusername" - smtp_pass: "examplepassword" + smtp_user: exampleusername + smtp_pass: examplepassword force_tls: true require_transport_security: true enable_tls: false - notif_from: "Your Friendly %(app)s homeserver " + tlsname: mail.server.example.com + notif_from: Your Friendly %(app)s homeserver app_name: my_branded_matrix_server enable_notifs: true notif_for_new_users: false - client_base_url: "http://localhost/riot" + client_base_url: http://localhost/riot validation_token_lifetime: 15m invite_client_location: https://app.element.io - subjects: - message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." - message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..." - messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..." - messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..." - messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." - messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..." - invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." - invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..." - password_reset: "[%(server_name)s] Password reset" - email_validation: "[%(server_name)s] Validate your email" + message_from_person_in_room: '[%(app)s] You have a message on %(app)s from %(person)s + in the %(room)s room...' + message_from_person: '[%(app)s] You have a message on %(app)s from %(person)s...' + messages_from_person: '[%(app)s] You have messages on %(app)s from %(person)s...' + messages_in_room: '[%(app)s] You have messages on %(app)s in the %(room)s room...' + messages_in_room_and_others: '[%(app)s] You have messages on %(app)s in the %(room)s + room and others...' + messages_from_person_and_others: '[%(app)s] You have messages on %(app)s from + %(person)s and others...' + invite_from_person_to_room: '[%(app)s] %(person)s has invited you to join the + %(room)s room on %(app)s...' + invite_from_person: '[%(app)s] %(person)s has invited you to chat on %(app)s...' + password_reset: '[%(server_name)s] Password reset' + email_validation: '[%(server_name)s] Validate your email' ``` --- ### `max_event_delay_duration` -The maximum allowed duration by which sent events can be delayed, as per -[MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). -Must be a positive value if set. +The maximum allowed duration by which sent events can be delayed, as per [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). Must be a positive value if set. -Defaults to no duration (`null`), which disallows sending delayed events. +If null or unset, sending of delayed events is disallowed. + +Defaults to `null`. Example configuration: ```yaml max_event_delay_duration: 24h ``` - -## Homeserver blocking -Useful options for Synapse admins. - --- +### `user_types` -### `admin_contact` +*(object)* Configuration settings related to the user types feature. -How to reach the server admin, used in `ResourceLimitError`. Defaults to none. +This setting has the following sub-options: + +* `default_user_type` (string|null): The default user type to use for registering new users when no value has been specified. Defaults to none. Defaults to `null`. + +* `extra_user_types` (array): Array of additional user types to allow. These are treated as real users. Defaults to `[]`. Example configuration: ```yaml -admin_contact: 'mailto:admin@server.com' +user_types: + default_user_type: custom + extra_user_types: + - custom + - custom2 ``` --- -### `hs_disabled` and `hs_disabled_message` +## Homeserver blocking -Blocks users from connecting to the homeserver and provides a human-readable reason -why the connection was blocked. Defaults to false. +Useful options for Synapse admins. + +--- +### `admin_contact` + +*(string|null)* How to reach the server admin, used in `ResourceLimitError`. Defaults to `null`. + +Example configuration: +```yaml +admin_contact: mailto:admin@server.com +``` +--- +### `hs_disabled` + +*(boolean)* Blocks users from connecting to the homeserver and provides the human-readable reason given in `hs_disabled_message`. Defaults to `false`. Example configuration: ```yaml hs_disabled: true -hs_disabled_message: 'Reason for why the HS is blocked' +``` +--- +### `hs_disabled_message` + +*(string)* Human-readable reason why the connection was blocked. Defaults to `"Homeserver is currently blocked"`. + +Example configuration: +```yaml +hs_disabled_message: Reason for why the HS is blocked ``` --- ### `limit_usage_by_mau` -This option disables/enables monthly active user blocking. Used in cases where the admin or -server owner wants to limit to the number of monthly active users. When enabled and a limit is -reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. -Defaults to false. If this is enabled, a value for `max_mau_value` must also be set. +*(boolean)* This option disables/enables monthly active user blocking. Used in cases where the admin or server owner wants to limit to the number of monthly active users. When enabled and a limit is reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. If this is enabled, a value for `max_mau_value` must also be set. See [Monthly Active Users](../administration/monthly_active_users.md) for details on how to configure MAU. +Defaults to `false`. + Example configuration: ```yaml limit_usage_by_mau: true @@ -816,8 +883,7 @@ limit_usage_by_mau: true --- ### `max_mau_value` -This option sets the hard limit of monthly active users above which the server will start -blocking user actions if `limit_usage_by_mau` is enabled. Defaults to 0. +*(integer)* This option sets the hard limit of monthly active users above which the server will start blocking user actions if `limit_usage_by_mau` is enabled. Defaults to `0`. Example configuration: ```yaml @@ -826,11 +892,7 @@ max_mau_value: 50 --- ### `mau_trial_days` -The option `mau_trial_days` is a means to add a grace period for active users. It -means that users must be active for the specified number of days before they -can be considered active and guards against the case where lots of users -sign up in a short space of time never to return after their initial -session. Defaults to 0. +*(integer)* The option `mau_trial_days` is a means to add a grace period for active users. It means that users must be active for the specified number of days before they can be considered active and guards against the case where lots of users sign up in a short space of time never to return after their initial session. Defaults to `0`. Example configuration: ```yaml @@ -839,10 +901,7 @@ mau_trial_days: 5 --- ### `mau_appservice_trial_days` -The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but applies a different -trial number if the user was registered by an appservice. A value -of 0 means no trial days are applied. Appservices not listed in this dictionary -use the value of `mau_trial_days` instead. +*(object)* The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but applies a different trial number if the user was registered by an appservice. A value of 0 means no trial days are applied. Appservices not listed in this dictionary use the value of `mau_trial_days` instead. Defaults to `{}`. Example configuration: ```yaml @@ -853,11 +912,7 @@ mau_appservice_trial_days: --- ### `mau_limit_alerting` -The option `mau_limit_alerting` is a means of limiting client-side alerting -should the mau limit be reached. This is useful for small instances -where the admin has 5 mau seats (say) for 5 specific people and no -interest increasing the mau limit further. Defaults to true, which -means that alerting is enabled. +*(boolean)* Limit client-side alerting should the mau limit be reached. This is useful for small instances where the admin has 5 mau seats (say) for 5 specific people and no interest increasing the mau limit further. Defaults to `true`. Example configuration: ```yaml @@ -866,9 +921,7 @@ mau_limit_alerting: false --- ### `mau_stats_only` -If enabled, the metrics for the number of monthly active users will -be populated, however no one will be limited based on these numbers. If `limit_usage_by_mau` -is true, this is implied to be true. Defaults to false. +*(boolean)* If enabled, the metrics for the number of monthly active users will be populated, however no one will be limited based on these numbers. If `limit_usage_by_mau` is true, this is implied to be true. Defaults to `false`. Example configuration: ```yaml @@ -877,22 +930,24 @@ mau_stats_only: true --- ### `mau_limit_reserved_threepids` -Sometimes the server admin will want to ensure certain accounts are -never blocked by mau checking. These accounts are specified by this option. -Defaults to none. Add accounts by specifying the `medium` and `address` of the -reserved threepid (3rd party identifier). +*(array)* Sometimes the server admin will want to ensure certain accounts are never blocked by mau checking. These accounts are specified by this option. Add accounts by specifying the `medium` and `address` of the reserved threepid (3rd party identifier). Defaults to `[]`. + +Options for each entry include: + +* `medium` (string): Medium of the account threepid. + +* `address` (string): Address of the account threepid. Example configuration: ```yaml mau_limit_reserved_threepids: - - medium: 'email' - address: 'reserved_user@example.com' +- medium: email + address: reserved_user@example.com ``` --- ### `server_context` -This option is used by phonehome stats to group together related servers. -Defaults to none. +*(string|null)* This option is used by phonehome stats to group together related servers. Defaults to `null`. Example configuration: ```yaml @@ -901,32 +956,30 @@ server_context: context --- ### `limit_remote_rooms` -When this option is enabled, the room "complexity" will be checked before a user -joins a new remote room. If it is above the complexity limit, the server will -disallow joining, or will instantly leave. This is useful for homeservers that are -resource-constrained. Options for this setting include: -* `enabled`: whether this check is enabled. Defaults to false. -* `complexity`: the limit above which rooms cannot be joined. The default is 1.0. -* `complexity_error`: override the error which is returned when the room is too complex with a - custom message. -* `admins_can_join`: allow server admins to join complex rooms. Default is false. +*(object)* When this option is enabled, the room "complexity" will be checked before a user joins a new remote room. If it is above the complexity limit, the server will disallow joining, or will instantly leave. This is useful for homeservers that are resource-constrained. Room complexity is an arbitrary measure based on factors such as the number of users in the room. -Room complexity is an arbitrary measure based on factors such as the number of -users in the room. +This setting has the following sub-options: + +* `enabled` (boolean): Whether this check is enabled. Defaults to `false`. + +* `complexity` (number): The limit above which rooms cannot be joined. Defaults to `1.0`. + +* `complexity_error` (string): Override the error which is returned when the room is too complex with a custom message. Defaults to `"Your homeserver is unable to join rooms this large or complex. Please speak to your server administrator, or upgrade your instance to join this room."`. + +* `admins_can_join` (boolean): Allow server admins to join complex rooms. Defaults to `false`. Example configuration: ```yaml limit_remote_rooms: enabled: true complexity: 0.5 - complexity_error: "I can't let you do that, Dave." + complexity_error: I can't let you do that, Dave. admins_can_join: true ``` --- ### `require_membership_for_aliases` -Whether to require a user to be in the room to add an alias to it. -Defaults to true. +*(boolean)* Whether to require a user to be in the room to add an alias to it. Defaults to `true`. Example configuration: ```yaml @@ -935,9 +988,7 @@ require_membership_for_aliases: false --- ### `allow_per_room_profiles` -Whether to allow per-room membership profiles through the sending of membership -events with profile information that differs from the target's global profile. -Defaults to true. +*(boolean)* Whether to allow per-room membership profiles through the sending of membership events with profile information that differs from the target's global profile. Defaults to `true`. Example configuration: ```yaml @@ -946,11 +997,12 @@ allow_per_room_profiles: false --- ### `max_avatar_size` -The largest permissible file size in bytes for a user avatar. Defaults to no restriction. -Use M for MB and K for KB. +The largest permissible file size in bytes for a user avatar. Defaults to no restriction. Use M for MB and K for KB. Note that user avatar changes will not work if this is set without using Synapse's media repository. +Defaults to `null`. + Example configuration: ```yaml max_avatar_size: 10M @@ -958,26 +1010,27 @@ max_avatar_size: 10M --- ### `allowed_avatar_mimetypes` -The MIME types allowed for user avatars. Defaults to no restriction. +*(array|null)* The MIME types allowed for user avatars. Defaults to no restriction. -Note that user avatar changes will not work if this is set without -using Synapse's media repository. +Note that user avatar changes will not work if this is set without using Synapse's media repository. + +Defaults to `null`. Example configuration: ```yaml -allowed_avatar_mimetypes: ["image/png", "image/jpeg", "image/gif"] +allowed_avatar_mimetypes: +- image/png +- image/jpeg +- image/gif ``` --- ### `redaction_retention_period` -How long to keep redacted events in unredacted form in the database. After -this period redacted events get replaced with their redacted form in the DB. +How long to keep redacted events in unredacted form in the database. After this period redacted events get replaced with their redacted form in the DB. -Synapse will check whether the rentention period has concluded for redacted -events every 5 minutes. Thus, even if this option is set to `0`, Synapse may -still take up to 5 minutes to purge redacted events from the database. +Synapse will check whether the rentention period has concluded for redacted events every 5 minutes. Thus, even if this option is set to `0`, Synapse may still take up to 5 minutes to purge redacted events from the database. Set to `null` to disable. -Defaults to `7d`. Set to `null` to disable. +Defaults to `"7d"`. Example configuration: ```yaml @@ -986,9 +1039,7 @@ redaction_retention_period: 28d --- ### `forgotten_room_retention_period` -How long to keep locally forgotten rooms before purging them from the DB. - -Defaults to `null`, meaning it's disabled. +How long to keep locally forgotten rooms before purging them from the DB. A value of `null` means it's disabled. Defaults to `null`. Example configuration: ```yaml @@ -997,9 +1048,7 @@ forgotten_room_retention_period: 28d --- ### `user_ips_max_age` -How long to track users' last seen time and IPs in the database. - -Defaults to `28d`. Set to `null` to disable clearing out of old rows. +How long to track users' last seen time and IPs in the database. Set to `null` to disable clearing out of old rows. Defaults to `"28d"`. Example configuration: ```yaml @@ -1008,13 +1057,7 @@ user_ips_max_age: 14d --- ### `request_token_inhibit_3pid_errors` -Inhibits the `/requestToken` endpoints from returning an error that might leak -information about whether an e-mail address is in use or not on this -homeserver. Defaults to false. -Note that for some endpoints the error situation is the e-mail already being -used, and for others the error is entering the e-mail being unused. -If this option is enabled, instead of returning an error, these endpoints will -act as if no error happened and return a fake session ID ('sid') to clients. +*(boolean)* Inhibits the `/requestToken` endpoints from returning an error that might leak information about whether an e-mail address is in use or not on this homeserver. Note that for some endpoints the error situation is the e-mail already being used, and for others the error is entering the e-mail being unused. If this option is enabled, instead of returning an error, these endpoints will act as if no error happened and return a fake session ID (`sid`) to clients. Defaults to `false`. Example configuration: ```yaml @@ -1023,36 +1066,30 @@ request_token_inhibit_3pid_errors: true --- ### `next_link_domain_whitelist` -A list of domains that the domain portion of `next_link` parameters -must match. +*(array|null)* A list of domains that the domain portion of `next_link` parameters must match. -This parameter is optionally provided by clients while requesting -validation of an email or phone number, and maps to a link that -users will be automatically redirected to after validation -succeeds. Clients can make use this parameter to aid the validation -process. +This parameter is optionally provided by clients while requesting validation of an email or phone number, and maps to a link that users will be automatically redirected to after validation succeeds. Clients can make use this parameter to aid the validation process. The whitelist is applied whether the homeserver or an identity server is handling validation. -The default value is no whitelist functionality; all domains are -allowed. Setting this value to an empty list will instead disallow -all domains. +The default value is no whitelist functionality; all domains are allowed. Setting this value to an empty list will instead disallow all domains. + +Defaults to `null`. Example configuration: ```yaml -next_link_domain_whitelist: ["matrix.org"] +next_link_domain_whitelist: matrix.org ``` --- -### `templates` and `custom_template_directory` +### `templates` -These options define templates to use when generating email or HTML page contents. -The `custom_template_directory` determines which directory Synapse will try to -find template files in to use to generate email or HTML page contents. -If not set, or a file is not found within the template directory, a default -template from within the Synapse package will be used. +*(object)* These options define templates to use when generating email or HTML page contents. -See [here](../../templates.md) for more -information about using custom templates. +See [here](../../templates.md) for more information about using custom templates. + +This setting has the following sub-options: + +* `custom_template_directory` (string|null): Determines which directory Synapse will try to find template files in to use to generate email or HTML page contents. If not set, or a file is not found within the template directory, a default template from within the Synapse package will be used. Defaults to `null`. Example configuration: ```yaml @@ -1062,62 +1099,49 @@ templates: --- ### `retention` -This option and the associated options determine message retention policy at the -server level. +*(object)* This option and the associated options determine message retention policy at the server level. -Room admins and mods can define a retention period for their rooms using the -`m.room.retention` state event, and server admins can cap this period by setting -the `allowed_lifetime_min` and `allowed_lifetime_max` config options. +Room admins and mods can define a retention period for their rooms using the `m.room.retention` state event, and server admins can cap this period by setting the `allowed_lifetime_min` and `allowed_lifetime_max` config options. -If this feature is enabled, Synapse will regularly look for and purge events -which are older than the room's maximum retention period. Synapse will also -filter events received over federation so that events that should have been -purged are ignored and not stored again. +If this feature is enabled, Synapse will regularly look for and purge events which are older than the room's maximum retention period. Synapse will also filter events received over federation so that events that should have been purged are ignored and not stored again. -The message retention policies feature is disabled by default. You can read more -about this feature [here](../../message_retention_policies.md). +The message retention policies feature is disabled by default. You can read more about this feature [here](../../message_retention_policies.md). This setting has the following sub-options: -* `default_policy`: Default retention policy. If set, Synapse will apply it to rooms that lack the - 'm.room.retention' state event. This option is further specified by the - `min_lifetime` and `max_lifetime` sub-options associated with it. Note that the - value of `min_lifetime` doesn't matter much because Synapse doesn't take it into account yet. -* `allowed_lifetime_min` and `allowed_lifetime_max`: Retention policy limits. If - set, and the state of a room contains a `m.room.retention` event in its state - which contains a `min_lifetime` or a `max_lifetime` that's out of these bounds, - Synapse will cap the room's policy to these limits when running purge jobs. +* `enabled` (boolean): Enforce message retention policies Defaults to `false`. -* `purge_jobs` and the associated `shortest_max_lifetime` and `longest_max_lifetime` sub-options: - Server admins can define the settings of the background jobs purging the - events whose lifetime has expired under the `purge_jobs` section. +* `default_policy` (object): Default message retention policy. If set, Synapse will apply it to rooms that lack the `m.room.retention` state event. - If no configuration is provided for this option, a single job will be set up to delete - expired events in every room daily. + This setting has the following sub-options: - Each job's configuration defines which range of message lifetimes the job - takes care of. For example, if `shortest_max_lifetime` is '2d' and - `longest_max_lifetime` is '3d', the job will handle purging expired events in - rooms whose state defines a `max_lifetime` that's both higher than 2 days, and - lower than or equal to 3 days. Both the minimum and the maximum value of a - range are optional, e.g. a job with no `shortest_max_lifetime` and a - `longest_max_lifetime` of '3d' will handle every room with a retention policy - whose `max_lifetime` is lower than or equal to three days. + * `min_lifetime`: Minimum message retention time of the default message retention policy. Synapse doesn't take this option into account yet. Defaults to `null`. - The rationale for this per-job configuration is that some rooms might have a - retention policy with a low `max_lifetime`, where history needs to be purged - of outdated messages on a more frequent basis than for the rest of the rooms - (e.g. every 12h), but not want that purge to be performed by a job that's - iterating over every room it knows, which could be heavy on the server. + * `max_lifetime`: Maximum message retention time of the default message retention policy. Defaults to `null`. - If any purge job is configured, it is strongly recommended to have at least - a single job with neither `shortest_max_lifetime` nor `longest_max_lifetime` - set, or one job without `shortest_max_lifetime` and one job without - `longest_max_lifetime` set. Otherwise some rooms might be ignored, even if - `allowed_lifetime_min` and `allowed_lifetime_max` are set, because capping a - room's policy to these values is done after the policies are retrieved from - Synapse's database (which is done using the range specified in a purge job's - configuration). +* `allowed_lifetime_min`: Retention policy limit. If set, and the state of a room contains a `m.room.retention` event in its state which contains a `min_lifetime` that's beyond this bound, Synapse will cap the room's policy to these limits when running purge jobs. Defaults to `null`. + +* `allowed_lifetime_max`: Retention policy limit. If set, and the state of a room contains a `m.room.retention` event in its state which contains a `max_lifetime` that's beyond this bound, Synapse will cap the room's policy to these limits when running purge jobs. Defaults to `null`. + +* `purge_jobs` (array|null): Server admins can define the settings of the background jobs purging the events whose lifetime has expired under the `purge_jobs` section. + + If no configuration is provided for this option, a single job will be set up to delete expired events in every room daily. + + Each job's configuration defines which range of message lifetimes the job takes care of. For example, if `shortest_max_lifetime` is "2d" and `longest_max_lifetime` is "3d", the job will handle purging expired events in rooms whose state defines a `max_lifetime` that's both higher than 2 days, and lower than or equal to 3 days. Both the minimum and the maximum value of a range are optional, e.g. a job with no `shortest_max_lifetime` and a `longest_max_lifetime` of "3d" will handle every room with a retention policy whose `max_lifetime` is lower than or equal to three days. + + The rationale for this per-job configuration is that some rooms might have a retention policy with a low `max_lifetime`, where history needs to be purged of outdated messages on a more frequent basis than for the rest of the rooms (e.g. every 12h), but not want that purge to be performed by a job that's iterating over every room it knows, which could be heavy on the server. + + If any purge job is configured, it is strongly recommended to have at least a single job with neither `shortest_max_lifetime` nor `longest_max_lifetime` set, or one job without `shortest_max_lifetime` and one job without `longest_max_lifetime` set. Otherwise some rooms might be ignored, even if `allowed_lifetime_min` and `allowed_lifetime_max` are set, because capping a room's policy to these values is done after the policies are retrieved from Synapse's database (which is done using the range specified in a purge job's configuration). + + Defaults to `null`. + + Options for each entry include: + + * `shortest_max_lifetime`: Apply job to rooms that have a `max_lifetime` higher than `shortest_max_lifetime`. A value of `null` never excludes any room. + + * `longest_max_lifetime`: Apply job to rooms that have a `max_lifetime` lower than or equal to `shortest_max_lifetime`. A value of `null` never excludes any room. + + * `interval` (duration): How often to run the job. Example configuration: ```yaml @@ -1129,10 +1153,10 @@ retention: allowed_lifetime_min: 1d allowed_lifetime_max: 1y purge_jobs: - - longest_max_lifetime: 3d - interval: 12h - - shortest_max_lifetime: 3d - interval: 1d + - longest_max_lifetime: 3d + interval: 12h + - shortest_max_lifetime: 3d + interval: 1d ``` --- ## TLS @@ -1142,32 +1166,29 @@ Options related to TLS. --- ### `tls_certificate_path` -This option specifies a PEM-encoded X509 certificate for TLS. -This certificate, as of Synapse 1.0, will need to be a valid and verifiable -certificate, signed by a recognised Certificate Authority. Defaults to none. +*(string|null)* This option specifies a PEM-encoded X509 certificate for TLS. This certificate, as of Synapse 1.0, will need to be a valid and verifiable certificate, signed by a recognised Certificate Authority. -Be sure to use a `.pem` file that includes the full certificate chain including -any intermediate certificates (for instance, if using certbot, use -`fullchain.pem` as your certificate, not `cert.pem`). +Be sure to use a `.pem` file that includes the full certificate chain including any intermediate certificates (for instance, if using certbot, use `fullchain.pem` as your certificate, not `cert.pem`). + +Defaults to `null`. Example configuration: ```yaml -tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt" +tls_certificate_path: CONFDIR/SERVERNAME.tls.crt ``` --- ### `tls_private_key_path` -PEM-encoded private key for TLS. Defaults to none. +*(string|null)* PEM-encoded private key for TLS. Defaults to `null`. Example configuration: ```yaml -tls_private_key_path: "CONFDIR/SERVERNAME.tls.key" +tls_private_key_path: CONFDIR/SERVERNAME.tls.key ``` --- ### `federation_verify_certificates` -Whether to verify TLS server certificates for outbound federation requests. -Defaults to true. To disable certificate verification, set the option to false. +*(boolean)* Whether to verify TLS server certificates for outbound federation requests. To disable certificate verification, set the option to false. Defaults to `true`. Example configuration: ```yaml @@ -1176,53 +1197,51 @@ federation_verify_certificates: false --- ### `federation_client_minimum_tls_version` -The minimum TLS version that will be used for outbound federation requests. +*(string)* The minimum TLS version that will be used for outbound federation requests. -Defaults to `"1"`. Configurable to `"1"`, `"1.1"`, `"1.2"`, or `"1.3"`. Note -that setting this value higher than `"1.2"` will prevent federation to most -of the public Matrix network: only configure it to `"1.3"` if you have an -entirely private federation setup and you can ensure TLS 1.3 support. +Configurable to `"1"`, `"1.1"`, `"1.2"`, or `"1.3"`. Note that setting this value higher than `"1.2"` will prevent federation to most of the public Matrix network: only configure it to `"1.3"` if you have an entirely private federation setup and you can ensure TLS 1.3 support. + +Defaults to `"1"`. Example configuration: ```yaml -federation_client_minimum_tls_version: "1.2" +federation_client_minimum_tls_version: '1.2' ``` --- ### `federation_certificate_verification_whitelist` -Skip federation certificate verification on a given whitelist -of domains. +*(array)* Skip federation certificate verification on a given whitelist of domains. -This setting should only be used in very specific cases, such as -federation over Tor hidden services and similar. For private networks -of homeservers, you likely want to use a private CA instead. +This setting should only be used in very specific cases, such as federation over Tor hidden services and similar. For private networks of homeservers, you likely want to use a private CA instead. Only effective if `federation_verify_certificates` is `true`. +Defaults to `[]`. + Example configuration: ```yaml federation_certificate_verification_whitelist: - - lon.example.com - - "*.domain.com" - - "*.onion" +- lon.example.com +- '*.domain.com' +- '*.onion' ``` --- ### `federation_custom_ca_list` -List of custom certificate authorities for federation traffic. +*(array)* List of custom certificate authorities for federation traffic. -This setting should only normally be used within a private network of -homeservers. +This setting should only normally be used within a private network of homeservers. -Note that this list will replace those that are provided by your -operating environment. Certificates must be in PEM format. +Note that this list will replace those that are provided by your operating environment. Certificates must be in PEM format. + +Defaults to `[]`. Example configuration: ```yaml federation_custom_ca_list: - - myCA1.pem - - myCA2.pem - - myCA3.pem +- myCA1.pem +- myCA2.pem +- myCA3.pem ``` --- ## Federation @@ -1232,31 +1251,25 @@ Options related to federation. --- ### `federation_domain_whitelist` -Restrict federation to the given whitelist of domains. -N.B. we recommend also firewalling your federation listener to limit -inbound federation traffic as early as possible, rather than relying -purely on this application-layer restriction. If not specified, the -default is to whitelist everything. +*(array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. If not specified, the default is to whitelist everything. -Note: this does not stop a server from joining rooms that servers not on the -whitelist are in. As such, this option is really only useful to establish a -"private federation", where a group of servers all whitelist each other and have -the same whitelist. +Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to establish a "private federation", where a group of servers all whitelist each other and have the same whitelist. + +Defaults to `[]`. Example configuration: ```yaml federation_domain_whitelist: - - lon.example.com - - nyc.example.com - - syd.example.com +- lon.example.com +- nyc.example.com +- syd.example.com ``` --- ### `federation_whitelist_endpoint_enabled` -Enables an endpoint for fetching the federation whitelist config. +*(boolean)* Enables an endpoint for fetching the federation whitelist config. -The request method and path is `GET /_synapse/client/v1/config/federation_whitelist`, and the -response format is: +The request method and path is `GET /_synapse/client/v1/config/federation_whitelist`, and the response format is: ```json { @@ -1271,6 +1284,8 @@ If `whitelist_enabled` is `false` then the server is permitted to federate with The endpoint requires authentication. +Defaults to `false`. + Example configuration: ```yaml federation_whitelist_endpoint_enabled: true @@ -1278,25 +1293,18 @@ federation_whitelist_endpoint_enabled: true --- ### `federation_metrics_domains` -Report prometheus metrics on the age of PDUs being sent to and received from -the given domains. This can be used to give an idea of "delay" on inbound -and outbound federation, though be aware that any delay can be due to problems -at either end or with the intermediate network. - -By default, no domains are monitored in this way. +*(array)* Report prometheus metrics on the age of PDUs being sent to and received from the given domains. This can be used to give an idea of "delay" on inbound and outbound federation, though be aware that any delay can be due to problems at either end or with the intermediate network. Defaults to `[]`. Example configuration: ```yaml federation_metrics_domains: - - matrix.org - - example.com +- matrix.org +- example.com ``` --- ### `allow_profile_lookup_over_federation` -Set to false to disable profile lookup over federation. By default, the -Federation API allows other homeservers to obtain profile data of any user -on this homeserver. +*(boolean)* Set to false to disable profile lookup over federation. By default, the Federation API allows other homeservers to obtain profile data of any user on this homeserver. Defaults to `true`. Example configuration: ```yaml @@ -1305,9 +1313,7 @@ allow_profile_lookup_over_federation: false --- ### `allow_device_name_lookup_over_federation` -Set this option to true to allow device display name lookup over federation. By default, the -Federation API prevents other homeservers from obtaining the display names of any user devices -on this homeserver. +*(boolean)* Set this option to true to allow device display name lookup over federation. By default, the Federation API prevents other homeservers from obtaining the display names of any user devices on this homeserver. Defaults to `false`. Example configuration: ```yaml @@ -1316,27 +1322,29 @@ allow_device_name_lookup_over_federation: true --- ### `federation` -The federation section defines some sub-options related to federation. +*(object)* The federation section defines some sub-options related to federation. -The following options are related to configuring timeout and retry logic for one request, -independently of the others. -Short retry algorithm is used when something or someone will wait for the request to have an -answer, while long retry is used for requests that happen in the background, -like sending a federation transaction. +The following options are related to configuring timeout and retry logic for one request, independently of the others. Short retry algorithm is used when something or someone will wait for the request to have an answer, while long retry is used for requests that happen in the background, like sending a federation transaction. -* `client_timeout`: timeout for the federation requests. Default to 60s. -* `max_short_retry_delay`: maximum delay to be used for the short retry algo. Default to 2s. -* `max_long_retry_delay`: maximum delay to be used for the short retry algo. Default to 60s. -* `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts. -* `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts. +`destination_*` options control the retry logic when communicating with a specific homeserver destination. Unlike the previous configuration options, these values apply across all requests for a given destination and the state of the backoff is stored in the database. -The following options control the retry logic when communicating with a specific homeserver destination. -Unlike the previous configuration options, these values apply across all requests -for a given destination and the state of the backoff is stored in the database. +This setting has the following sub-options: -* `destination_min_retry_interval`: the initial backoff, after the first request fails. Defaults to 10m. -* `destination_retry_multiplier`: how much we multiply the backoff by after each subsequent fail. Defaults to 2. -* `destination_max_retry_interval`: a cap on the backoff. Defaults to a week. +* `client_timeout` (duration): Timeout for the federation requests. Defaults to `"60s"`. + +* `max_short_retry_delay` (duration): Maximum delay to be used for the short retry algo. Defaults to `"2s"`. + +* `max_long_retry_delay` (duration): Maximum delay to be used for the long retry algo. Defaults to `"60s"`. + +* `max_short_retries` (integer): Maximum number of retries for the short retry algo. Defaults to `3`. + +* `max_long_retries` (integer): Maximum number of retries for the long retry algo. Defaults to `10`. + +* `destination_min_retry_interval` (duration): The initial backoff, after the first request fails. Defaults to `"10m"`. + +* `destination_retry_multiplier` (integer): How much we multiply the backoff by after each subsequent fail. Defaults to `2`. + +* `destination_max_retry_interval` (duration): A cap on the backoff. Defaults to `"1w"`. Example configuration: ```yaml @@ -1358,93 +1366,69 @@ Options related to caching. --- ### `event_cache_size` -The number of events to cache in memory. Defaults to 10K. Like other caches, -this is affected by `caches.global_factor` (see below). +*(size)* The number of events to cache in memory. Defaults to 10K. Like other caches, this is affected by `caches.global_factor` (see below). For example, the default is 10K and the global_factor default is 0.5. Since 10K * 0.5 is 5K then the event cache size will be 5K. -The cache affected by this configuration is named as "*getEvent*". +The cache affected by this configuration is named as "\*getEvent\*". Note that this option is not part of the `caches` section. +Defaults to `"10K"`. + Example configuration: ```yaml event_cache_size: 15K ``` --- -### `caches` and associated values +### `caches` -A cache 'factor' is a multiplier that can be applied to each of -Synapse's caches in order to increase or decrease the maximum -number of entries that can be stored. +*(object)* A cache "factor" is a multiplier that can be applied to each of Synapse's caches in order to increase or decrease the maximum number of entries that can be stored. -`caches` can be configured through the following sub-options: +This setting has the following sub-options: -* `global_factor`: Controls the global cache factor, which is the default cache factor - for all caches if a specific factor for that cache is not otherwise - set. +* `global_factor` (number): Controls the global cache factor, which is the default cache factor for all caches if a specific factor for that cache is not otherwise set. - This can also be set by the `SYNAPSE_CACHE_FACTOR` environment - variable. Setting by environment variable takes priority over - setting through the config file. + This can also be set by the `SYNAPSE_CACHE_FACTOR` environment variable. Setting by environment variable takes priority over setting through the config file. Defaults to 0.5, which will halve the size of all caches. Note that changing this value also affects the HTTP connection pool. -* `per_cache_factors`: A dictionary of cache name to cache factor for that individual - cache. Overrides the global cache factor for a given cache. + Defaults to `0.5`. - These can also be set through environment variables comprised - of `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital - letters and underscores. Setting by environment variable - takes priority over setting through the config file. - Ex. `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` +* `per_cache_factors` (object): A dictionary of cache name to cache factor for that individual cache. Overrides the global cache factor for a given cache. - Some caches have '*' and other characters that are not - alphanumeric or underscores. These caches can be named with or - without the special characters stripped. For example, to specify - the cache factor for `*stateGroupCache*` via an environment - variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. + These can also be set through environment variables comprised of `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital letters and underscores. Setting by environment variable takes priority over setting through the config file. Ex. `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` -* `expire_caches`: Controls whether cache entries are evicted after a specified time - period. Defaults to true. Set to false to disable this feature. Note that never expiring - caches may result in excessive memory usage. + Some caches have '*' and other characters that are not alphanumeric or underscores. These caches can be named with or without the special characters stripped. For example, to specify the cache factor for `*stateGroupCache*` via an environment variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. -* `cache_entry_ttl`: If `expire_caches` is enabled, this flag controls how long an entry can - be in a cache without having been accessed before being evicted. - Defaults to 30m. + Defaults to `{}`. -* `sync_response_cache_duration`: Controls how long the results of a /sync request are - cached for after a successful response is returned. A higher duration can help clients - with intermittent connections, at the cost of higher memory usage. - A value of zero means that sync responses are not cached. - Defaults to 2m. +* `expire_caches` (boolean): Controls whether cache entries are evicted after a specified time period. Set to false to disable this feature. Note that never expiring caches may result in excessive memory usage. Defaults to `true`. + +* `cache_entry_ttl` (duration): If `expire_caches` is enabled, this flag controls how long an entry can be in a cache without having been accessed before being evicted. Defaults to `"30m"`. + +* `sync_response_cache_duration` (duration): Controls how long the results of a /sync request are cached for after a successful response is returned. A higher duration can help clients with intermittent connections, at the cost of higher memory usage. A value of zero means that sync responses are not cached. *Changed in Synapse 1.62.0*: The default was changed from 0 to 2m. -* `cache_autotuning` and its sub-options `max_cache_memory_usage`, `target_cache_memory_usage`, and - `min_cache_ttl` work in conjunction with each other to maintain a balance between cache memory - usage and cache entry availability. You must be using [jemalloc](../administration/admin_faq.md#help-synapse-is-slow-and-eats-all-my-ramcpu) - to utilize this option, and all three of the options must be specified for this feature to work. This option - defaults to off, enable it by providing values for the sub-options listed below. Please note that the feature will not work - and may cause unstable behavior (such as excessive emptying of caches or exceptions) if all of the values are not provided. - Please see the [Config Conventions](#config-conventions) for information on how to specify memory size and cache expiry - durations. - * `max_cache_memory_usage` sets a ceiling on how much memory the cache can use before caches begin to be continuously evicted. - They will continue to be evicted until the memory usage drops below the `target_cache_memory_usage`, set in - the setting below, or until the `min_cache_ttl` is hit. There is no default value for this option. - * `target_cache_memory_usage` sets a rough target for the desired memory usage of the caches. There is no default value - for this option. - * `min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when - caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches - from being emptied while Synapse is evicting due to memory. There is no default value for this option. + Defaults to `"2m"`. + +* `cache_autotuning` (object): `cache_autotuning` and its sub-options `max_cache_memory_usage`, `target_cache_memory_usage`, and `min_cache_ttl` work in conjunction with each other to maintain a balance between cache memory usage and cache entry availability. You must be using [jemalloc](../administration/admin_faq.md#help-synapse-is-slow-and-eats-all-my-ramcpu) to utilize this option, and all three of the options must be specified for this feature to work. This option defaults to off, enable it by providing values for the sub-options listed below. Please note that the feature will not work and may cause unstable behavior (such as excessive emptying of caches or exceptions) if all of the values are not provided. Please see the [Config Conventions](#config-conventions) for information on how to specify memory size and cache expiry durations. + + This setting has the following sub-options: + + * `max_cache_memory_usage`: Sets a ceiling on how much memory the cache can use before caches begin to be continuously evicted. They will continue to be evicted until the memory usage drops below the `target_cache_memory_usage`, set in the setting below, or until the `min_cache_ttl` is hit. Defaults to `null`. + + * `target_cache_memory_usage`: Sets a rough target for the desired memory usage of the caches. Defaults to `null`. + + * `min_cache_ttl`: Sets a limit under which newer cache entries are not evicted and is only applied when caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches from being emptied while Synapse is evicting due to memory. Defaults to `null`. Example configuration: ```yaml -event_cache_size: 15K caches: global_factor: 1.0 per_cache_factors: @@ -1458,54 +1442,42 @@ caches: ### Reloading cache factors -The cache factors (i.e. `caches.global_factor` and `caches.per_cache_factors`) may be reloaded at any time by sending a -[`SIGHUP`](https://en.wikipedia.org/wiki/SIGHUP) signal to Synapse using e.g. +The cache factors (i.e. `caches.global_factor` and `caches.per_cache_factors`) may be reloaded at any time by sending a [`SIGHUP`](https://en.wikipedia.org/wiki/SIGHUP) signal to Synapse using e.g. ```commandline kill -HUP [PID_OF_SYNAPSE_PROCESS] ``` -If you are running multiple workers, you must individually update the worker -config file and send this signal to each worker process. +If you are running multiple workers, you must individually update the worker config file and send this signal to each worker process. -If you're using the [example systemd service](https://github.com/element-hq/synapse/blob/develop/contrib/systemd/matrix-synapse.service) -file in Synapse's `contrib` directory, you can send a `SIGHUP` signal by using -`systemctl reload matrix-synapse`. +If you're using the [example systemd service](https://github.com/element-hq/synapse/blob/develop/contrib/systemd/matrix-synapse.service) file in Synapse's `contrib` directory, you can send a `SIGHUP` signal by using `systemctl reload matrix-synapse`. --- ## Database + Config options related to database settings. --- ### `database` -The `database` setting defines the database that synapse uses to store all of -its data. +*(object)* The `database` setting defines the database that synapse uses to store all of its data. -Associated sub-options: +For more information on using Synapse with Postgres, see [here](../../postgres.md). -* `name`: this option specifies the database engine to use: either `sqlite3` (for SQLite) - or `psycopg2` (for PostgreSQL). If no name is specified Synapse will default to SQLite. +This setting has the following sub-options: -* `txn_limit` gives the maximum number of transactions to run per connection - before reconnecting. Defaults to 0, which means no limit. +* `name` (string): This option specifies the database engine to use: either `sqlite3` (for SQLite) or `psycopg2` (for PostgreSQL). If no name is specified Synapse will default to SQLite. Defaults to `"sqlite3"`. -* `allow_unsafe_locale` is an option specific to Postgres. Under the default behavior, Synapse will refuse to - start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended) - by setting `allow_unsafe_locale` to true. Note that doing so may corrupt your database. You can find more information - [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and [here](https://wiki.postgresql.org/wiki/Locale_data_changes). +* `txn_limit` (integer): Gives the maximum number of transactions to run per connection before reconnecting. 0 means no limit. Defaults to `0`. -* `args` gives options which are passed through to the database engine, - except for options starting with `cp_`, which are used to configure the Twisted - connection pool. For a reference to valid arguments, see: - * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) - * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) - * for [the connection pool](https://docs.twistedmatrix.com/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) +* `allow_unsafe_locale` (boolean): This option is specific to Postgres. Under the default behavior, Synapse will refuse to start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended) by setting `allow_unsafe_locale` to true. Note that doing so may corrupt your database. You can find more information [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and [here](https://wiki.postgresql.org/wiki/Locale_data_changes). Defaults to `false`. -For more information on using Synapse with Postgres, -see [here](../../postgres.md). +* `args` (object): Gives options which are passed through to the database engine, except for options starting with `cp_`, which are used to configure the Twisted connection pool. For a reference to valid arguments, see: + * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) + * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) + * for [the connection pool](https://docs.twistedmatrix.com/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) -Example SQLite configuration: +Example configurations: ```yaml database: name: sqlite3 @@ -1513,7 +1485,6 @@ database: database: /path/to/homeserver.db ``` -Example Postgres configuration: ```yaml database: name: psycopg2 @@ -1530,19 +1501,11 @@ database: --- ### `databases` -The `databases` option allows specifying a mapping between certain database tables and -database host details, spreading the load of a single Synapse instance across multiple -database backends. This is often referred to as "database sharding". This option is only -supported for PostgreSQL database backends. +*(object)* The `databases` option allows specifying a mapping between certain database tables and database host details, spreading the load of a single Synapse instance across multiple database backends. This is often referred to as "database sharding". This option is only supported for PostgreSQL database backends. -**Important note:** This is a supported option, but is not currently used in production by the -Matrix.org Foundation. Proceed with caution and always make backups. +**Important note:** This is a supported option, but is not currently used in production by the Matrix.org Foundation. Proceed with caution and always make backups. -`databases` is a dictionary of arbitrarily-named database entries. Each entry is equivalent -to the value of the `database` homeserver config option (see above), with the addition of -a `data_stores` key. `data_stores` is an array of strings that specifies the data store(s) -(a defined label for a set of tables) that should be stored on the associated database -backend entry. +`databases` is a dictionary of arbitrarily-named database entries. Each entry is equivalent to the value of the `database` homeserver config option (see above), with the addition of a `data_stores` key. `data_stores` is an array of strings that specifies the data store(s) (a defined label for a set of tables) that should be stored on the associated database backend entry. The currently defined values for `data_stores` are: @@ -1558,30 +1521,22 @@ The currently defined values for `data_stores` are: * `"main"`: All other database tables and sequences. -All databases will end up with additional tables used for tracking database schema migrations -and any pending background updates. Synapse will create these automatically on startup when checking for -and/or performing database schema migrations. +All databases will end up with additional tables used for tracking database schema migrations and any pending background updates. Synapse will create these automatically on startup when checking for and/or performing database schema migrations. -To migrate an existing database configuration (e.g. all tables on a single database) to a different -configuration (e.g. the "main" data store on one database, and "state" on another), do the following: +To migrate an existing database configuration (e.g. all tables on a single database) to a different configuration (e.g. the "main" data store on one database, and "state" on another), do the following: 1. Take a backup of your existing database. Things can and do go wrong and database corruption is no joke! -2. Ensure all pending database migrations have been applied and background updates have run. The simplest - way to do this is to use the `update_synapse_database` script supplied with your Synapse installation. +2. Ensure all pending database migrations have been applied and background updates have run. The simplest way to do this is to use the `update_synapse_database` script supplied with your Synapse installation. ```sh update_synapse_database --database-config homeserver.yaml --run-background-updates ``` -3. Copy over the necessary tables and sequences from one database to the other. Tables relating to database - migrations, schemas, schema versions and background updates should **not** be copied. +3. Copy over the necessary tables and sequences from one database to the other. Tables relating to database migrations, schemas, schema versions and background updates should **not** be copied. - As an example, say that you'd like to split out the "state" data store from an existing database which - currently contains all data stores. + As an example, say that you'd like to split out the "state" data store from an existing database which currently contains all data stores. - Simply copy the tables and sequences defined above for the "state" datastore from the existing database - to the secondary database. As noted above, additional tables will be created in the secondary database - when Synapse is started. + Simply copy the tables and sequences defined above for the "state" datastore from the existing database to the secondary database. As noted above, additional tables will be created in the secondary database when Synapse is started. 4. Modify/create the `databases` option in your `homeserver.yaml` to match the desired database configuration. 5. Start Synapse. Check that it starts up successfully and that things generally seem to be working. @@ -1589,14 +1544,16 @@ configuration (e.g. the "main" data store on one database, and "state" on anothe Only one of the options `database` or `databases` may be specified in your config, but not both. -Example configuration: +Defaults to `{}`. +Example configuration: ```yaml databases: basement_box: name: psycopg2 txn_limit: 10000 - data_stores: ["main"] + data_stores: + - main args: user: synapse_user password: secretpassword @@ -1605,11 +1562,11 @@ databases: port: 5432 cp_min: 5 cp_max: 10 - my_other_database: name: psycopg2 txn_limit: 10000 - data_stores: ["state"] + data_stores: + - state args: user: synapse_user password: secretpassword @@ -1621,240 +1578,345 @@ databases: ``` --- ## Logging + Config options related to logging. --- ### `log_config` -This option specifies a yaml python logging config file as described -[here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). +*(string|null)* This option specifies a yaml python logging config file as described [here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). Defaults to `null`. Example configuration: ```yaml -log_config: "CONFDIR/SERVERNAME.log.config" +log_config: CONFDIR/SERVERNAME.log.config ``` --- ## Ratelimiting + Options related to ratelimiting in Synapse. Each ratelimiting configuration is made of two parameters: - - `per_second`: number of requests a client can send per second. - - `burst_count`: number of requests a client can send before being throttled. +- `per_second`: number of requests a client can send per second. +- `burst_count`: number of requests a client can send before being throttled. + --- ### `rc_message` +*(object)* Ratelimiting settings for client messaging. -Ratelimiting settings for client messaging. +This is a ratelimiting option for messages that ratelimits sending based on the account the client is using. -This is a ratelimiting option for messages that ratelimits sending based on the account the client -is using. It defaults to: `per_second: 0.2`, `burst_count: 10`. +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_message: + per_second: 0.2 + burst_count: 10.0 +``` Example configuration: ```yaml rc_message: per_second: 0.5 - burst_count: 15 + burst_count: 15.0 ``` --- ### `rc_registration` -This option ratelimits registration requests based on the client's IP address. -It defaults to `per_second: 0.17`, `burst_count: 3`. +*(object)* This option ratelimits registration requests based on the client's IP address. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_registration: + per_second: 0.17 + burst_count: 3.0 +``` Example configuration: ```yaml rc_registration: per_second: 0.15 - burst_count: 2 + burst_count: 2.0 ``` --- ### `rc_registration_token_validity` -This option checks the validity of registration tokens that ratelimits requests based on -the client's IP address. -Defaults to `per_second: 0.1`, `burst_count: 5`. +*(object)* This option checks the validity of registration tokens that ratelimits requests based on the client's IP address. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_registration_token_validity: + per_second: 0.1 + burst_count: 5.0 +``` Example configuration: ```yaml rc_registration_token_validity: per_second: 0.3 - burst_count: 6 + burst_count: 6.0 ``` --- ### `rc_login` -This option specifies several limits for login: -* `address` ratelimits login requests based on the client's IP - address. Defaults to `per_second: 0.003`, `burst_count: 5`. +*(object)* This option specifies several limits for login. -* `account` ratelimits login requests based on the account the - client is attempting to log into. Defaults to `per_second: 0.003`, - `burst_count: 5`. +This setting has the following sub-options: -* `failed_attempts` ratelimits login requests based on the account the - client is attempting to log into, based on the amount of failed login - attempts for this account. Defaults to `per_second: 0.17`, `burst_count: 3`. +* `address` (object): Ratelimits login requests based on the client's IP address. Defaults to `{"per_second": 0.003, "burst_count": 5.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `account` (object): Ratelimits login requests based on the account the client is attempting to log into. Defaults to `{"per_second": 0.003, "burst_count": 5.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `failed_attempts` (object): Ratelimits login requests based on the account the client is attempting to log into, based on the amount of failed login attempts for this account. Defaults to `{"per_second": 0.17, "burst_count": 3.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. Example configuration: ```yaml rc_login: address: per_second: 0.15 - burst_count: 5 + burst_count: 5.0 account: per_second: 0.18 - burst_count: 4 + burst_count: 4.0 failed_attempts: per_second: 0.19 - burst_count: 7 + burst_count: 7.0 ``` --- ### `rc_admin_redaction` -This option sets ratelimiting redactions by room admins. If this is not explicitly -set then it uses the same ratelimiting as per `rc_message`. This is useful -to allow room admins to deal with abuse quickly. +*(object)* This option sets ratelimiting redactions by room admins. If this is not explicitly set then it uses the same ratelimiting as per `rc_message`. This is useful to allow room admins to deal with abuse quickly. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. Example configuration: ```yaml rc_admin_redaction: - per_second: 1 - burst_count: 50 + per_second: 1.0 + burst_count: 50.0 ``` --- ### `rc_joins` -This option allows for ratelimiting number of rooms a user can join. This setting has the following sub-options: +*(object)* This option allows for ratelimiting number of rooms a user can join. -* `local`: ratelimits when users are joining rooms the server is already in. - Defaults to `per_second: 0.1`, `burst_count: 10`. +This setting has the following sub-options: -* `remote`: ratelimits when users are trying to join rooms not on the server (which - can be more computationally expensive than restricting locally). Defaults to - `per_second: 0.01`, `burst_count: 10` +* `local` (object): Ratelimits when users are joining rooms the server is already in. Defaults to `{"per_second": 0.1, "burst_count": 10.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `remote` (object): Ratelimits when users are trying to join rooms not on the server (which can be more computationally expensive than restricting locally). Defaults to `{"per_second": 0.01, "burst_count": 10.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. Example configuration: ```yaml rc_joins: local: per_second: 0.2 - burst_count: 15 + burst_count: 15.0 remote: per_second: 0.03 - burst_count: 12 + burst_count: 12.0 ``` --- ### `rc_joins_per_room` -This option allows admins to ratelimit joins to a room based on the number of recent -joins (local or remote) to that room. It is intended to mitigate mass-join spam -waves which target multiple homeservers. - -By default, one join is permitted to a room every second, with an accumulating -buffer of up to ten instantaneous joins. - -Example configuration (default values): -```yaml -rc_joins_per_room: - per_second: 1 - burst_count: 10 -``` +*(object)* This option allows admins to ratelimit joins to a room based on the number of recent joins (local or remote) to that room. It is intended to mitigate mass-join spam waves which target multiple homeservers. _Added in Synapse 1.64.0._ +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_joins_per_room: + per_second: 1.0 + burst_count: 10.0 +``` + +Example configuration: +```yaml +rc_joins_per_room: + per_second: 1.0 + burst_count: 10.0 +``` --- ### `rc_3pid_validation` -This option ratelimits how often a user or IP can attempt to validate a 3PID. -Defaults to `per_second: 0.003`, `burst_count: 5`. +*(object)* This option ratelimits how often a user or IP can attempt to validate a 3PID. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_3pid_validation: + per_second: 0.003 + burst_count: 5.0 +``` Example configuration: ```yaml rc_3pid_validation: per_second: 0.003 - burst_count: 5 + burst_count: 5.0 ``` --- ### `rc_invites` -This option sets ratelimiting how often invites can be sent in a room or to a -specific user. `per_room` defaults to `per_second: 0.3`, `burst_count: 10`, -`per_user` defaults to `per_second: 0.003`, `burst_count: 5`, and `per_issuer` -defaults to `per_second: 0.3`, `burst_count: 10`. +*(object)* This option sets ratelimiting how often invites can be sent in a room or to a specific user. -Client requests that invite user(s) when [creating a -room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom) -will count against the `rc_invites.per_room` limit, whereas -client requests to [invite a single user to a -room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidinvite) -will count against both the `rc_invites.per_user` and `rc_invites.per_room` limits. +Client requests that invite user(s) when [creating a room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom) will count against the `rc_invites.per_room` limit, whereas client requests to [invite a single user to a room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidinvite) will count against both the `rc_invites.per_user` and `rc_invites.per_room` limits. -Federation requests to invite a user will count against the `rc_invites.per_user` -limit only, as Synapse presumes ratelimiting by room will be done by the sending server. - -The `rc_invites.per_user` limit applies to the *receiver* of the invite, rather than the -sender, meaning that a `rc_invite.per_user.burst_count` of 5 mandates that a single user -cannot *receive* more than a burst of 5 invites at a time. - -In contrast, the `rc_invites.per_issuer` limit applies to the *issuer* of the invite, meaning that a `rc_invite.per_issuer.burst_count` of 5 mandates that single user cannot *send* more than a burst of 5 invites at a time. +Federation requests to invite a user will count against the `rc_invites.per_user` limit only, as Synapse presumes ratelimiting by room will be done by the sending server. _Changed in version 1.63:_ added the `per_issuer` limit. +This setting has the following sub-options: + +* `per_room` (object): Applies to the room of the invitation. Defaults to `{"per_second": 0.3, "burst_count": 10.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `per_user` (object): Applies to the *receiver* of the invite, rather than the sender, meaning that a `rc_invite.per_user.burst_count` of 5 mandates that a single user cannot *receive* more than a burst of 5 invites at a time. Defaults to `{"per_second": 0.003, "burst_count": 5.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `per_issuer` (object): Applies to the *issuer* of the invite, meaning that a `rc_invite.per_issuer.burst_count` of 5 mandates that single user cannot *send* more than a burst of 5 invites at a time. Defaults to `{"per_second": 0.3, "burst_count": 10.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + Example configuration: ```yaml rc_invites: per_room: per_second: 0.5 - burst_count: 5 + burst_count: 5.0 per_user: per_second: 0.004 - burst_count: 3 + burst_count: 3.0 per_issuer: per_second: 0.5 - burst_count: 5 + burst_count: 5.0 ``` - --- ### `rc_third_party_invite` -This option ratelimits 3PID invites (i.e. invites sent to a third-party ID -such as an email address or a phone number) based on the account that's -sending the invite. Defaults to `per_second: 0.2`, `burst_count: 10`. +*(object)* This option ratelimits 3PID invites (i.e. invites sent to a third-party ID such as an email address or a phone number) based on the account that's sending the invite. -Example configuration: +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: ```yaml rc_third_party_invite: per_second: 0.2 - burst_count: 10 + burst_count: 10.0 ``` --- ### `rc_media_create` -This option ratelimits creation of MXC URIs via the `/_matrix/media/v1/create` -endpoint based on the account that's creating the media. Defaults to -`per_second: 10`, `burst_count: 50`. +*(object)* This option ratelimits creation of MXC URIs via the `/_matrix/media/v1/create` endpoint based on the account that's creating the media. -Example configuration: +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: ```yaml rc_media_create: - per_second: 10 - burst_count: 50 + per_second: 10.0 + burst_count: 50.0 ``` --- ### `rc_federation` -Defines limits on federation requests. +*(object)* Defines limits on federation requests. -The `rc_federation` configuration has the following sub-options: -* `window_size`: window size in milliseconds. Defaults to 1000. -* `sleep_limit`: number of federation requests from a single server in - a window before the server will delay processing the request. Defaults to 10. -* `sleep_delay`: duration in milliseconds to delay processing events - from remote servers by if they go over the sleep limit. Defaults to 500. -* `reject_limit`: maximum number of concurrent federation requests - allowed from a single server. Defaults to 50. -* `concurrent`: number of federation requests to concurrently process - from a single server. Defaults to 3. +This setting has the following sub-options: + +* `window_size` (integer): Window size in milliseconds. Defaults to `1000`. + +* `sleep_limit` (integer): Number of federation requests from a single server in a window before the server will delay processing the request. Defaults to `10`. + +* `sleep_delay` (integer): Duration in milliseconds to delay processing events from remote servers by if they go over the sleep limit. Defaults to `500`. + +* `reject_limit` (integer): Maximum number of concurrent federation requests allowed from a single server. Defaults to `50`. + +* `concurrent` (integer): Number of federation requests to concurrently process from a single server. Defaults to `3`. Example configuration: ```yaml @@ -1866,13 +1928,128 @@ rc_federation: concurrent: 5 ``` --- +### `rc_presence` + +*(object)* This option sets ratelimiting for presence. + +This setting has the following sub-options: + +* `per_user` (object): Sets rate limits on how often a specific users' presence updates are evaluated. Ratelimited presence updates sent via sync are ignored, and no error is returned to the client. This option also sets the rate limit for the [`PUT /_matrix/client/v3/presence/{userId}/status`] endpoint. + + [`PUT /_matrix/client/v3/presence/{userId}/status`]: + + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_presence: + per_user: + per_second: 0.1 + burst_count: 1.0 +``` + +Example configuration: +```yaml +rc_presence: + per_user: + per_second: 0.05 + burst_count: 1.0 +``` +--- +### `rc_delayed_event_mgmt` + +*(object)* Ratelimiting settings for delayed event management. + +This is a ratelimiting option that ratelimits attempts to restart, cancel, or view delayed events based on the sending client's account and device ID. + +Attempts to create or send delayed events are ratelimited not by this setting, but by `rc_message`. + +Setting this to a high value allows clients to make delayed event management requests often (such as repeatedly restarting a delayed event with a short timeout, or restarting several different delayed events all at once) without the risk of being ratelimited. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_delayed_event_mgmt: + per_second: 1.0 + burst_count: 5.0 +``` + +Example configuration: +```yaml +rc_delayed_event_mgmt: + per_second: 2.0 + burst_count: 20.0 +``` +--- +### `rc_reports` + +*(object)* Ratelimiting settings for reporting content. +This is a ratelimiting option that ratelimits reports made by users about content they see. +Setting this to a high value allows users to report content quickly, possibly in duplicate. This can result in higher database usage. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_reports: + per_user: + per_second: 1.0 + burst_count: 5.0 +``` + +Example configuration: +```yaml +rc_reports: + per_second: 2.0 + burst_count: 20.0 +``` +--- +### `rc_room_creation` + +*(object)* Sets rate limits for how often users are able to create rooms. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_room_creation: + per_user: + per_second: 0.016 + burst_count: 10.0 +``` + +Example configuration: +```yaml +rc_room_creation: + per_second: 1.0 + burst_count: 5.0 +``` +--- ### `federation_rr_transactions_per_room_per_second` -Sets outgoing federation transaction frequency for sending read-receipts, -per-room. +*(integer)* Sets outgoing federation transaction frequency for sending read-receipts, per-room. -If we end up trying to send out more read-receipts, they will get buffered up -into fewer transactions. Defaults to 50. +If we end up trying to send out more read-receipts, they will get buffered up into fewer transactions. + +Defaults to `50`. Example configuration: ```yaml @@ -1880,37 +2057,29 @@ federation_rr_transactions_per_room_per_second: 40 ``` --- ## Media Store + Config options related to Synapse's media store. --- ### `enable_authenticated_media` -When set to true, all subsequent media uploads will be marked as authenticated, and will not be available over legacy -unauthenticated media endpoints (`/_matrix/media/(r0|v3|v1)/download` and `/_matrix/media/(r0|v3|v1)/thumbnail`) - requests for authenticated media over these endpoints will result in a 404. All media, including authenticated media, will be available over the authenticated media endpoints `_matrix/client/v1/media/download` and `_matrix/client/v1/media/thumbnail`. Media uploaded prior to setting this option to true will still be available over the legacy endpoints. Note if the setting is switched to false -after enabling, media marked as authenticated will be available over legacy endpoints. Defaults to true (previously false). In a future release of Synapse, this option will be removed and become always-on. +*(boolean)* When set to true, all subsequent media uploads will be marked as authenticated, and will not be available over legacy unauthenticated media endpoints (`/_matrix/media/(r0|v3|v1)/download` and `/_matrix/media/(r0|v3|v1)/thumbnail`) – requests for authenticated media over these endpoints will result in a 404. All media, including authenticated media, will be available over the authenticated media endpoints `_matrix/client/v1/media/download` and `_matrix/client/v1/media/thumbnail`. Media uploaded prior to setting this option to true will still be available over the legacy endpoints. Note if the setting is switched to false after enabling, media marked as authenticated will be available over legacy endpoints. Defaults to true (previously false). In a future release of Synapse, this option will be removed and become always-on. -In all cases, authenticated requests to download media will succeed, but for unauthenticated requests, this -case-by-case breakdown describes whether media downloads are permitted: +In all cases, authenticated requests to download media will succeed, but for unauthenticated requests, this case-by-case breakdown describes whether media downloads are permitted: * `enable_authenticated_media = False`: * unauthenticated client or homeserver requesting local media: allowed - * unauthenticated client or homeserver requesting remote media: allowed as long as the media is in the cache, - or as long as the remote homeserver does not require authentication to retrieve the media + * unauthenticated client or homeserver requesting remote media: allowed as long as the media is in the cache, or as long as the remote homeserver does not require authentication to retrieve the media * `enable_authenticated_media = True`: - * unauthenticated client or homeserver requesting local media: - allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); - otherwise denied. - * unauthenticated client or homeserver requesting remote media: the same as for local media; - allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); - otherwise denied. + * unauthenticated client or homeserver requesting local media: allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); otherwise denied. + * unauthenticated client or homeserver requesting remote media: the same as for local media; allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); otherwise denied. -It is especially notable that media downloaded before this option existed (in older Synapse versions), or whilst this option was set to `False`, -will perpetually be available over the legacy, unauthenticated endpoint, even after this option is set to `True`. -This is for backwards compatibility with older clients and homeservers that do not yet support requesting authenticated media; -those older clients or homeservers will not be cut off from media they can already see. +It is especially notable that media downloaded before this option existed (in older Synapse versions), or whilst this option was set to `False`, will perpetually be available over the legacy, unauthenticated endpoint, even after this option is set to `True`. This is for backwards compatibility with older clients and homeservers that do not yet support requesting authenticated media; those older clients or homeservers will not be cut off from media they can already see. _Changed in Synapse 1.120:_ This option now defaults to `True` when not set, whereas before this version it defaulted to `False`. +Defaults to `true`. + Example configuration: ```yaml enable_authenticated_media: false @@ -1918,8 +2087,7 @@ enable_authenticated_media: false --- ### `enable_media_repo` -Enable the media store service in the Synapse master. Defaults to true. -Set to false if you are using a separate media store worker. +*(boolean)* Enable the media store service in the Synapse master. Set to false if you are using a separate media store worker. Defaults to `true`. Example configuration: ```yaml @@ -1928,18 +2096,16 @@ enable_media_repo: false --- ### `media_store_path` -Directory where uploaded images and attachments are stored. +*(string)* Directory where uploaded images and attachments are stored. Defaults to `"media_store"`. Example configuration: ```yaml -media_store_path: "DATADIR/media_store" +media_store_path: DATADIR/media_store ``` --- ### `max_pending_media_uploads` -How many *pending media uploads* can a given user have? A pending media upload -is a created MXC URI that (a) is not expired (the `unused_expires_at` timestamp -has not passed) and (b) the media has not yet been uploaded for. Defaults to 5. +*(integer)* How many *pending media uploads* can a given user have? A pending media upload is a created MXC URI that (a) is not expired (the `unused_expires_at` timestamp has not passed) and (b) the media has not yet been uploaded for. Defaults to `5`. Example configuration: ```yaml @@ -1948,51 +2114,80 @@ max_pending_media_uploads: 5 --- ### `unused_expiration_time` -How long to wait in milliseconds before expiring created media IDs. Defaults to -"24h" +*(duration)* How long to wait in milliseconds before expiring created media IDs. Defaults to `"24h"`. Example configuration: ```yaml -unused_expiration_time: "1h" +unused_expiration_time: 1h ``` --- ### `media_storage_providers` -Media storage providers allow media to be stored in different -locations. Defaults to none. Associated sub-options are: -* `module`: type of resource, e.g. `file_system`. -* `store_local`: whether to store newly uploaded local files -* `store_remote`: whether to store newly downloaded local files -* `store_synchronous`: whether to wait for successful storage for local uploads -* `config`: sets a path to the resource through the `directory` option +*(array)* Media storage providers allow media to be stored in different locations. Defaults to `[]`. + +Options for each entry include: + +* `module` (string): Type of resource, e.g. `file_system`. + +* `store_local` (boolean): Whether to store newly uploaded local files. + +* `store_remote` (boolean): Whether to store newly downloaded local files. + +* `store_synchronous` (boolean): Whether to wait for successful storage for local uploads. + +* `config` (object): Sets a path to the resource through the `directory` option. + + This setting has the following sub-options: + + * `directory` (string): Path to the resource. Example configuration: ```yaml media_storage_providers: - - module: file_system - store_local: false - store_remote: false - store_synchronous: false - config: - directory: /mnt/some/other/directory +- module: file_system + store_local: false + store_remote: false + store_synchronous: false + config: + directory: /mnt/some/other/directory ``` --- ### `max_upload_size` -The largest allowed upload size in bytes. +*(byte size)* The largest allowed upload size in bytes. -If you are using a reverse proxy you may also need to set this value in -your reverse proxy's config. Defaults to 50M. Notably Nginx has a small max body size by default. -See [here](../../reverse_proxy.md) for more on using a reverse proxy with Synapse. +If you are using a reverse proxy you may also need to set this value in your reverse proxy's config. Notably Nginx has a small max body size by default. See [here](../../reverse_proxy.md) for more on using a reverse proxy with Synapse. + +Defaults to `"50M"`. Example configuration: ```yaml max_upload_size: 60M ``` --- +### `media_upload_limits` + +*(array)* A list of media upload limits defining how much data a given user can upload in a given time period. +These limits are applied in addition to the `max_upload_size` limit above (which applies to individual uploads). + +An empty list means no limits are applied. + +These settings can be overridden using the `get_media_upload_limits_for_user` module API [callback](../../modules/media_repository_callbacks.md#get_media_upload_limits_for_user). + +Defaults to `[]`. + +Example configuration: +```yaml +media_upload_limits: +- time_period: 1h + max_size: 100M +- time_period: 1w + max_size: 500M +``` +--- ### `max_image_pixels` -Maximum number of pixels that will be thumbnailed. Defaults to 32M. +*(byte size)* Maximum number of pixels that will be thumbnailed. Defaults to `"32M"`. Example configuration: ```yaml @@ -2001,7 +2196,7 @@ max_image_pixels: 35M --- ### `remote_media_download_burst_count` -Remote media downloads are ratelimited using a [leaky bucket algorithm](https://en.wikipedia.org/wiki/Leaky_bucket), where a given "bucket" is keyed to the IP address of the requester when requesting remote media downloads. This configuration option sets the size of the bucket against which the size in bytes of downloads are penalized - if the bucket is full, ie a given number of bytes have already been downloaded, further downloads will be denied until the bucket drains. Defaults to 500MiB. See also `remote_media_download_per_second` which determines the rate at which the "bucket" is emptied and thus has available space to authorize new requests. +*(byte size)* Remote media downloads are ratelimited using a [leaky bucket algorithm](https://en.wikipedia.org/wiki/Leaky_bucket), where a given "bucket" is keyed to the IP address of the requester when requesting remote media downloads. This configuration option sets the size of the bucket against which the size in bytes of downloads are penalized – if the bucket is full, i.e. a given number of bytes have already been downloaded, further downloads will be denied until the bucket drains. See also `remote_media_download_per_second` which determines the rate at which the "bucket" is emptied and thus has available space to authorize new requests. Defaults to `"500MiB"`. Example configuration: ```yaml @@ -2010,7 +2205,7 @@ remote_media_download_burst_count: 200M --- ### `remote_media_download_per_second` -Works in conjunction with `remote_media_download_burst_count` to ratelimit remote media downloads - this configuration option determines the rate at which the "bucket" (see above) leaks in bytes per second. As requests are made to download remote media, the size of those requests in bytes is added to the bucket, and once the bucket has reached it's capacity, no more requests will be allowed until a number of bytes has "drained" from the bucket. This setting determines the rate at which bytes drain from the bucket, with the practical effect that the larger the number, the faster the bucket leaks, allowing for more bytes downloaded over a shorter period of time. Defaults to 87KiB per second. See also `remote_media_download_burst_count`. +*(byte size)* Works in conjunction with `remote_media_download_burst_count` to ratelimit remote media downloads – this configuration option determines the rate at which the "bucket" (see above) leaks in bytes per second. As requests are made to download remote media, the size of those requests in bytes is added to the bucket, and once the bucket has reached it's capacity, no more requests will be allowed until a number of bytes has "drained" from the bucket. This setting determines the rate at which bytes drain from the bucket, with the practical effect that the larger the number, the faster the bucket leaks, allowing for more bytes downloaded over a shorter period of time. Defaults to 87KiB per second. See also `remote_media_download_burst_count`. Defaults to `"87KiB"`. Example configuration: ```yaml @@ -2019,36 +2214,24 @@ remote_media_download_per_second: 40K --- ### `prevent_media_downloads_from` -A list of domains to never download media from. Media from these -domains that is already downloaded will not be deleted, but will be -inaccessible to users. This option does not affect admin APIs trying -to download/operate on media. +*(array)* A list of domains to never download media from. Media from these domains that is already downloaded will not be deleted, but will be inaccessible to users. This option does not affect admin APIs trying to download/operate on media. -This will not prevent the listed domains from accessing media themselves. -It simply prevents users on this server from downloading media originating -from the listed servers. +This will not prevent the listed domains from accessing media themselves. It simply prevents users on this server from downloading media originating from the listed servers. -This will have no effect on media originating from the local server. This only -affects media downloaded from other Matrix servers, to control URL previews see -[`url_preview_ip_range_blacklist`](#url_preview_ip_range_blacklist) or -[`url_preview_url_blacklist`](#url_preview_url_blacklist). +This will have no effect on media originating from the local server. This only affects media downloaded from other Matrix servers, to control URL previews see [`url_preview_ip_range_blacklist`](#url_preview_ip_range_blacklist) or [`url_preview_url_blacklist`](#url_preview_url_blacklist). -Defaults to an empty list (nothing blocked). +Defaults to `[]`. Example configuration: ```yaml prevent_media_downloads_from: - - evil.example.org - - evil2.example.org +- evil.example.org +- evil2.example.org ``` --- ### `dynamic_thumbnails` -Whether to generate new thumbnails on the fly to precisely match -the resolution requested by the client. If true then whenever -a new resolution is requested by the client the server will -generate a new thumbnail. If false the server will pick a thumbnail -from a precalculated list. Defaults to false. +*(boolean)* Whether to generate new thumbnails on the fly to precisely match the resolution requested by the client. If true then whenever a new resolution is requested by the client the server will generate a new thumbnail. If false the server will pick a thumbnail from a precalculated list. Defaults to `false`. Example configuration: ```yaml @@ -2057,68 +2240,62 @@ dynamic_thumbnails: true --- ### `thumbnail_sizes` -List of thumbnails to precalculate when an image is uploaded. Associated sub-options are: -* `width` -* `height` -* `method`: i.e. `crop`, `scale`, etc. +*(array)* List of thumbnails to precalculate when an image is uploaded. -Example configuration: +Options for each entry include: + +* `width` (integer): Width of the generated thumbnail. + +* `height` (integer): Height of the generated thumbnail. + +* `method` (string): Method to fit the thumbnail dimensions. Current options are `crop` and `scale`. + +Default configuration: ```yaml thumbnail_sizes: - - width: 32 - height: 32 - method: crop - - width: 96 - height: 96 - method: crop - - width: 320 - height: 240 - method: scale - - width: 640 - height: 480 - method: scale - - width: 800 - height: 600 - method: scale +- width: 32 + height: 32 + method: crop +- width: 96 + height: 96 + method: crop +- width: 320 + height: 240 + method: scale +- width: 640 + height: 480 + method: scale +- width: 800 + height: 600 + method: scale ``` --- ### `media_retention` -Controls whether local media and entries in the remote media cache -(media that is downloaded from other homeservers) should be removed -under certain conditions, typically for the purpose of saving space. +*(object)* Controls whether local media and entries in the remote media cache (media that is downloaded from other homeservers) should be removed under certain conditions, typically for the purpose of saving space. -Purging media files will be the carried out by the media worker -(that is, the worker that has the `enable_media_repo` homeserver config -option set to 'true'). This may be the main process. +Purging media files will be the carried out by the media worker (that is, the worker that has the `enable_media_repo` homeserver config option set to `true`). This may be the main process. -The `media_retention.local_media_lifetime` and -`media_retention.remote_media_lifetime` config options control whether -media will be purged if it has not been accessed in a given amount of -time. Note that media is 'accessed' when loaded in a room in a client, or -otherwise downloaded by a local or remote user. If the media has never -been accessed, the media's creation time is used instead. Both thumbnails -and the original media will be removed. If either of these options are unset, -then media of that type will not be purged. +The `media_retention.local_media_lifetime` and `media_retention.remote_media_lifetime` config options control whether media will be purged if it has not been accessed in a given amount of time. Note that media is "accessed" when loaded in a room in a client, or otherwise downloaded by a local or remote user. If the media has never been accessed, the media's creation time is used instead. Both thumbnails and the original media will be removed. If either of these options are unset, then media of that type will not be purged. -Local or cached remote media that has been -[quarantined](../../admin_api/media_admin_api.md#quarantining-media-in-a-room) -will not be deleted. Similarly, local media that has been marked as -[protected from quarantine](../../admin_api/media_admin_api.md#protecting-media-from-being-quarantined) -will not be deleted. +Local or cached remote media that has been [quarantined](../../admin_api/media_admin_api.md#quarantining-media-in-a-room) will not be deleted. Similarly, local media that has been marked as [protected from quarantine](../../admin_api/media_admin_api.md#protecting-media-from-being-quarantined) will not be deleted. + +This setting has the following sub-options: + +* `local_media_lifetime`: Duration without access to a local media resource after which it will be purged. If the media has never been accessed, the media's creation time is used instead. Both thumbnails and the original media will be removed. If unset or null, local media will not be purged. Defaults to `null`. + +* `remote_media_lifetime`: Duration without access to a remote media resource after which it will be purged. If the media has never been accessed, the media's creation time is used instead. Both thumbnails and the original media will be removed. If unset or null, remote media will not be purged. Defaults to `null`. Example configuration: ```yaml media_retention: - local_media_lifetime: 90d - remote_media_lifetime: 14d + local_media_lifetime: 90d + remote_media_lifetime: 14d ``` --- ### `url_preview_enabled` -This setting determines whether the preview URL API is enabled. -It is disabled by default. Set to true to enable. If enabled you must specify a -`url_preview_ip_range_blacklist` blacklist. +*(boolean)* This setting determines whether the preview URL API is enabled. Set to true to enable. If enabled you must specify a `url_preview_ip_range_blacklist` blacklist. Defaults to `false`. Example configuration: ```yaml @@ -2127,111 +2304,80 @@ url_preview_enabled: true --- ### `url_preview_ip_range_blacklist` -List of IP address CIDR ranges that the URL preview spider is denied -from accessing. There are no defaults: you must explicitly -specify a list for URL previewing to work. You should specify any -internal services in your network that you do not want synapse to try -to connect to, otherwise anyone in any Matrix room could cause your -synapse to issue arbitrary GET requests to your internal services, -causing serious security issues. +*(array|null)* List of IP address CIDR ranges that the URL preview spider is denied from accessing. There are no defaults: you must explicitly specify a list for URL previewing to work. You should specify any internal services in your network that you do not want synapse to try to connect to, otherwise anyone in any Matrix room could cause your synapse to issue arbitrary GET requests to your internal services, causing serious security issues. -(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly -listed here, since they correspond to unroutable addresses.) +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly listed here, since they correspond to unroutable addresses.) -This must be specified if `url_preview_enabled` is set. It is recommended that -you use the following example list as a starting point. +This must be specified if `url_preview_enabled` is set. It is recommended that you use the following example list as a starting point. Note: The value is ignored when an HTTP proxy is in use. +Defaults to `null`. + Example configuration: ```yaml url_preview_ip_range_blacklist: - - '127.0.0.0/8' - - '10.0.0.0/8' - - '172.16.0.0/12' - - '192.168.0.0/16' - - '100.64.0.0/10' - - '192.0.0.0/24' - - '169.254.0.0/16' - - '192.88.99.0/24' - - '198.18.0.0/15' - - '192.0.2.0/24' - - '198.51.100.0/24' - - '203.0.113.0/24' - - '224.0.0.0/4' - - '::1/128' - - 'fe80::/10' - - 'fc00::/7' - - '2001:db8::/32' - - 'ff00::/8' - - 'fec0::/10' +- 127.0.0.0/8 +- 10.0.0.0/8 +- 172.16.0.0/12 +- 192.168.0.0/16 +- 100.64.0.0/10 +- 192.0.0.0/24 +- 169.254.0.0/16 +- 192.88.99.0/24 +- 198.18.0.0/15 +- 192.0.2.0/24 +- 198.51.100.0/24 +- 203.0.113.0/24 +- 224.0.0.0/4 +- ::1/128 +- fe80::/10 +- fc00::/7 +- 2001:db8::/32 +- ff00::/8 +- fec0::/10 ``` --- ### `url_preview_ip_range_whitelist` -This option sets a list of IP address CIDR ranges that the URL preview spider is allowed -to access even if they are specified in `url_preview_ip_range_blacklist`. -This is useful for specifying exceptions to wide-ranging blacklisted -target IP ranges - e.g. for enabling URL previews for a specific private -website only visible in your network. Defaults to none. +*(array)* This option sets a list of IP address CIDR ranges that the URL preview spider is allowed to access even if they are specified in `url_preview_ip_range_blacklist`. This is useful for specifying exceptions to wide-ranging blacklisted target IP ranges – e.g. for enabling URL previews for a specific private website only visible in your network. Defaults to `[]`. Example configuration: ```yaml url_preview_ip_range_whitelist: - - '192.168.1.1' +- 192.168.1.1 ``` --- ### `url_preview_url_blacklist` -Optional list of URL matches that the URL preview spider is denied from -accessing. This is a usability feature, not a security one. You should use -`url_preview_ip_range_blacklist` in preference to this, otherwise someone could -define a public DNS entry that points to a private IP address and circumvent -the blacklist. Applications that perform redirects or serve different content -when detecting that Synapse is accessing them can also bypass the blacklist. -This is more useful if you know there is an entire shape of URL that you know -that you do not want Synapse to preview. +*(array)* Optional list of URL matches that the URL preview spider is denied from accessing. This is a usability feature, not a security one. You should use `url_preview_ip_range_blacklist` in preference to this, otherwise someone could define a public DNS entry that points to a private IP address and circumvent the blacklist. Applications that perform redirects or serve different content when detecting that Synapse is accessing them can also bypass the blacklist. This is more useful if you know there is an entire shape of URL that you know that you do not want Synapse to preview. -Each list entry is a dictionary of url component attributes as returned -by urlparse.urlsplit as applied to the absolute form of the URL. See -[here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) for more -information. Some examples are: +Each list entry is a dictionary of url component attributes as returned by urlparse.urlsplit as applied to the absolute form of the URL. See [here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) for more information. Some examples are: * `username` * `netloc` * `scheme` * `path` -The values of the dictionary are treated as a filename match pattern -applied to that component of URLs, unless they start with a ^ in which -case they are treated as a regular expression match. If all the -specified component matches for a given list item succeed, the URL is -blacklisted. +The values of the dictionary are treated as a filename match pattern applied to that component of URLs, unless they start with a ^ in which case they are treated as a regular expression match. If all the specified component matches for a given list item succeed, the URL is blacklisted. + +Defaults to `[]`. Example configuration: ```yaml url_preview_url_blacklist: - # blacklist any URL with a username in its URI - - username: '*' - - # blacklist all *.google.com URLs - - netloc: 'google.com' - - netloc: '*.google.com' - - # blacklist all plain HTTP URLs - - scheme: 'http' - - # blacklist http(s)://www.acme.com/foo - - netloc: 'www.acme.com' - path: '/foo' - - # blacklist any URL with a literal IPv4 address - - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' +- username: '*' +- netloc: google.com +- netloc: '*.google.com' +- scheme: http +- netloc: www.acme.com + path: /foo +- netloc: ^[0-9]+.[0-9]+.[0-9]+.[0-9]+$ ``` --- ### `max_spider_size` -The largest allowed URL preview spidering size in bytes. Defaults to 10M. +*(byte size)* The largest allowed URL preview spidering size in bytes. Defaults to `"10M"`. Example configuration: ```yaml @@ -2240,43 +2386,43 @@ max_spider_size: 8M --- ### `url_preview_accept_language` -A list of values for the Accept-Language HTTP header used when -downloading webpages during URL preview generation. This allows -Synapse to specify the preferred languages that URL previews should -be in when communicating with remote servers. +*(array)* A list of values for the Accept-Language HTTP header used when downloading webpages during URL preview generation. This allows Synapse to specify the preferred languages that URL previews should be in when communicating with remote servers. -Each value is a IETF language tag; a 2-3 letter identifier for a -language, optionally followed by subtags separated by '-', specifying -a country or region variant. +Each value is a IETF language tag; a 2-3 letter identifier for a language, optionally followed by subtags separated by `-`, specifying a country or region variant. -Multiple values can be provided, and a weight can be added to each by -using quality value syntax (;q=). '*' translates to any language. +Multiple values can be provided, and a weight can be added to each by using quality value syntax (;q=). `*` translates to any language. -Defaults to "en". +Default configuration: +```yaml +url_preview_accept_language: +- en +``` Example configuration: ```yaml - url_preview_accept_language: - - 'en-UK' - - 'en-US;q=0.9' - - 'fr;q=0.8' - - '*;q=0.7' +url_preview_accept_language: +- en-UK +- en-US;q=0.9 +- fr;q=0.8 +- '*;q=0.7' ``` --- ### `oembed` -oEmbed allows for easier embedding content from a website. It can be -used for generating URLs previews of services which support it. A default list of oEmbed providers -is included with Synapse. Set `disable_default_providers` to true to disable using -these default oEmbed URLs. Use `additional_providers` to specify additional files with oEmbed configuration (each -should be in the form of providers.json). By default this list is empty. +*(object)* oEmbed allows for easier embedding content from a website. It can be used for generating URLs previews of services which support it. A default list of oEmbed providers is included with Synapse. + +This setting has the following sub-options: + +* `disable_default_providers` (boolean): Do not use Synapse's default list of oEmbed providers. Defaults to `false`. + +* `additional_providers` (array): Additional files with oEmbed configuration (each should be in the form of providers.json). Defaults to `[]`. Example configuration: ```yaml oembed: disable_default_providers: true additional_providers: - - oembed/my_providers.json + - oembed/my_providers.json ``` --- ## Captcha @@ -2286,33 +2432,60 @@ See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha. --- ### `recaptcha_public_key` -This homeserver's ReCAPTCHA public key. Must be specified if -[`enable_registration_captcha`](#enable_registration_captcha) is enabled. +*(string|null)* This homeserver's ReCAPTCHA public key. Must be specified if [`enable_registration_captcha`](#enable_registration_captcha) is enabled. Defaults to `null`. Example configuration: ```yaml -recaptcha_public_key: "YOUR_PUBLIC_KEY" +recaptcha_public_key: YOUR_PUBLIC_KEY +``` +--- +### `recaptcha_public_key_path` + +*(string|null)* An alternative to [`recaptcha_public_key`](#recaptcha_public_key): allows the public key to be specified in an external file. + +The file should be a plain text file, containing only the public key. Synapse reads the public key from the given file once at startup. + +_Added in Synapse 1.135.0._ + +Defaults to `null`. + +Example configuration: +```yaml +recaptcha_public_key_path: /path/to/key/file ``` --- ### `recaptcha_private_key` -This homeserver's ReCAPTCHA private key. Must be specified if -[`enable_registration_captcha`](#enable_registration_captcha) is -enabled. +*(string|null)* This homeserver's ReCAPTCHA private key. Must be specified if [`enable_registration_captcha`](#enable_registration_captcha) is enabled. Defaults to `null`. Example configuration: ```yaml -recaptcha_private_key: "YOUR_PRIVATE_KEY" +recaptcha_private_key: YOUR_PRIVATE_KEY +``` +--- +### `recaptcha_private_key_path` + +*(string|null)* An alternative to [`recaptcha_private_key`](#recaptcha_private_key): allows the private key to be specified in an external file. + +The file should be a plain text file, containing only the private key. Synapse reads the private key from the given file once at startup. + +_Added in Synapse 1.135.0._ + +Defaults to `null`. + +Example configuration: +```yaml +recaptcha_private_key_path: /path/to/key/file ``` --- ### `enable_registration_captcha` -Set to `true` to require users to complete a CAPTCHA test when registering an account. -Requires a valid ReCaptcha public/private key. -Defaults to `false`. +*(boolean)* Set to `true` to require users to complete a CAPTCHA test when registering an account. Requires a valid ReCaptcha public/private key. Note that [`enable_registration`](#enable_registration) must also be set to allow account registration. +Defaults to `false`. + Example configuration: ```yaml enable_registration_captcha: true @@ -2320,65 +2493,73 @@ enable_registration_captcha: true --- ### `recaptcha_siteverify_api` -The API endpoint to use for verifying `m.login.recaptcha` responses. -Defaults to `https://www.recaptcha.net/recaptcha/api/siteverify`. +*(string)* The API endpoint to use for verifying `m.login.recaptcha` responses. Defaults to `"https://www.recaptcha.net/recaptcha/api/siteverify"`. Example configuration: ```yaml -recaptcha_siteverify_api: "https://my.recaptcha.site" +recaptcha_siteverify_api: https://my.recaptcha.site ``` --- ## TURN + Options related to adding a TURN server to Synapse. --- ### `turn_uris` -The public URIs of the TURN server to give to clients. +*(array)* The public URIs of the TURN server to give to clients. Defaults to `[]`. Example configuration: ```yaml -turn_uris: [turn:example.org] +turn_uris: +- turn:example.org ``` --- ### `turn_shared_secret` -The shared secret used to compute passwords for the TURN server. +*(string|null)* The shared secret used to compute passwords for the TURN server. Defaults to `null`. Example configuration: ```yaml -turn_shared_secret: "YOUR_SHARED_SECRET" +turn_shared_secret: YOUR_SHARED_SECRET ``` --- ### `turn_shared_secret_path` -An alternative to [`turn_shared_secret`](#turn_shared_secret): -allows the shared secret to be specified in an external file. +*(string|null)* An alternative to [`turn_shared_secret`](#turn_shared_secret): allows the shared secret to be specified in an external file. -The file should be a plain text file, containing only the shared secret. -Synapse reads the shared secret from the given file once at startup. +The file should be a plain text file, containing only the shared secret. Synapse reads the shared secret from the given file once at startup. + +_Added in Synapse 1.116.0._ + +Defaults to `null`. Example configuration: ```yaml turn_shared_secret_path: /path/to/secrets/file ``` - -_Added in Synapse 1.116.0._ - --- -### `turn_username` and `turn_password` +### `turn_username` -The Username and password if the TURN server needs them and does not use a token. +*(string|null)* TURN server username if not using a token. Defaults to `null`. Example configuration: ```yaml -turn_username: "TURNSERVER_USERNAME" -turn_password: "TURNSERVER_PASSWORD" +turn_username: TURNSERVER_USERNAME +``` +--- +### `turn_password` + +*(string|null)* TURN server password if not using a token. Defaults to `null`. + +Example configuration: +```yaml +turn_password: TURNSERVER_PASSWORD ``` --- ### `turn_user_lifetime` -How long generated TURN credentials last. Defaults to 1h. +*(duration)* How long generated TURN credentials last. Defaults to `"1h"`. Example configuration: ```yaml @@ -2387,37 +2568,33 @@ turn_user_lifetime: 2h --- ### `turn_allow_guests` -Whether guests should be allowed to use the TURN server. This defaults to true, otherwise -VoIP will be unreliable for guests. However, it does introduce a slight security risk as -it allows users to connect to arbitrary endpoints without having first signed up for a valid account (e.g. by passing a CAPTCHA). +*(boolean)* Whether guests should be allowed to use the TURN server. If false, VoIP will be unreliable for guests. However, it does introduce a slight security risk as it allows users to connect to arbitrary endpoints without having first signed up for a valid account (e.g. by passing a CAPTCHA). Defaults to `true`. Example configuration: ```yaml turn_allow_guests: false ``` --- -## Registration ## +## Registration Registration can be rate-limited using the parameters in the [Ratelimiting](#ratelimiting) section of this manual. --- ### `enable_registration` -Enable registration for new users. Defaults to `false`. +*(boolean)* Enable registration for new users. -It is highly recommended that if you enable registration, you set one or more -or the following options, to avoid abuse of your server by "bots": +It is highly recommended that if you enable registration, you set one or more or the following options, to avoid abuse of your server by "bots": - * [`enable_registration_captcha`](#enable_registration_captcha) - * [`registrations_require_3pid`](#registrations_require_3pid) - * [`registration_requires_token`](#registration_requires_token) +* [`enable_registration_captcha`](#enable_registration_captcha) +* [`registrations_require_3pid`](#registrations_require_3pid) +* [`registration_requires_token`](#registration_requires_token) -(In order to enable registration without any verification, you must also set -[`enable_registration_without_verification`](#enable_registration_without_verification).) +(In order to enable registration without any verification, you must also set [`enable_registration_without_verification`](#enable_registration_without_verification).) -Note that even if this setting is disabled, new accounts can still be created -via the admin API if -[`registration_shared_secret`](#registration_shared_secret) is set. +Note that even if this setting is disabled, new accounts can still be created via the admin API if [`registration_shared_secret`](#registration_shared_secret) is set. + +Defaults to `false`. Example configuration: ```yaml @@ -2426,9 +2603,7 @@ enable_registration: true --- ### `enable_registration_without_verification` -Enable registration without email or captcha verification. Note: this option is *not* recommended, -as registration without verification is a known vector for spam and abuse. Defaults to `false`. Has no effect -unless [`enable_registration`](#enable_registration) is also enabled. +*(boolean)* Enable registration without email or captcha verification. Note: this option is *not* recommended, as registration without verification is a known vector for spam and abuse. Has no effect unless [`enable_registration`](#enable_registration) is also enabled. Defaults to `false`. Example configuration: ```yaml @@ -2437,21 +2612,22 @@ enable_registration_without_verification: true --- ### `registrations_require_3pid` -If this is set, users must provide all of the specified types of [3PID](https://spec.matrix.org/latest/appendices/#3pid-types) when registering an account. +*(array)* If this is set, users must provide all of the specified types of [3PID](https://spec.matrix.org/latest/appendices/#3pid-types) when registering an account. Note that [`enable_registration`](#enable_registration) must also be set to allow account registration. +Defaults to `[]`. + Example configuration: ```yaml registrations_require_3pid: - - email - - msisdn +- email +- msisdn ``` --- ### `disable_msisdn_registration` -Explicitly disable asking for MSISDNs from the registration -flow (overrides `registrations_require_3pid` if MSISDNs are set as required). +*(boolean)* Explicitly disable asking for MSISDNs from the registration flow (overrides `registrations_require_3pid` if MSISDNs are set as required). Defaults to `false`. Example configuration: ```yaml @@ -2460,26 +2636,32 @@ disable_msisdn_registration: true --- ### `allowed_local_3pids` -Mandate that users are only allowed to associate certain formats of -3PIDs with accounts on this server, as specified by the `medium` and `pattern` sub-options. -`pattern` is a [Perl-like regular expression](https://docs.python.org/3/library/re.html#module-re). +*(array|null)* Mandate that users are only allowed to associate certain formats of 3PIDs with accounts on this server, as specified by the `medium` and `pattern` sub-options. `pattern` is a [Perl-like regular expression](https://docs.python.org/3/library/re.html#module-re). More information about 3PIDs, allowed `medium` types and their `address` syntax can be found [in the Matrix spec](https://spec.matrix.org/latest/appendices/#3pid-types). +Defaults to `null`. + +Options for each entry include: + +* `medium` (string): The medium for which to allow 3PID association. + +* `pattern` (string): A [Perl-like regular expression](https://docs.python.org/3/library/re.html#module-re) allowing association of a 3PID to a local account if it matches the given format. + Example configuration: ```yaml allowed_local_3pids: - - medium: email - pattern: '^[^@]+@matrix\.org$' - - medium: email - pattern: '^[^@]+@vector\.im$' - - medium: msisdn - pattern: '^44\d{10}$' +- medium: email + pattern: ^[^@]+@matrix\.org$ +- medium: email + pattern: ^[^@]+@vector\.im$ +- medium: msisdn + pattern: ^44\d{10}$ ``` --- ### `enable_3pid_lookup` -Enable 3PIDs lookup requests to identity servers from this server. Defaults to true. +*(boolean)* Enable 3PIDs lookup requests to identity servers from this server. Defaults to `true`. Example configuration: ```yaml @@ -2488,14 +2670,12 @@ enable_3pid_lookup: false --- ### `registration_requires_token` -Require users to submit a token during registration. -Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md). -Disabling this option will not delete any tokens previously generated. -Defaults to `false`. Set to `true` to enable. - +*(boolean)* Require users to submit a token during registration. Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md). Disabling this option will not delete any tokens previously generated. Note that [`enable_registration`](#enable_registration) must also be set to allow account registration. +Defaults to `false`. + Example configuration: ```yaml registration_requires_token: true @@ -2503,47 +2683,44 @@ registration_requires_token: true --- ### `registration_shared_secret` -If set, allows registration of standard or admin accounts by anyone who has the -shared secret, even if [`enable_registration`](#enable_registration) is not -set. +*(string|null)* If set, allows registration of standard or admin accounts by anyone who has the shared secret, even if [`enable_registration`](#enable_registration) is not set. -This is primarily intended for use with the `register_new_matrix_user` script -(see [Registering a user](../../setup/installation.md#registering-a-user)); -however, the interface is [documented](../../admin_api/register_api.html). +This is primarily intended for use with the `register_new_matrix_user` script (see [Registering a user](../../setup/installation.md#registering-a-user)); however, the interface is [documented](../../admin_api/register_api.html). + +Replacing an existing `registration_shared_secret` with a new one requires users of the [Shared-Secret Registration API](../../admin_api/register_api.html) to start using the new secret for requesting any further one-time nonces. + +> ⚠️ **Warning** – The additional consequences of replacing [`macaroon_secret_key`](#macaroon_secret_key) will apply in case it delegates to `registration_shared_secret`. See also [`registration_shared_secret_path`](#registration_shared_secret_path). +Defaults to `null`. + Example configuration: ```yaml registration_shared_secret: ``` - --- ### `registration_shared_secret_path` -An alternative to [`registration_shared_secret`](#registration_shared_secret): -allows the shared secret to be specified in an external file. +*(string|null)* An alternative to [`registration_shared_secret`](#registration_shared_secret): allows the shared secret to be specified in an external file. The file should be a plain text file, containing only the shared secret. -If this file does not exist, Synapse will create a new shared -secret on startup and store it in this file. +If this file does not exist, Synapse will create a new shared secret on startup and store it in this file. + +_Added in Synapse 1.67.0._ + +Defaults to `null`. Example configuration: ```yaml registration_shared_secret_path: /path/to/secrets/file ``` - -_Added in Synapse 1.67.0._ - --- ### `bcrypt_rounds` -Set the number of bcrypt rounds used to generate password hash. -Larger numbers increase the work factor needed to generate the hash. -The default number is 12 (which equates to 2^12 rounds). -N.B. that increasing this will exponentially increase the time required -to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. +*(integer)* Set the number of bcrypt rounds used to generate password hash. Larger numbers increase the work factor needed to generate the hash. The default number is 12 (which equates to 2^12 rounds). N.B. that increasing this will exponentially increase the time required to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. Defaults to `12`. + Example configuration: ```yaml bcrypt_rounds: 14 @@ -2551,9 +2728,7 @@ bcrypt_rounds: 14 --- ### `allow_guest_access` -Allows users to register as guests without a password/email/etc, and -participate in rooms hosted on this server which have been made -accessible to anonymous users. Defaults to false. +*(boolean)* Allows users to register as guests without a password/email/etc, and participate in rooms hosted on this server which have been made accessible to anonymous users. Defaults to `false`. Example configuration: ```yaml @@ -2562,11 +2737,11 @@ allow_guest_access: true --- ### `default_identity_server` -The identity server which we suggest that clients should use when users log -in on this server. +*(string|null)* The identity server which we suggest that clients should use when users log in on this server. -(By default, no suggestion is made, so it is left up to the client. -This setting is ignored unless `public_baseurl` is also explicitly set.) +(By default, no suggestion is made, so it is left up to the client. This setting is ignored unless `public_baseurl` is also explicitly set.) + +Defaults to `null`. Example configuration: ```yaml @@ -2575,39 +2750,37 @@ default_identity_server: https://matrix.org --- ### `account_threepid_delegates` -Delegate verification of phone numbers to an identity server. +*(object)* Delegate verification of phone numbers to an identity server. -When a user wishes to add a phone number to their account, we need to verify that they -actually own that phone number, which requires sending them a text message (SMS). -Currently Synapse does not support sending those texts itself and instead delegates the -task to an identity server. The base URI for the identity server to be used is -specified by the `account_threepid_delegates.msisdn` option. +When a user wishes to add a phone number to their account, we need to verify that they actually own that phone number, which requires sending them a text message (SMS). Currently Synapse does not support sending those texts itself and instead delegates the task to an identity server. The base URI for the identity server to be used is specified by the `account_threepid_delegates.msisdn` option. -If this is left unspecified, Synapse will not allow users to add phone numbers to -their account. +If this is left unspecified, Synapse will not allow users to add phone numbers to their account. -(Servers handling the these requests must answer the `/requestToken` endpoints defined -by the Matrix Identity Service API -[specification](https://matrix.org/docs/spec/identity_service/latest).) +(Servers handling the these requests must answer the `/requestToken` endpoints defined by the Matrix Identity Service API [specification](https://matrix.org/docs/spec/identity_service/latest).) *Deprecated in Synapse 1.64.0*: The `email` option is deprecated. -*Removed in Synapse 1.66.0*: The `email` option has been removed. -If present, Synapse will report a configuration error on startup. +*Removed in Synapse 1.66.0*: The `email` option has been removed. If present, Synapse will report a configuration error on startup. + +Defaults to `{}`. + +This setting has the following sub-options: + +* `msisdn` (string|null): Identity server base URI for MSISDN (phone numbers). See above. Example configuration: ```yaml account_threepid_delegates: - msisdn: http://localhost:8090 # Delegate SMS sending to this local process + msisdn: http://localhost:8090 ``` --- ### `enable_set_displayname` -Whether users are allowed to change their displayname after it has -been initially set. Useful when provisioning users based on the -contents of a third-party directory. +*(boolean)* Whether users are allowed to change their displayname after it has been initially set. Useful when provisioning users based on the contents of a third-party directory. -Does not apply to server administrators. Defaults to true. +Does not apply to server administrators. + +Defaults to `true`. Example configuration: ```yaml @@ -2616,11 +2789,11 @@ enable_set_displayname: false --- ### `enable_set_avatar_url` -Whether users are allowed to change their avatar after it has been -initially set. Useful when provisioning users based on the contents -of a third-party directory. +*(boolean)* Whether users are allowed to change their avatar after it has been initially set. Useful when provisioning users based on the contents of a third-party directory. -Does not apply to server administrators. Defaults to true. +Does not apply to server administrators. + +Defaults to `true`. Example configuration: ```yaml @@ -2629,10 +2802,7 @@ enable_set_avatar_url: false --- ### `enable_3pid_changes` -Whether users can change the third-party IDs associated with their accounts -(email address and msisdn). - -Defaults to true. +*(boolean)* Whether users can change the third-party IDs associated with their accounts (email address and msisdn). Defaults to `true`. Example configuration: ```yaml @@ -2641,39 +2811,30 @@ enable_3pid_changes: false --- ### `auto_join_rooms` -Users who register on this homeserver will automatically be joined -to the rooms listed under this option. +*(array)* Users who register on this homeserver will automatically be joined to the rooms listed under this option. -By default, any room aliases included in this list will be created -as a publicly joinable room when the first user registers for the -homeserver. If the room already exists, make certain it is a publicly joinable -room, i.e. the join rule of the room must be set to 'public'. You can find more options -relating to auto-joining rooms below. +By default, any room aliases included in this list will be created as a publicly joinable room when the first user registers for the homeserver. If the room already exists, make certain it is a publicly joinable room, i.e. the join rule of the room must be set to `public`. You can find more options relating to auto-joining rooms below. -As Spaces are just rooms under the hood, Space aliases may also be -used. +As Spaces are just rooms under the hood, Space aliases may also be used. + +Defaults to `[]`. Example configuration: ```yaml auto_join_rooms: - - "#exampleroom:example.com" - - "#anotherexampleroom:example.com" +- '#exampleroom:example.com' +- '#anotherexampleroom:example.com' ``` --- ### `autocreate_auto_join_rooms` -Where `auto_join_rooms` are specified, setting this flag ensures that -the rooms exist by creating them when the first user on the -homeserver registers. This option will not create Spaces. +*(boolean)* Where `auto_join_rooms` are specified, setting this flag ensures that the rooms exist by creating them when the first user on the homeserver registers. This option will not create Spaces. -By default the auto-created rooms are publicly joinable from any federated -server. Use the `autocreate_auto_join_rooms_federated` and -`autocreate_auto_join_room_preset` settings to customise this behaviour. +By default the auto-created rooms are publicly joinable from any federated server. Use the `autocreate_auto_join_rooms_federated` and `autocreate_auto_join_room_preset` settings to customise this behaviour. -Setting to false means that if the rooms are not manually created, -users cannot be auto-joined since they do not exist. +Setting to false means that if the rooms are not manually created, users cannot be auto-joined since they do not exist. -Defaults to true. +Defaults to `true`. Example configuration: ```yaml @@ -2682,15 +2843,13 @@ autocreate_auto_join_rooms: false --- ### `autocreate_auto_join_rooms_federated` -Whether the rooms listed in `auto_join_rooms` that are auto-created are available -via federation. Only has an effect if `autocreate_auto_join_rooms` is true. +*(boolean)* Whether the rooms listed in `auto_join_rooms` that are auto-created are available via federation. Only has an effect if `autocreate_auto_join_rooms` is true. -Note that whether a room is federated cannot be modified after -creation. +Note that whether a room is federated cannot be modified after creation. -Defaults to true: the room will be joinable from other servers. -Set to false to prevent users from other homeservers from -joining these rooms. +If true, the room will be joinable from other servers. If false, users from other homeservers are prevented from joining these rooms. + +Defaults to `true`. Example configuration: ```yaml @@ -2699,25 +2858,18 @@ autocreate_auto_join_rooms_federated: false --- ### `autocreate_auto_join_room_preset` -The room preset to use when auto-creating one of `auto_join_rooms`. Only has an -effect if `autocreate_auto_join_rooms` is true. +*(string)* The room preset to use when auto-creating one of `auto_join_rooms`. Only has an effect if `autocreate_auto_join_rooms` is true. Possible values for this option are: -* "public_chat": the room is joinable by anyone, including - federated servers if `autocreate_auto_join_rooms_federated` is true (the default). +* "public_chat": the room is joinable by anyone, including federated servers if `autocreate_auto_join_rooms_federated` is true (the default). * "private_chat": an invitation is required to join these rooms. -* "trusted_private_chat": an invitation is required to join this room and the invitee is - assigned a power level of 100 upon joining the room. +* "trusted_private_chat": an invitation is required to join this room and the invitee is assigned a power level of 100 upon joining the room. -Each preset will set up a room in the same manner as if it were provided as the `preset` parameter when -calling the -[`POST /_matrix/client/v3/createRoom`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom) -Client-Server API endpoint. +Each preset will set up a room in the same manner as if it were provided as the `preset` parameter when calling the [`POST /_matrix/client/v3/createRoom`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom) Client-Server API endpoint. -If a value of "private_chat" or "trusted_private_chat" is used then -`auto_join_mxid_localpart` must also be configured. +If a value of "private_chat" or "trusted_private_chat" is used then `auto_join_mxid_localpart` must also be configured. -Defaults to "public_chat". +Defaults to `"public_chat"`. Example configuration: ```yaml @@ -2726,22 +2878,17 @@ autocreate_auto_join_room_preset: private_chat --- ### `auto_join_mxid_localpart` -The local part of the user id which is used to create `auto_join_rooms` if -`autocreate_auto_join_rooms` is true. If this is not provided then the -initial user account that registers will be used to create the rooms. +*(string|null)* The local part of the user id which is used to create `auto_join_rooms` if `autocreate_auto_join_rooms` is true. If this is not provided then the initial user account that registers will be used to create the rooms. -The user id is also used to invite new users to any auto-join rooms which -are set to invite-only. +The user id is also used to invite new users to any auto-join rooms which are set to invite-only. -It *must* be configured if `autocreate_auto_join_room_preset` is set to -"private_chat" or "trusted_private_chat". +It *must* be configured if `autocreate_auto_join_room_preset` is set to "private_chat" or "trusted_private_chat". -Note that this must be specified in order for new users to be correctly -invited to any auto-join rooms which have been set to invite-only (either -at the time of creation or subsequently). +Note that this must be specified in order for new users to be correctly invited to any auto-join rooms which have been set to invite-only (either at the time of creation or subsequently). -Note that, if the room already exists, this user must be joined and -have the appropriate permissions to invite new members. +Note that, if the room already exists, this user must be joined and have the appropriate permissions to invite new members. + +Defaults to `null`. Example configuration: ```yaml @@ -2750,10 +2897,7 @@ auto_join_mxid_localpart: system --- ### `auto_join_rooms_for_guests` -When `auto_join_rooms` is specified, setting this flag to false prevents -guest accounts from being automatically joined to the rooms. - -Defaults to true. +*(boolean)* When `auto_join_rooms` is specified, setting this flag to false prevents guest accounts from being automatically joined to the rooms. Defaults to `true`. Example configuration: ```yaml @@ -2762,31 +2906,36 @@ auto_join_rooms_for_guests: false --- ### `inhibit_user_in_use_error` -Whether to inhibit errors raised when registering a new account if the user ID -already exists. If turned on, requests to `/register/available` will always -show a user ID as available, and Synapse won't raise an error when starting -a registration with a user ID that already exists. However, Synapse will still -raise an error if the registration completes and the username conflicts. - -Defaults to false. +*(boolean)* Whether to inhibit errors raised when registering a new account if the user ID already exists. If turned on, requests to `/register/available` will always show a user ID as available, and Synapse won't raise an error when starting a registration with a user ID that already exists. However, Synapse will still raise an error if the registration completes and the username conflicts. Defaults to `false`. Example configuration: ```yaml inhibit_user_in_use_error: true ``` --- +### `allow_underscore_prefixed_registration` + +*(boolean)* Whether users are allowed to register with a underscore-prefixed localpart. By default, AppServices use prefixes like `_example` to namespace their associated ghost users. If turned on, this may result in clashes or confusion. Useful when provisioning users from an external identity provider. Defaults to `false`. + +Example configuration: +```yaml +allow_underscore_prefixed_registration: true +``` +--- ## User session management + +Config options related to user session management. + --- ### `session_lifetime` -Time that a user's session remains valid for, after they log in. +*(duration)* Time that a user's session remains valid for, after they log in. Note that this is not currently compatible with guest logins. -Note also that this is calculated at login time: changes are not applied retrospectively to users who have already -logged in. +Note also that this is calculated at login time: changes are not applied retrospectively to users who have already logged in. -By default, this is infinite. +Defaults to `"infinity"`. Example configuration: ```yaml @@ -2795,16 +2944,15 @@ session_lifetime: 24h --- ### `refreshable_access_token_lifetime` -Time that an access token remains valid for, if the session is using refresh tokens. +*(duration)* Time that an access token remains valid for, if the session is using refresh tokens. For more information about refresh tokens, please see the [manual](user_authentication/refresh_tokens.md). Note that this only applies to clients which advertise support for refresh tokens. -Note also that this is calculated at login time and refresh time: changes are not applied to -existing sessions until they are refreshed. +Note also that this is calculated at login time and refresh time: changes are not applied to existing sessions until they are refreshed. -By default, this is 5 minutes. +Defaults to `"5m"`. Example configuration: ```yaml @@ -2813,15 +2961,11 @@ refreshable_access_token_lifetime: 10m --- ### `refresh_token_lifetime` -Time that a refresh token remains valid for (provided that it is not -exchanged for another one first). -This option can be used to automatically log-out inactive sessions. -Please see the manual for more information. +*(duration)* Time that a refresh token remains valid for (provided that it is not exchanged for another one first). This option can be used to automatically log-out inactive sessions. Please see the manual for more information. -Note also that this is calculated at login time and refresh time: -changes are not applied to existing sessions until they are refreshed. +Note also that this is calculated at login time and refresh time: changes are not applied to existing sessions until they are refreshed. -By default, this is infinite. +Defaults to `"infinity"`. Example configuration: ```yaml @@ -2830,17 +2974,13 @@ refresh_token_lifetime: 24h --- ### `nonrefreshable_access_token_lifetime` -Time that an access token remains valid for, if the session is NOT -using refresh tokens. +*(duration)* Time that an access token remains valid for, if the session is NOT using refresh tokens. -Please note that not all clients support refresh tokens, so setting -this to a short value may be inconvenient for some users who will -then be logged out frequently. +Please note that not all clients support refresh tokens, so setting this to a short value may be inconvenient for some users who will then be logged out frequently. -Note also that this is calculated at login time: changes are not applied -retrospectively to existing sessions for users that have already logged in. +Note also that this is calculated at login time: changes are not applied retrospectively to existing sessions for users that have already logged in. -By default, this is infinite. +Defaults to `"infinity"`. Example configuration: ```yaml @@ -2851,54 +2991,50 @@ nonrefreshable_access_token_lifetime: 24h The amount of time to allow a user-interactive authentication session to be active. -This defaults to 0, meaning the user is queried for their credentials -before every action, but this can be overridden to allow a single -validation to be re-used. This weakens the protections afforded by -the user-interactive authentication process, by allowing for multiple -(and potentially different) operations to use the same validation session. +This defaults to 0, meaning the user is queried for their credentials before every action, but this can be overridden to allow a single validation to be re-used. This weakens the protections afforded by the user-interactive authentication process, by allowing for multiple (and potentially different) operations to use the same validation session. -This is ignored for potentially "dangerous" operations (including -deactivating an account, modifying an account password, adding a 3PID, -and minting additional login tokens). +This is ignored for potentially "dangerous" operations (including deactivating an account, modifying an account password, adding a 3PID, and minting additional login tokens). Use the `session_timeout` sub-option here to change the time allowed for credential validation. +Defaults to `0`. + Example configuration: ```yaml ui_auth: - session_timeout: "15s" + session_timeout: 15s ``` --- ### `login_via_existing_session` -Matrix supports the ability of an existing session to mint a login token for -another client. +*(object)* Matrix supports the ability of an existing session to mint a login token for another client. -Synapse disables this by default as it has security ramifications -- a malicious -client could use the mechanism to spawn more than one session. +Synapse disables this by default as it has security ramifications – a malicious client could use the mechanism to spawn more than one session. -The duration of time the generated token is valid for can be configured with the -`token_timeout` sub-option. +This setting has the following sub-options: -User-interactive authentication is required when this is enabled unless the -`require_ui_auth` sub-option is set to `False`. +* `enabled` (boolean): Enable login via existing session. Defaults to `false`. + +* `require_ui_auth` (boolean): Require user-interactive authentication. Defaults to `true`. + +* `token_timeout` (duration): Duration of time the generated token is valid. Defaults to `"5m"`. Example configuration: ```yaml login_via_existing_session: - enabled: true - require_ui_auth: false - token_timeout: "5m" + enabled: true + require_ui_auth: false + token_timeout: 5m ``` --- ## Metrics + Config options related to metrics. --- ### `enable_metrics` -Set to true to enable collection and rendering of performance metrics. -Defaults to false. +*(boolean)* Set to true to enable collection and rendering of performance metrics. Defaults to `false`. Example configuration: ```yaml @@ -2907,51 +3043,46 @@ enable_metrics: true --- ### `sentry` -Use this option to enable sentry integration. Provide the DSN assigned to you by sentry -with the `dsn` setting. +*(object)* Use this option to enable sentry integration. Provide the DSN assigned to you by sentry with the `dsn` setting. - An optional `environment` field can be used to specify an environment. This allows - for log maintenance based on different environments, ensuring better organization - and analysis.. +An optional `environment` field can be used to specify an environment. This allows for log maintenance based on different environments, ensuring better organization and analysis. -NOTE: While attempts are made to ensure that the logs don't contain -any sensitive information, this cannot be guaranteed. By enabling -this option the sentry server may therefore receive sensitive -information, and it in turn may then disseminate sensitive information -through insecure notification channels if so configured. +NOTE: While attempts are made to ensure that the logs don't contain any sensitive information, this cannot be guaranteed. By enabling this option the sentry server may therefore receive sensitive information, and it in turn may then disseminate sensitive information through insecure notification channels if so configured. + +This setting has the following sub-options: + +* `dsn` (string|null): The DSN assigned by sentry. If unset or null, sentry integration is disabled. Defaults to `null`. + +* `environment` (string|null): Sentry environment. Defaults to `null`. Example configuration: ```yaml sentry: - environment: "production" - dsn: "..." + environment: production + dsn: '...' ``` --- ### `metrics_flags` -Flags to enable Prometheus metrics which are not suitable to be -enabled by default, either for performance reasons or limited use. -Currently the only option is `known_servers`, which publishes -`synapse_federation_known_servers`, a gauge of the number of -servers this homeserver knows about, including itself. May cause -performance problems on large homeservers. +*(object)* Flags to enable Prometheus metrics which are not suitable to be enabled by default, either for performance reasons or limited use. Currently the only option is `known_servers`. + +This setting has the following sub-options: + +* `known_servers` (boolean): Publishes `synapse_federation_known_servers`, a gauge of the number of servers this homeserver knows about, including itself. May cause performance problems on large homeservers. Defaults to `false`. Example configuration: ```yaml metrics_flags: - known_servers: true + known_servers: true ``` --- ### `report_stats` -Whether or not to report homeserver usage statistics. This is originally -set when generating the config. Set this option to true or false to change the current -behavior. See -[Reporting Homeserver Usage Statistics](../administration/monitoring/reporting_homeserver_usage_statistics.md) -for information on what data is reported. +*(boolean)* Whether or not to report homeserver usage statistics. This is originally set when generating the config. Set this option to true or false to change the current behavior. See [Reporting Homeserver Usage Statistics](../administration/monitoring/reporting_homeserver_usage_statistics.md) for information on what data is reported. -Statistics will be reported 5 minutes after Synapse starts, and then every 3 hours -after that. +Statistics will be reported 5 minutes after Synapse starts, and then every 3 hours after that. + +Defaults to `false`. Example configuration: ```yaml @@ -2960,8 +3091,7 @@ report_stats: true --- ### `report_stats_endpoint` -The endpoint to report homeserver usage statistics to. -Defaults to https://matrix.org/report-usage-stats/push +*(string)* The endpoint to report homeserver usage statistics to. Defaults to `"https://matrix.org/report-usage-stats/push"`. Example configuration: ```yaml @@ -2969,14 +3099,13 @@ report_stats_endpoint: https://example.com/report-usage-stats/push ``` --- ## API Configuration -Config settings related to the client/server API + +Config settings related to the client/server API. --- ### `room_prejoin_state` -This setting controls the state that is shared with users upon receiving an -invite to a room, or in reply to a knock on a room. By default, the following -state events are shared with users: +*(object)* This setting controls the state that is shared with users upon receiving an invite to a room, or in reply to a knock on a room. By default, the following state events are shared with users: - `m.room.join_rules` - `m.room.canonical_alias` @@ -2986,56 +3115,43 @@ state events are shared with users: - `m.room.create` - `m.room.topic` -To change the default behavior, use the following sub-options: -* `disable_default_event_types`: boolean. Set to `true` to disable the above - defaults. If this is enabled, only the event types listed in - `additional_event_types` are shared. Defaults to `false`. -* `additional_event_types`: A list of additional state events to include in the - events to be shared. By default, this list is empty (so only the default event - types are shared). +*Changed in Synapse 1.74:* admins can filter the events in prejoin state based on their state key. - Each entry in this list should be either a single string or a list of two - strings. - * A standalone string `t` represents all events with type `t` (i.e. - with no restrictions on state keys). - * A pair of strings `[t, s]` represents a single event with type `t` and - state key `s`. The same type can appear in two entries with different state - keys: in this situation, both state keys are included in prejoin state. +This setting has the following sub-options: + +* `disable_default_event_types` (boolean): Set to `true` to disable the above defaults. If this is enabled, only the event types listed in `additional_event_types` are shared. Defaults to `false`. + +* `additional_event_types` (array): A list of additional state events to include in the events to be shared. By default, this list is empty (so only the default event types are shared). + + Each entry in this list should be either a single string or a list of two strings. + * A standalone string `t` represents all events with type `t` (i.e. with no restrictions on state keys). + * A pair of strings `[t, s]` represents a single event with type `t` and state key `s`. The same type can appear in two entries with different state keys: in this situation, both state keys are included in prejoin state. + + Defaults to `[]`. Example configuration: ```yaml room_prejoin_state: - disable_default_event_types: false - additional_event_types: - # Share all events of type `org.example.custom.event.typeA` - - org.example.custom.event.typeA - # Share only events of type `org.example.custom.event.typeB` whose - # state_key is "foo" - - ["org.example.custom.event.typeB", "foo"] - # Share only events of type `org.example.custom.event.typeC` whose - # state_key is "bar" or "baz" - - ["org.example.custom.event.typeC", "bar"] - - ["org.example.custom.event.typeC", "baz"] + disable_default_event_types: false + additional_event_types: + - org.example.custom.event.typeA + - - org.example.custom.event.typeB + - foo + - - org.example.custom.event.typeC + - bar + - - org.example.custom.event.typeC + - baz ``` - -*Changed in Synapse 1.74:* admins can filter the events in prejoin state based -on their state key. - --- ### `track_puppeted_user_ips` -We record the IP address of clients used to access the API for various -reasons, including displaying it to the user in the "Where you're signed in" -dialog. +*(boolean)* We record the IP address of clients used to access the API for various reasons, including displaying it to the user in the "Where you're signed in" dialog. -By default, when puppeting another user via the admin API, the client IP -address is recorded against the user who created the access token (ie, the -admin user), and *not* the puppeted user. +By default, when puppeting another user via the admin API, the client IP address is recorded against the user who created the access token (ie, the admin user), and *not* the puppeted user. -Set this option to true to also record the IP address against the puppeted -user. (This also means that the puppeted user will count as an "active" user -for the purpose of monthly active user tracking - see `limit_usage_by_mau` etc -above.) +Set this option to true to also record the IP address against the puppeted user. (This also means that the puppeted user will count as an "active" user for the purpose of monthly active user tracking – see `limit_usage_by_mau` etc above.) + +Defaults to `false`. Example configuration: ```yaml @@ -3044,19 +3160,18 @@ track_puppeted_user_ips: true --- ### `app_service_config_files` -A list of application service config files to use. +*(array)* A list of application service config files to use. Defaults to `[]`. Example configuration: ```yaml app_service_config_files: - - app_service_1.yaml - - app_service_2.yaml +- app_service_1.yaml +- app_service_2.yaml ``` --- ### `track_appservice_user_ips` -Defaults to false. Set to true to enable tracking of application service IP addresses. -Implicitly enables MAU tracking for application service users. +*(boolean)* Set to true to enable tracking of application service IP addresses. Implicitly enables MAU tracking for application service users. Defaults to `false`. Example configuration: ```yaml @@ -3065,91 +3180,122 @@ track_appservice_user_ips: true --- ### `use_appservice_legacy_authorization` -Whether to send the application service access tokens via the `access_token` query parameter -per older versions of the Matrix specification. Defaults to false. Set to true to enable sending -access tokens via a query parameter. +*(boolean)* Whether to send the application service access tokens via the `access_token` query parameter per older versions of the Matrix specification. Defaults to false. Set to true to enable sending access tokens via a query parameter. -**Enabling this option is considered insecure and is not recommended. ** +**Enabling this option is considered insecure and is not recommended.** + +Defaults to `false`. Example configuration: ```yaml use_appservice_legacy_authorization: true ``` - --- ### `macaroon_secret_key` -A secret which is used to sign +*(string|null)* A secret which is used to sign - access token for guest users, - short-term login token used during SSO logins (OIDC or SAML2) and - token used for unsubscribing from email notifications. -If none is specified, the `registration_shared_secret` is used, if one is given; -otherwise, a secret key is derived from the signing key. +If none is specified, the `registration_shared_secret` is used, if one is given; otherwise, a secret key is derived from the signing key. + +> ⚠️ **Warning** – Replacing an existing `macaroon_secret_key` with a new one will lead to invalidation of access tokens for all guest users. It will also break unsubscribe links in emails sent before the change. An unlucky user might encounter a broken SSO login flow and would have to start again. + +Defaults to `null`. Example configuration: ```yaml macaroon_secret_key: ``` --- +### `macaroon_secret_key_path` + +*(string|null)* An alternative to [`macaroon_secret_key`](#macaroon_secret_key): allows the secret key to be specified in an external file. + +The file should be a plain text file, containing only the secret key. Synapse reads the secret key from the given file once at startup. + +_Added in Synapse 1.121.0._ + +Defaults to `null`. + +Example configuration: +```yaml +macaroon_secret_key_path: /path/to/secrets/file +``` +--- ### `form_secret` -A secret which is used to calculate HMACs for form values, to stop -falsification of values. Must be specified for the User Consent -forms to work. +*(string|null)* A secret which is used to calculate HMACs for form values, to stop falsification of values. Must be specified for the User Consent forms to work. + +Replacing an existing `form_secret` with a new one might break the user consent page for an unlucky user and require them to reopen the page from a new link. + +Defaults to `null`. Example configuration: ```yaml form_secret: ``` --- +### `form_secret_path` + +*(string|null)* An alternative to [`form_secret`](#form_secret): allows the secret to be specified in an external file. + +The file should be a plain text file, containing only the secret. Synapse reads the secret from the given file once at startup. + +_Added in Synapse 1.126.0._ + +Defaults to `null`. + +Example configuration: +```yaml +form_secret_path: /path/to/secrets/file +``` +--- ## Signing Keys -Config options relating to signing keys + +Config options relating to signing keys. --- ### `signing_key_path` -Path to the signing key to sign events and federation requests with. +*(string|null)* Path to the signing key to sign events and federation requests with. -*New in Synapse 1.67*: If this file does not exist, Synapse will create a new signing -key on startup and store it in this file. +*New in Synapse 1.67*: If this file does not exist, Synapse will create a new signing key on startup and store it in this file. + +Defaults to `null`. Example configuration: ```yaml -signing_key_path: "CONFDIR/SERVERNAME.signing.key" +signing_key_path: CONFDIR/SERVERNAME.signing.key ``` --- ### `old_signing_keys` -The keys that the server used to sign messages with but won't use -to sign new messages. For each key, `key` should be the base64-encoded public key, and -`expired_ts`should be the time (in milliseconds since the unix epoch) that -it was last used. +*(object)* The keys that the server used to sign messages with but won't use to sign new messages. -It is possible to build an entry from an old `signing.key` file using the -`export_signing_key` script which is provided with synapse. +It is possible to build an entry from an old `signing.key` file using the `export_signing_key` script which is provided with synapse. -If you have lost the private key file, you can ask another server you trust to -tell you the public keys it has seen from your server. To fetch the keys from -`matrix.org`, try something like: +If you have lost the private key file, you can ask another server you trust to tell you the public keys it has seen from your server. To fetch the keys from `matrix.org`, try something like: ``` curl https://matrix-federation.matrix.org/_matrix/key/v2/query/myserver.example.com | jq '.server_keys | map(.verify_keys) | add' ``` +Defaults to `{}`. + Example configuration: ```yaml old_signing_keys: - "ed25519:id": { key: "base64string", expired_ts: 123456789123 } + ed25519:id: + key: base64string + expired_ts: 123456789123 ``` --- ### `key_refresh_interval` -How long key response published by this server is valid for. -Used to set the `valid_until_ts` in `/key/v2` APIs. -Determines how quickly servers will query to check which keys -are still valid. Defaults to 1d. +*(duration)* How long key response published by this server is valid for. Used to set the `valid_until_ts` in `/key/v2` APIs. Determines how quickly servers will query to check which keys are still valid. Defaults to `"1d"`. Example configuration: ```yaml @@ -3158,57 +3304,41 @@ key_refresh_interval: 2d --- ### `trusted_key_servers` -The trusted servers to download signing keys from. +*(array)* The trusted servers to download signing keys from. When we need to fetch a signing key, each server is tried in parallel. -Normally, the connection to the key server is validated via TLS certificates. -Additional security can be provided by configuring a `verify key`, which -will make synapse check that the response is signed by that key. +Normally, the connection to the key server is validated via TLS certificates. Additional security can be provided by configuring a `verify key`, which will make synapse check that the response is signed by that key. -This setting supersedes an older setting named `perspectives`. The old format -is still supported for backwards-compatibility, but it is deprecated. +This setting supersedes an older setting named `perspectives`. The old format is still supported for backwards-compatibility, but it is deprecated. -`trusted_key_servers` defaults to matrix.org, but using it will generate a -warning on start-up. To suppress this warning, set -`suppress_key_server_warning` to true. +`trusted_key_servers` defaults to matrix.org, but using it will generate a warning on start-up. To suppress this warning, set `suppress_key_server_warning` to true. -If the use of a trusted key server has to be deactivated, e.g. in a private -federation or for privacy reasons, this can be realised by setting -an empty array (`trusted_key_servers: []`). Then Synapse will request the keys -directly from the server that owns the keys. If Synapse does not get keys directly -from the server, the events of this server will be rejected. +If the use of a trusted key server has to be deactivated, e.g. in a private federation or for privacy reasons, this can be realised by setting an empty array (`trusted_key_servers: []`). Then Synapse will request the keys directly from the server that owns the keys. If Synapse does not get keys directly from the server, the events of this server will be rejected. -Options for each entry in the list include: -* `server_name`: the name of the server. Required. -* `verify_keys`: an optional map from key id to base64-encoded public key. - If specified, we will check that the response is signed by at least - one of the given keys. -* `accept_keys_insecurely`: a boolean. Normally, if `verify_keys` is unset, - and `federation_verify_certificates` is not `true`, synapse will refuse - to start, because this would allow anyone who can spoof DNS responses - to masquerade as the trusted key server. If you know what you are doing - and are sure that your network environment provides a secure connection - to the key server, you can set this to `true` to override this behaviour. - -Example configuration #1: +Default configuration: ```yaml trusted_key_servers: - - server_name: "my_trusted_server.example.com" - verify_keys: - "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" - - server_name: "my_other_trusted_server.example.com" +- server_name: matrix.org ``` -Example configuration #2: + +Example configurations: ```yaml trusted_key_servers: - - server_name: "matrix.org" +- server_name: my_trusted_server.example.com + verify_keys: + ed25519:auto: abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr +- server_name: my_other_trusted_server.example.com +``` + +```yaml +trusted_key_servers: +- server_name: matrix.org ``` --- ### `suppress_key_server_warning` -Set the following to true to disable the warning that is emitted when the -`trusted_key_servers` include 'matrix.org'. See above. +*(boolean)* Set the following to true to disable the warning that is emitted when the `trusted_key_servers` include "matrix.org". See above. Defaults to `false`. Example configuration: ```yaml @@ -3217,639 +3347,509 @@ suppress_key_server_warning: true --- ### `key_server_signing_keys_path` -The signing keys to use when acting as a trusted key server. If not specified -defaults to the server signing key. +*(string|null)* The signing keys to use when acting as a trusted key server. If not specified defaults to the server signing key. Can contain multiple keys, one per line. +Defaults to `null`. + Example configuration: ```yaml -key_server_signing_keys_path: "key_server_signing_keys.key" +key_server_signing_keys_path: key_server_signing_keys.key ``` --- ## Single sign-on integration -The following settings can be used to make Synapse use a single sign-on -provider for authentication, instead of its internal password database. +The following settings can be used to make Synapse use a single sign-on provider for authentication, instead of its internal password database. -You will probably also want to set the following options to `false` to -disable the regular login/registration flows: - * [`enable_registration`](#enable_registration) - * [`password_config.enabled`](#password_config) +You will probably also want to set the following options to `false` to disable the regular login/registration flows: +* [`enable_registration`](#enable_registration) +* [`password_config.enabled`](#password_config) --- ### `saml2_config` -Enable SAML2 for registration and login. Uses pysaml2. To learn more about pysaml and -to find a full list options for configuring pysaml, read the docs [here](https://pysaml2.readthedocs.io/en/latest/). +*(object)* Enable SAML2 for registration and login. Uses pysaml2. To learn more about pysaml and to find a full list options for configuring pysaml, read the docs [here](https://pysaml2.readthedocs.io/en/latest/). + +At least one of `sp_config` or `config_path` must be set in this section to enable SAML login. You can either put your entire pysaml config inline using the `sp_config` option, or you can specify a path to a psyaml config file with the sub-option `config_path`. + +Once SAML support is enabled, a metadata file will be exposed at `https://:/_synapse/client/saml2/metadata.xml`, which you may be able to use to configure your SAML IdP with. Alternatively, you can manually configure the IdP to use an ACS location of `https://:/_synapse/client/saml2/authn_response`. -At least one of `sp_config` or `config_path` must be set in this section to -enable SAML login. You can either put your entire pysaml config inline using the `sp_config` -option, or you can specify a path to a psyaml config file with the sub-option `config_path`. This setting has the following sub-options: -* `idp_name`: A user-facing name for this identity provider, which is used to - offer the user a choice of login mechanisms. -* `idp_icon`: An optional icon for this identity provider, which is presented - by clients and Synapse's own IdP picker page. If given, must be an - MXC URI of the format `mxc:///`. (An easy way to - obtain such an MXC URI is to upload an image to an (unencrypted) room - and then copy the "url" from the source of the event.) -* `idp_brand`: An optional brand for this identity provider, allowing clients - to style the login flow according to the identity provider in question. - See the [spec](https://spec.matrix.org/latest/) for possible options here. -* `sp_config`: the configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config. - Default values will be used for the `entityid` and `service` settings, - so it is not normally necessary to specify them unless you need to - override them. Here are a few useful sub-options for configuring pysaml: - * `metadata`: Point this to the IdP's metadata. You must provide either a local - file via the `local` attribute or (preferably) a URL via the - `remote` attribute. - * `accepted_time_diff: 3`: Allowed clock difference in seconds between the homeserver and IdP. - Defaults to 0. - * `service`: By default, the user has to go to our login page first. If you'd like - to allow IdP-initiated login, set `allow_unsolicited` to true under `sp` in the `service` - section. -* `config_path`: specify a separate pysaml2 configuration file thusly: - `config_path: "CONFDIR/sp_conf.py"` -* `saml_session_lifetime`: The lifetime of a SAML session. This defines how long a user has to - complete the authentication process, if `allow_unsolicited` is unset. The default is 15 minutes. -* `user_mapping_provider`: Using this option, an external module can be provided as a - custom solution to mapping attributes returned from a saml provider onto a matrix user. The - `user_mapping_provider` has the following attributes: - * `module`: The custom module's class. - * `config`: Custom configuration values for the module. Use the values provided in the - example if you are using the built-in user_mapping_provider, or provide your own - config values for a custom class if you are using one. This section will be passed as a Python - dictionary to the module's `parse_config` method. The built-in provider takes the following two - options: - * `mxid_source_attribute`: The SAML attribute (after mapping via the attribute maps) to use - to derive the Matrix ID from. It is 'uid' by default. Note: This used to be configured by the - `saml2_config.mxid_source_attribute option`. If that is still defined, its value will be used instead. - * `mxid_mapping`: The mapping system to use for mapping the saml attribute onto a - matrix ID. Options include: `hexencode` (which maps unpermitted characters to '=xx') - and `dotreplace` (which replaces unpermitted characters with '.'). - The default is `hexencode`. Note: This used to be configured by the - `saml2_config.mxid_mapping option`. If that is still defined, its value will be used instead. -* `grandfathered_mxid_source_attribute`: In previous versions of synapse, the mapping from SAML attribute to - MXID was always calculated dynamically rather than stored in a table. For backwards- compatibility, we will look for `user_ids` - matching such a pattern before creating a new account. This setting controls the SAML attribute which will be used for this - backwards-compatibility lookup. Typically it should be 'uid', but if the attribute maps are changed, it may be necessary to change it. - The default is 'uid'. -* `attribute_requirements`: It is possible to configure Synapse to only allow logins if SAML attributes - match particular values. The requirements can be listed under - `attribute_requirements` as shown in the example. All of the listed attributes must - match for the login to be permitted. -* `idp_entityid`: If the metadata XML contains multiple IdP entities then the `idp_entityid` - option must be set to the entity to redirect users to. - Most deployments only have a single IdP entity and so should omit this option. +* `idp_name` (string): A user-facing name for this identity provider, which is used to offer the user a choice of login mechanisms. +* `idp_icon` (string|null): An optional icon for this identity provider, which is presented by clients and Synapse's own IdP picker page. If given, must be an MXC URI of the format `mxc:///`. (An easy way to obtain such an MXC URI is to upload an image to an (unencrypted) room and then copy the URL from the source of the event.) -Once SAML support is enabled, a metadata file will be exposed at -`https://:/_synapse/client/saml2/metadata.xml`, which you may be able to -use to configure your SAML IdP with. Alternatively, you can manually configure -the IdP to use an ACS location of -`https://:/_synapse/client/saml2/authn_response`. +* `idp_brand`: An optional brand for this identity provider, allowing clients to style the login flow according to the identity provider in question. See the [spec](https://spec.matrix.org/latest/) for possible options here. + +* `sp_config` (object|null): Configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config. Default values will be used for the `entityid` and `service` settings, so it is not normally necessary to specify them unless you need to override them. Here are a few useful sub-options for configuring pysaml: + * `metadata`: Point this to the IdP's metadata. You must provide either a local file via the `local` attribute or (preferably) a URL via the `remote` attribute. + * `accepted_time_diff: 3`: Allowed clock difference in seconds between the homeserver and IdP. Defaults to 0. + * `service`: By default, the user has to go to our login page first. If you'd like to allow IdP-initiated login, set `allow_unsolicited` to true under `sp` in the `service` section. Defaults to `null`. + +* `config_path` (string|null): Specify a separate pysaml2 configuration file. Defaults to `null`. + +* `saml_session_lifetime` (duration): The lifetime of a SAML session. This defines how long a user has to complete the authentication process, if `allow_unsolicited` is unset. Defaults to `"15m"`. + +* `user_mapping_provider` (object): Using this option, an external module can be provided as a custom solution to mapping attributes returned from a saml provider onto a matrix user. + + This setting has the following sub-options: + + * `module` (string): The custom module's class. + + * `config` (object): Custom configuration values for the module. Use the values provided in the example if you are using the built-in user_mapping_provider, or provide your own config values for a custom class if you are using one. This section will be passed as a Python dictionary to the module's `parse_config` method. The built-in provider takes the following two options: + * `mxid_source_attribute`: The SAML attribute (after mapping via the attribute maps) to use to derive the Matrix ID from. It is "uid" by default. Note: This used to be configured by the `saml2_config.mxid_source_attribute option`. If that is still defined, its value will be used instead. + * `mxid_mapping`: The mapping system to use for mapping the saml attribute onto a matrix ID. Options include: `hexencode` (which maps unpermitted characters to `=xx`) and `dotreplace` (which replaces unpermitted characters with `.`). The default is `hexencode`. Note: This used to be configured by the `saml2_config.mxid_mapping option`. If that is still defined, its value will be used instead. + +* `grandfathered_mxid_source_attribute` (string): In previous versions of synapse, the mapping from SAML attribute to MXID was always calculated dynamically rather than stored in a table. For backwards-compatibility, we will look for `user_ids` matching such a pattern before creating a new account. This setting controls the SAML attribute which will be used for this backwards-compatibility lookup. Typically it should be "uid", but if the attribute maps are changed, it may be necessary to change it. Defaults to `"uid"`. + +* `attribute_requirements` (array): It is possible to configure Synapse to only allow logins if SAML attributes match particular values. The requirements can be listed under `attribute_requirements` as shown in the example. All of the listed attributes must match for the login to be permitted. Values can be specified in a `one_of` list to allow multiple values for an attribute. + + Options for each entry include: + + * `attribute` (string): SAML attribute for which to allow logins. + + * `value` (string): Value the SAML attribute must match. + + * `one_of` (array): List of values the SAML attribute must all match. + +* `idp_entityid` (string|null): If the metadata XML contains multiple IdP entities then the `idp_entityid` option must be set to the entity to redirect users to. Most deployments only have a single IdP entity and so should omit this option. Defaults to `null`. Example configuration: ```yaml saml2_config: sp_config: metadata: - local: ["saml2/idp.xml"] + local: + - saml2/idp.xml remote: - - url: https://our_idp/metadata.xml + - url: https://our_idp/metadata.xml accepted_time_diff: 3 - service: sp: allow_unsolicited: true - - # The examples below are just used to generate our metadata xml, and you - # may well not need them, depending on your setup. Alternatively you - # may need a whole lot more detail - see the pysaml2 docs! - description: ["My awesome SP", "en"] - name: ["Test SP", "en"] - + description: + - My awesome SP + - en + name: + - Test SP + - en ui_info: display_name: - - lang: en - text: "Display Name is the descriptive name of your service." + - lang: en + text: Display Name is the descriptive name of your service. description: - - lang: en - text: "Description should be a short paragraph explaining the purpose of the service." + - lang: en + text: Description should be a short paragraph explaining the purpose of the + service. information_url: - - lang: en - text: "https://example.com/terms-of-service" + - lang: en + text: https://example.com/terms-of-service privacy_statement_url: - - lang: en - text: "https://example.com/privacy-policy" + - lang: en + text: https://example.com/privacy-policy keywords: - - lang: en - text: ["Matrix", "Element"] + - lang: en + text: + - Matrix + - Element logo: - - lang: en - text: "https://example.com/logo.svg" - width: "200" - height: "80" - + - lang: en + text: https://example.com/logo.svg + width: '200' + height: '80' organization: name: Example com display_name: - - ["Example co", "en"] - url: "http://example.com" - + - - Example co + - en + url: http://example.com contact_person: - - given_name: Bob - sur_name: "the Sysadmin" - email_address: ["admin@example.com"] - contact_type: technical - + - given_name: Bob + sur_name: the Sysadmin + email_address: + - admin@example.com + contact_type: technical saml_session_lifetime: 5m - user_mapping_provider: - # Below options are intended for the built-in provider, they should be - # changed if using a custom module. config: mxid_source_attribute: displayName mxid_mapping: dotreplace - grandfathered_mxid_source_attribute: upn - attribute_requirements: - - attribute: userGroup - value: "staff" - - attribute: department - value: "sales" - - idp_entityid: 'https://our_idp/entityid' + - attribute: userGroup + value: staff + - attribute: department + one_of: + - sales + - admins + idp_entityid: https://our_idp/entityid ``` --- ### `oidc_providers` -List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration -and login. See [here](../../openid.md) -for information on how to configure these options. +*(array)* List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration and login. See [here](../../openid.md) for information on how to configure these options. -For backwards compatibility, it is also possible to configure a single OIDC -provider via an `oidc_config` setting. This is now deprecated and admins are -advised to migrate to the `oidc_providers` format. (When doing that migration, -use `oidc` for the `idp_id` to ensure that existing users continue to be -recognised.) +For backwards compatibility, it is also possible to configure a single OIDC provider via an `oidc_config` setting. This is now deprecated and admins are advised to migrate to the `oidc_providers` format. (When doing that migration, use `oidc` for the `idp_id` to ensure that existing users continue to be recognised.) -Options for each entry include: -* `idp_id`: a unique identifier for this identity provider. Used internally - by Synapse; should be a single word such as 'github'. - Note that, if this is changed, users authenticating via that provider - will no longer be recognised as the same user! - (Use "oidc" here if you are migrating from an old `oidc_config` configuration.) - -* `idp_name`: A user-facing name for this identity provider, which is used to - offer the user a choice of login mechanisms. - -* `idp_icon`: An optional icon for this identity provider, which is presented - by clients and Synapse's own IdP picker page. If given, must be an - MXC URI of the format `mxc:///`. (An easy way to - obtain such an MXC URI is to upload an image to an (unencrypted) room - and then copy the "url" from the source of the event.) - -* `idp_brand`: An optional brand for this identity provider, allowing clients - to style the login flow according to the identity provider in question. - See the [spec](https://spec.matrix.org/latest/) for possible options here. - -* `discover`: set to false to disable the use of the OIDC discovery mechanism - to discover endpoints. Defaults to true. - -* `issuer`: Required. The OIDC issuer. Used to validate tokens and (if discovery - is enabled) to discover the provider's endpoints. - -* `client_id`: Required. oauth2 client id to use. - -* `client_secret`: oauth2 client secret to use. May be omitted if - `client_secret_jwt_key` is given, or if `client_auth_method` is 'none'. - Must be omitted if `client_secret_path` is specified. - -* `client_secret_path`: path to the oauth2 client secret to use. With that - it's not necessary to leak secrets into the config file itself. - Mutually exclusive with `client_secret`. Can be omitted if - `client_secret_jwt_key` is specified. - - *Added in Synapse 1.91.0.* - -* `client_secret_jwt_key`: Alternative to client_secret: details of a key used - to create a JSON Web Token to be used as an OAuth2 client secret. If - given, must be a dictionary with the following properties: - - * `key`: a pem-encoded signing key. Must be a suitable key for the - algorithm specified. Required unless `key_file` is given. - - * `key_file`: the path to file containing a pem-encoded signing key file. - Required unless `key` is given. - - * `jwt_header`: a dictionary giving properties to include in the JWT - header. Must include the key `alg`, giving the algorithm used to - sign the JWT, such as "ES256", using the JWA identifiers in - RFC7518. - - * `jwt_payload`: an optional dictionary giving properties to include in - the JWT payload. Normally this should include an `iss` key. - -* `client_auth_method`: auth method to use when exchanging the token. Valid - values are `client_secret_basic` (default), `client_secret_post` and - `none`. - -* `pkce_method`: Whether to use proof key for code exchange when requesting - and exchanging the token. Valid values are: `auto`, `always`, or `never`. Defaults - to `auto`, which uses PKCE if supported during metadata discovery. Set to `always` - to force enable PKCE or `never` to force disable PKCE. - -* `scopes`: list of scopes to request. This should normally include the "openid" - scope. Defaults to `["openid"]`. - -* `authorization_endpoint`: the oauth2 authorization endpoint. Required if - provider discovery is disabled. - -* `token_endpoint`: the oauth2 token endpoint. Required if provider discovery is - disabled. - -* `userinfo_endpoint`: the OIDC userinfo endpoint. Required if discovery is - disabled and the 'openid' scope is not requested. - -* `jwks_uri`: URI where to fetch the JWKS. Required if discovery is disabled and - the 'openid' scope is used. - -* `skip_verification`: set to 'true' to skip metadata verification. Use this if - you are connecting to a provider that is not OpenID Connect compliant. - Defaults to false. Avoid this in production. - -* `user_profile_method`: Whether to fetch the user profile from the userinfo - endpoint, or to rely on the data returned in the id_token from the `token_endpoint`. - Valid values are: `auto` or `userinfo_endpoint`. - Defaults to `auto`, which uses the userinfo endpoint if `openid` is - not included in `scopes`. Set to `userinfo_endpoint` to always use the - userinfo endpoint. - -* `additional_authorization_parameters`: String to string dictionary that will be passed as - additional parameters to the authorization grant URL. - -* `allow_existing_users`: set to true to allow a user logging in via OIDC to - match a pre-existing account instead of failing. This could be used if - switching from password logins to OIDC. Defaults to false. - -* `enable_registration`: set to 'false' to disable automatic registration of new - users. This allows the OIDC SSO flow to be limited to sign in only, rather than - automatically registering users that have a valid SSO login but do not have - a pre-registered account. Defaults to true. - -* `user_mapping_provider`: Configuration for how attributes returned from a OIDC - provider are mapped onto a matrix user. This setting has the following - sub-properties: - - * `module`: The class name of a custom mapping module. Default is - `synapse.handlers.oidc.JinjaOidcMappingProvider`. - See [OpenID Mapping Providers](../../sso_mapping_providers.md#openid-mapping-providers) - for information on implementing a custom mapping provider. - - * `config`: Configuration for the mapping provider module. This section will - be passed as a Python dictionary to the user mapping provider - module's `parse_config` method. - - For the default provider, the following settings are available: - - * `subject_template`: Jinja2 template for a unique identifier for the user. - Defaults to `{{ user.sub }}`, which OpenID Connect compliant providers should provide. - - This replaces and overrides `subject_claim`. - - * `subject_claim`: name of the claim containing a unique identifier - for the user. Defaults to 'sub', which OpenID Connect - compliant providers should provide. - - *Deprecated in Synapse v1.75.0.* - - * `picture_template`: Jinja2 template for an url for the user's profile picture. - Defaults to `{{ user.picture }}`, which OpenID Connect compliant providers should - provide and has to refer to a direct image file such as PNG, JPEG, or GIF image file. - - This replaces and overrides `picture_claim`. - - Currently only supported in monolithic (single-process) server configurations - where the media repository runs within the Synapse process. - - * `picture_claim`: name of the claim containing an url for the user's profile picture. - Defaults to 'picture', which OpenID Connect compliant providers should provide - and has to refer to a direct image file such as PNG, JPEG, or GIF image file. - - Currently only supported in monolithic (single-process) server configurations - where the media repository runs within the Synapse process. - - *Deprecated in Synapse v1.75.0.* - - * `localpart_template`: Jinja2 template for the localpart of the MXID. - If this is not set, the user will be prompted to choose their - own username (see the documentation for the `sso_auth_account_details.html` - template). This template can use the `localpart_from_email` filter. - - * `confirm_localpart`: Whether to prompt the user to validate (or - change) the generated localpart (see the documentation for the - 'sso_auth_account_details.html' template), instead of - registering the account right away. - - * `display_name_template`: Jinja2 template for the display name to set - on first login. If unset, no displayname will be set. - - * `email_template`: Jinja2 template for the email address of the user. - If unset, no email address will be added to the account. - - * `extra_attributes`: a map of Jinja2 templates for extra attributes - to send back to the client during login. Note that these are non-standard and clients will ignore them - without modifications. - - When rendering, the Jinja2 templates are given a 'user' variable, - which is set to the claims returned by the UserInfo Endpoint and/or - in the ID Token. - -* `backchannel_logout_enabled`: set to `true` to process OIDC Back-Channel Logout notifications. - Those notifications are expected to be received on `/_synapse/client/oidc/backchannel_logout`. - Defaults to `false`. - -* `backchannel_logout_ignore_sub`: by default, the OIDC Back-Channel Logout feature checks that the - `sub` claim matches the subject claim received during login. This check can be disabled by setting - this to `true`. Defaults to `false`. - - You might want to disable this if the `subject_claim` returned by the mapping provider is not `sub`. - -It is possible to configure Synapse to only allow logins if certain attributes -match particular values in the OIDC userinfo. The requirements can be listed under -`attribute_requirements` as shown here: +It is possible to configure Synapse to only allow logins if certain attributes match particular values in the OIDC userinfo. The requirements can be listed under `attribute_requirements` as shown here: ```yaml attribute_requirements: - - attribute: family_name - value: "Stephensson" - - attribute: groups - value: "admin" + - attribute: family_name + one_of: ["Stephensson", "Smith"] + - attribute: groups + value: "admin" + # If `value` or `one_of` are not specified, the attribute only needs + # to exist, regardless of value. + - attribute: picture ``` -All of the listed attributes must match for the login to be permitted. Additional attributes can be added to -userinfo by expanding the `scopes` section of the OIDC config to retrieve -additional information from the OIDC provider. -If the OIDC claim is a list, then the attribute must match any value in the list. -Otherwise, it must exactly match the value of the claim. Using the example -above, the `family_name` claim MUST be "Stephensson", but the `groups` -claim MUST contain "admin". +`attribute` is a required field, while `value` and `one_of` are optional. + +All of the listed attributes must match for the login to be permitted. Additional attributes can be added to userinfo by expanding the `scopes` section of the OIDC config to retrieve additional information from the OIDC provider. + +If the OIDC claim is a list, then the attribute must match any value in the list. Otherwise, it must exactly match the value of the claim. Using the example above, the `family_name` claim MUST be either "Stephensson" or "Smith", but the `groups` claim MUST contain "admin". + +Defaults to `[]`. + +Options for each entry include: + +* `idp_id` (string): A unique identifier for this identity provider. Used internally by Synapse; should be a single word such as "github". Note that, if this is changed, users authenticating via that provider will no longer be recognised as the same user! (Use "oidc" here if you are migrating from an old `oidc_config` configuration.) + +* `idp_name` (string): A user-facing name for this identity provider, which is used to offer the user a choice of login mechanisms. + +* `idp_icon` (string): An optional icon for this identity provider, which is presented by clients and Synapse's own IdP picker page. If given, must be an MXC URI of the format `mxc:///`. (An easy way to obtain such an MXC URI is to upload an image to an (unencrypted) room and then copy the URL from the source of the event.) + +* `idp_brand` (string): An optional brand for this identity provider, allowing clients to style the login flow according to the identity provider in question. See the [spec](https://spec.matrix.org/latest/) for possible options here. + +* `discover` (boolean): Set to false to disable the use of the OIDC discovery mechanism to discover endpoints. Defaults to true. + +* `issuer` (string): Required. The OIDC issuer. Used to validate tokens and (if discovery is enabled) to discover the provider's endpoints. + +* `client_id` (string): Required. OAuth2 client id to use. + +* `client_secret` (string|null): OAuth2 client secret to use. May be omitted if `client_secret_jwt_key` is given, or if `client_auth_method` is `none`. Must be omitted if `client_secret_path` is specified. + +* `client_secret_path` (string|null): Path to the OAuth2 client secret to use. With that it's not necessary to leak secrets into the config file itself. Mutually exclusive with `client_secret`. Can be omitted if `client_secret_jwt_key` is specified. + + *Added in Synapse 1.91.0.* + +* `client_secret_jwt_key` (object|null): Alternative to client_secret: details of a key used to create a JSON Web Token to be used as an OAuth2 client secret. + + This setting has the following sub-options: + + * `key` (string|null): A pem-encoded signing key. Must be a suitable key for the algorithm specified. Required unless `key_file` is given. + + * `key_file` (string|null): Path to the file containing a pem-encoded signing key. Required unless `key` is given. + + * `jwt_header` (object): Dictionary giving properties to include in the JWT header. Must include the key `alg`. + + This setting has the following sub-options: + + * `alg` (string): Algorithm used to sign the JWT, such as ES256, using the JWA identifiers in RFC7518. + + * `jwt_payload` (object): Optional dictionary giving properties to include in the JWT payload. Normally this should include an `iss` key. + +* `client_auth_method` (string|null): Auth method to use when exchanging the token. Valid values are `client_secret_basic` (default), `client_secret_post` and `none`. + +* `pkce_method` (string|null): Whether to use proof key for code exchange when requesting and exchanging the token. Valid values are: `auto`, `always`, or `never`. Defaults to `auto`, which uses PKCE if supported during metadata discovery. Set to `always` to force enable PKCE or `never` to force disable PKCE. + +* `id_token_signing_alg_values_supported` (array): List of the JWS signing algorithms (`alg` values) that are supported for signing the `id_token`. + + This is *not* required if `discovery` is disabled. We default to supporting `RS256` in the downstream usage if no algorithms are configured here or in the discovery document. + + According to the spec, the algorithm `"RS256"` MUST be included. The absolute rigid approach would be to reject this provider as non-compliant if it's not included but we simply allow whatever and see what happens (you're the one that configured the value and cooperating with the identity provider). + + The `alg` value `"none"` MAY be supported but can only be used if the Authorization Endpoint does not include `id_token` in the `response_type` (ex. `/authorize?response_type=code` where `none` can apply, `/authorize?response_type=code%20id_token` where `none` can't apply) (such as when using the Authorization Code Flow). + +* `scopes` (array|null): List of scopes to request. This should normally include the "openid" scope. Defaults to `["openid"]`. + +* `authorization_endpoint` (string): The OAuth2 authorization endpoint. Required if provider discovery is disabled. + +* `token_endpoint` (string): The OAuth2 token endpoint. Required if provider discovery is disabled. + +* `userinfo_endpoint` (string): The OIDC userinfo endpoint. Required if discovery is disabled and the "openid" scope is not requested. + +* `jwks_uri` (string): URI where to fetch the JWKS. Required if discovery is disabled and the "openid" scope is used. + +* `skip_verification` (boolean): Set to `true` to skip metadata verification. Use this if you are connecting to a provider that is not OpenID Connect compliant. Defaults to false. Avoid this in production. + +* `user_profile_method` (string|null): Whether to fetch the user profile from the userinfo endpoint, or to rely on the data returned in the id_token from the `token_endpoint`. Valid values are: `auto` or `userinfo_endpoint`. Defaults to `auto`, which uses the userinfo endpoint if `openid` is not included in `scopes`. Set to `userinfo_endpoint` to always use the userinfo endpoint. + +* `redirect_uri` (string|null): An optional string, that if set will override the `redirect_uri` parameter sent in the requests to the authorization and token endpoints. Useful if you want to redirect the client to another endpoint as part of the OIDC login. Be aware that the client must then call Synapse's OIDC callback URL (`/_synapse/client/oidc/callback`) manually afterwards. Must be a valid URL including scheme and path. + +* `additional_authorization_parameters` (object): String to string dictionary that will be passed as additional parameters to the authorization grant URL. + +* `passthrough_authorization_parameters` (array): List of parameters that will be passed through from the redirect endpoint to the authorization grant URL. + +* `allow_existing_users` (boolean): Set to true to allow a user logging in via OIDC to match a pre-existing account instead of failing. This could be used if switching from password logins to OIDC. Defaults to false. + +* `enable_registration` (boolean): Set to `false` to disable automatic registration of new users. This allows the OIDC SSO flow to be limited to sign in only, rather than automatically registering users that have a valid SSO login but do not have a pre-registered account. Defaults to true. + +* `user_mapping_provider` (object): Configuration for how attributes returned from a OIDC provider are mapped onto a matrix user. + + When rendering, the Jinja2 templates are given a `user` variable, which is set to the claims returned by the UserInfo Endpoint and/or in the ID Token. + + This setting has the following sub-options: + + * `module` (string): The class name of a custom mapping module. Default is `synapse.handlers.oidc.JinjaOidcMappingProvider`. See [OpenID Mapping Providers](../../sso_mapping_providers.md#openid-mapping-providers) for information on implementing a custom mapping provider. + + * `config` (object): Configuration for the mapping provider module. This section will be passed as a Python dictionary to the user mapping provider module's `parse_config` method. + + For the default provider, the following settings are available: + + * `subject_template`: Jinja2 template for a unique identifier for the user. Defaults to `{{ user.sub }}`, which OpenID Connect compliant providers should provide. + + This replaces and overrides `subject_claim`. + + * `subject_claim`: name of the claim containing a unique identifier for the user. Defaults to `sub`, which OpenID Connect compliant providers should provide. + + *Deprecated in Synapse v1.75.0.* + + * `picture_template`: Jinja2 template for an url for the user's profile picture. Defaults to `{{ user.picture }}`, which OpenID Connect compliant providers should provide and has to refer to a direct image file such as PNG, JPEG, or GIF image file. + + This replaces and overrides `picture_claim`. + + Currently only supported in monolithic (single-process) server configurations where the media repository runs within the Synapse process. + + * `picture_claim`: name of the claim containing an url for the user's profile picture. Defaults to "picture", which OpenID Connect compliant providers should provide and has to refer to a direct image file such as PNG, JPEG, or GIF image file. + + Currently only supported in monolithic (single-process) server configurations where the media repository runs within the Synapse process. + + *Deprecated in Synapse v1.75.0.* + + * `localpart_template`: Jinja2 template for the localpart of the MXID. If this is not set, the user will be prompted to choose their own username (see the documentation for the `sso_auth_account_details.html` template). This template can use the `localpart_from_email` filter. + + * `confirm_localpart`: Whether to prompt the user to validate (or change) the generated localpart (see the documentation for the "sso_auth_account_details.html" template), instead of registering the account right away. + + * `display_name_template`: Jinja2 template for the display name to set on first login. If unset, no displayname will be set. + + * `email_template`: Jinja2 template for the email address of the user. If unset, no email address will be added to the account. + + * `extra_attributes`: a map of Jinja2 templates for extra attributes to send back to the client during login. Note that these are non-standard and clients will ignore them without modifications. + +* `backchannel_logout_enabled` (boolean): Set to `true` to process OIDC Back-Channel Logout notifications. Those notifications are expected to be received on `/_synapse/client/oidc/backchannel_logout`. Defaults to `false`. + +* `backchannel_logout_ignore_sub` (boolean): By default, the OIDC Back-Channel Logout feature checks that the `sub` claim matches the subject claim received during login. This check can be disabled by setting this to `true`. Defaults to `false`. + + You might want to disable this if the `subject_claim` returned by the mapping provider is not `sub`. Example configuration: ```yaml oidc_providers: - # Generic example - # - - idp_id: my_idp - idp_name: "My OpenID provider" - idp_icon: "mxc://example.com/mediaid" - discover: false - issuer: "https://accounts.example.com/" - client_id: "provided-by-your-issuer" - client_secret: "provided-by-your-issuer" - client_auth_method: client_secret_post - scopes: ["openid", "profile"] - authorization_endpoint: "https://accounts.example.com/oauth2/auth" - token_endpoint: "https://accounts.example.com/oauth2/token" - userinfo_endpoint: "https://accounts.example.com/userinfo" - jwks_uri: "https://accounts.example.com/.well-known/jwks.json" - additional_authorization_parameters: - acr_values: 2fa - skip_verification: true - enable_registration: true - user_mapping_provider: - config: - subject_claim: "id" - localpart_template: "{{ user.login }}" - display_name_template: "{{ user.name }}" - email_template: "{{ user.email }}" - attribute_requirements: - - attribute: userGroup - value: "synapseUsers" +- idp_id: my_idp + idp_name: My OpenID provider + idp_icon: mxc://example.com/mediaid + discover: false + issuer: https://accounts.example.com/ + client_id: provided-by-your-issuer + client_secret: provided-by-your-issuer + client_auth_method: client_secret_post + scopes: + - openid + - profile + authorization_endpoint: https://accounts.example.com/oauth2/auth + token_endpoint: https://accounts.example.com/oauth2/token + userinfo_endpoint: https://accounts.example.com/userinfo + jwks_uri: https://accounts.example.com/.well-known/jwks.json + additional_authorization_parameters: + acr_values: 2fa + passthrough_authorization_parameters: + - login_hint + skip_verification: true + enable_registration: true + user_mapping_provider: + config: + subject_claim: id + localpart_template: '{{ user.login }}' + display_name_template: '{{ user.name }}' + email_template: '{{ user.email }}' + attribute_requirements: + - attribute: userGroup + value: synapseUsers ``` --- ### `cas_config` -Enable Central Authentication Service (CAS) for registration and login. -Has the following sub-options: -* `enabled`: Set this to true to enable authorization against a CAS server. - Defaults to false. -* `idp_name`: A user-facing name for this identity provider, which is used to - offer the user a choice of login mechanisms. -* `idp_icon`: An optional icon for this identity provider, which is presented - by clients and Synapse's own IdP picker page. If given, must be an - MXC URI of the format `mxc:///`. (An easy way to - obtain such an MXC URI is to upload an image to an (unencrypted) room - and then copy the "url" from the source of the event.) -* `idp_brand`: An optional brand for this identity provider, allowing clients - to style the login flow according to the identity provider in question. - See the [spec](https://spec.matrix.org/latest/) for possible options here. -* `server_url`: The URL of the CAS authorization endpoint. -* `protocol_version`: The CAS protocol version, defaults to none (version 3 is required if you want to use "required_attributes"). -* `displayname_attribute`: The attribute of the CAS response to use as the display name. - If no name is given here, no displayname will be set. -* `required_attributes`: It is possible to configure Synapse to only allow logins if CAS attributes - match particular values. All of the keys given below must exist - and the values must match the given value. Alternately if the given value - is `None` then any value is allowed (the attribute just must exist). - All of the listed attributes must match for the login to be permitted. -* `enable_registration`: set to 'false' to disable automatic registration of new - users. This allows the CAS SSO flow to be limited to sign in only, rather than - automatically registering users that have a valid SSO login but do not have - a pre-registered account. Defaults to true. -* `allow_numeric_ids`: set to 'true' allow numeric user IDs (default false). - This allows CAS SSO flow to provide user IDs composed of numbers only. - These identifiers will be prefixed by the letter "u" by default. - The prefix can be configured using the "numeric_ids_prefix" option. - Be careful to choose the prefix correctly to avoid any possible conflicts - (e.g. user 1234 becomes u1234 when a user u1234 already exists). -* `numeric_ids_prefix`: the prefix you wish to add in front of a numeric user ID - when the "allow_numeric_ids" option is set to "true". - By default, the prefix is the letter "u" and only alphanumeric characters are allowed. +*(object)* Enable Central Authentication Service (CAS) for registration and login. - *Added in Synapse 1.93.0.* +This setting has the following sub-options: + +* `enabled` (boolean): Set this to true to enable authorization against a CAS server. Defaults to `false`. + +* `idp_name` (string): A user-facing name for this identity provider, which is used to offer the user a choice of login mechanisms. + +* `idp_icon` (string|null): An optional icon for this identity provider, which is presented by clients and Synapse's own IdP picker page. If given, must be an MXC URI of the format `mxc:///`. (An easy way to obtain such an MXC URI is to upload an image to an (unencrypted) room and then copy the URL from the source of the event.) Defaults to `null`. + +* `idp_brand` (string|null): An optional brand for this identity provider, allowing clients to style the login flow according to the identity provider in question. See the [spec](https://spec.matrix.org/latest/) for possible options here. Defaults to `null`. + +* `server_url` (string): The URL of the CAS authorization endpoint. + +* `protocol_version` (integer|null): The CAS protocol version. (Version 3 is required if you want to use `required_attributes`). Defaults to `null`. + +* `displayname_attribute` (string|null): The attribute of the CAS response to use as the display name. If no name is given here, no displayname will be set. Defaults to `null`. + +* `required_attributes` (object): It is possible to configure Synapse to only allow logins if CAS attributes match particular values. All of the keys given below must exist and the values must match the given value. Alternately if the given value is `None` then any value is allowed (the attribute just must exist). All of the listed attributes must match for the login to be permitted. Defaults to `{}`. + +* `enable_registration` (boolean): Set to `false` to disable automatic registration of new users. This allows the CAS SSO flow to be limited to sign in only, rather than automatically registering users that have a valid SSO login but do not have a pre-registered account. Defaults to `true`. + +* `allow_numeric_ids` (boolean): Set to `true` allow numeric user IDs. This allows CAS SSO flow to provide user IDs composed of numbers only. These identifiers will be prefixed by the letter "u" by default. The prefix can be configured using the `numeric_ids_prefix` option. Be careful to choose the prefix correctly to avoid any possible conflicts (e.g. user 1234 becomes u1234 when a user u1234 already exists). Defaults to `false`. + +* `numeric_ids_prefix` (string): The prefix you wish to add in front of a numeric user ID when the `allow_numeric_ids` option is set to `true`. Only alphanumeric characters are allowed. + + *Added in Synapse 1.93.0.* + + Defaults to `"u"`. Example configuration: ```yaml cas_config: enabled: true - server_url: "https://cas-server.com" + server_url: https://cas-server.com protocol_version: 3 displayname_attribute: name required_attributes: - userGroup: "staff" + userGroup: staff department: None enable_registration: true allow_numeric_ids: true - numeric_ids_prefix: "numericuser" + numeric_ids_prefix: numericuser ``` --- ### `sso` -Additional settings to use with single-sign on systems such as OpenID Connect, -SAML2 and CAS. +*(object)* Additional settings to use with single-sign on systems such as OpenID Connect, SAML2 and CAS. -Server admins can configure custom templates for pages related to SSO. See -[here](../../templates.md) for more information. +Server admins can configure custom templates for pages related to SSO. See [here](../../templates.md) for more information. -Options include: -* `client_whitelist`: A list of client URLs which are whitelisted so that the user does not - have to confirm giving access to their account to the URL. Any client - whose URL starts with an entry in the following list will not be subject - to an additional confirmation step after the SSO login is completed. - WARNING: An entry such as "https://my.client" is insecure, because it - will also match "https://my.client.evil.site", exposing your users to - phishing attacks from evil.site. To avoid this, include a slash after the - hostname: "https://my.client/". - The login fallback page (used by clients that don't natively support the - required login flows) is whitelisted in addition to any URLs in this list. - By default, this list contains only the login fallback page. -* `update_profile_information`: Use this setting to keep a user's profile fields in sync with information from - the identity provider. Currently only syncing the displayname is supported. Fields - are checked on every SSO login, and are updated if necessary. - Note that enabling this option will override user profile information, - regardless of whether users have opted-out of syncing that - information when first signing in. Defaults to false. +This setting has the following sub-options: +* `client_whitelist` (array|null): A list of client URLs which are whitelisted so that the user does not have to confirm giving access to their account to the URL. Any client whose URL starts with an entry in the following list will not be subject to an additional confirmation step after the SSO login is completed. + + WARNING: An entry such as "https://my.client" is insecure, because it will also match "https://my.client.evil.site", exposing your users to phishing attacks from evil.site. To avoid this, include a slash after the hostname: "https://my.client/". + + The login fallback page (used by clients that don't natively support the required login flows) is whitelisted in addition to any URLs in this list. By default, this list contains only the login fallback page. + + Defaults to `null`. + +* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Currently only syncing the displayname is supported. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Defaults to `false`. Example configuration: ```yaml sso: - client_whitelist: - - https://riot.im/develop - - https://my.custom.client/ - update_profile_information: true + client_whitelist: + - https://riot.im/develop + - https://my.custom.client/ + update_profile_information: true ``` --- ### `jwt_config` -JSON web token integration. The following settings can be used to make -Synapse JSON web tokens for authentication, instead of its internal -password database. +*(object)* JSON web token integration. The following settings can be used to make Synapse JSON web tokens for authentication, instead of its internal password database. -Each JSON Web Token needs to contain a "sub" (subject) claim, which is -used as the localpart of the mxid. +Each JSON Web Token needs to contain a "sub" (subject) claim, which is used as the localpart of the mxid. -Additionally, the expiration time ("exp"), not before time ("nbf"), -and issued at ("iat") claims are validated if present. +Additionally, the expiration time ("exp"), not before time ("nbf"), and issued at ("iat") claims are validated if present. -Note that this is a non-standard login type and client support is -expected to be non-existent. +Note that this is a non-standard login type and client support is expected to be non-existent. See [here](../../jwt.md) for more. -Additional sub-options for this setting include: -* `enabled`: Set to true to enable authorization using JSON web - tokens. Defaults to false. -* `secret`: This is either the private shared secret or the public key used to - decode the contents of the JSON web token. Required if `enabled` is set to true. -* `algorithm`: The algorithm used to sign (or HMAC) the JSON web token. - Supported algorithms are listed - [here (section JWS)](https://docs.authlib.org/en/latest/specs/rfc7518.html). - Required if `enabled` is set to true. -* `subject_claim`: Name of the claim containing a unique identifier for the user. - Optional, defaults to `sub`. -* `display_name_claim`: Name of the claim containing the display name for the user. Optional. - If provided, the display name will be set to the value of this claim upon first login. -* `issuer`: The issuer to validate the "iss" claim against. Optional. If provided the - "iss" claim will be required and validated for all JSON web tokens. -* `audiences`: A list of audiences to validate the "aud" claim against. Optional. - If provided the "aud" claim will be required and validated for all JSON web tokens. - Note that if the "aud" claim is included in a JSON web token then - validation will fail without configuring audiences. +This setting has the following sub-options: + +* `enabled` (boolean): Set to true to enable authorization using JSON web tokens. Defaults to `false`. + +* `secret` (string): This is either the private shared secret or the public key used to decode the contents of the JSON web token. Required if `enabled` is set to true. + +* `algorithm` (string): The algorithm used to sign (or HMAC) the JSON web token. Supported algorithms are listed [here (section JWS)](https://docs.authlib.org/en/latest/specs/rfc7518.html). Required if `enabled` is set to true. + +* `subject_claim` (string|null): Name of the claim containing a unique identifier for the user. Defaults to `"sub"`. + +* `display_name_claim` (string|null): Name of the claim containing the display name for the user. If provided, the display name will be set to the value of this claim upon first login. Defaults to `null`. + +* `issuer` (string|null): The issuer to validate the "iss" claim against. If provided the "iss" claim will be required and validated for all JSON web tokens. Defaults to `null`. + +* `audiences` (array|null): A list of audiences to validate the "aud" claim against. If provided the "aud" claim will be required and validated for all JSON web tokens. Note that if the "aud" claim is included in a JSON web token then validation will fail without configuring audiences. Defaults to `null`. Example configuration: ```yaml jwt_config: - enabled: true - secret: "provided-by-your-issuer" - algorithm: "provided-by-your-issuer" - subject_claim: "name_of_claim" - display_name_claim: "name_of_claim" - issuer: "provided-by-your-issuer" - audiences: - - "provided-by-your-issuer" + enabled: true + secret: provided-by-your-issuer + algorithm: provided-by-your-issuer + subject_claim: name_of_claim + display_name_claim: name_of_claim + issuer: provided-by-your-issuer + audiences: + - provided-by-your-issuer ``` --- ### `password_config` -Use this setting to enable password-based logins. +*(object)* Use this setting to enable password-based logins. This setting has the following sub-options: -* `enabled`: Defaults to true. - Set to false to disable password authentication. - Set to `only_for_reauth` to allow users with existing passwords to use them - to reauthenticate (not log in), whilst preventing new users from setting passwords. -* `localdb_enabled`: Set to false to disable authentication against the local password - database. This is ignored if `enabled` is false, and is only useful - if you have other `password_providers`. Defaults to true. -* `pepper`: Set the value here to a secret random string for extra security. - DO NOT CHANGE THIS AFTER INITIAL SETUP! -* `policy`: Define and enforce a password policy, such as minimum lengths for passwords, etc. - Each parameter is optional. This is an implementation of MSC2000. Parameters are as follows: - * `enabled`: Defaults to false. Set to true to enable. - * `minimum_length`: Minimum accepted length for a password. Defaults to 0. - * `require_digit`: Whether a password must contain at least one digit. - Defaults to false. - * `require_symbol`: Whether a password must contain at least one symbol. - A symbol is any character that's not a number or a letter. Defaults to false. - * `require_lowercase`: Whether a password must contain at least one lowercase letter. - Defaults to false. - * `require_uppercase`: Whether a password must contain at least one uppercase letter. - Defaults to false. +* `enabled` (boolean|string): Set to false to disable password authentication. Set to `only_for_reauth` to allow users with existing passwords to use them to reauthenticate (not log in), whilst preventing new users from setting passwords. Defaults to `true`. + +* `localdb_enabled` (boolean): Set to false to disable authentication against the local password database. This is ignored if `enabled` is false, and is only useful if you have other `password_providers`. Defaults to `true`. + +* `pepper` (string|null): Set the value here to a secret random string for extra security. DO NOT CHANGE THIS AFTER INITIAL SETUP! Defaults to `null`. + +* `policy` (object): Define and enforce a password policy, such as minimum lengths for passwords, etc. This is an implementation of MSC2000. + + This setting has the following sub-options: + + * `enabled` (boolean): Set to true to enable. Defaults to `false`. + + * `minimum_length` (integer): Minimum accepted length for a password. Defaults to `0`. + + * `require_digit` (boolean): Whether a password must contain at least one digit. Defaults to `false`. + + * `require_symbol` (boolean): Whether a password must contain at least one symbol. A symbol is any character that's not a number or a letter. Defaults to `false`. + + * `require_lowercase` (boolean): Whether a password must contain at least one lowercase letter. Defaults to `false`. + + * `require_uppercase` (boolean): Whether a password must contain at least one uppercase letter. Defaults to `false`. Example configuration: ```yaml password_config: - enabled: false - localdb_enabled: false - pepper: "EVEN_MORE_SECRET" - - policy: - enabled: true - minimum_length: 15 - require_digit: true - require_symbol: true - require_lowercase: true - require_uppercase: true + enabled: false + localdb_enabled: false + pepper: EVEN_MORE_SECRET + policy: + enabled: true + minimum_length: 15 + require_digit: true + require_symbol: true + require_lowercase: true + require_uppercase: true ``` --- ## Push -Configuration settings related to push notifications + +Configuration settings related to push notifications. --- ### `push` -This setting defines options for push notifications. +*(object)* This setting defines options for push notifications. -This option has a number of sub-options. They are as follows: -* `enabled`: Enables or disables push notification calculation. Note, disabling this will also - stop unread counts being calculated for rooms. This mode of operation is intended - for homeservers which may only have bots or appservice users connected, or are otherwise - not interested in push/unread counters. This is enabled by default. -* `include_content`: Clients requesting push notifications can either have the body of - the message sent in the notification poke along with other details - like the sender, or just the event ID and room ID (`event_id_only`). - If clients choose the to have the body sent, this option controls whether the - notification request includes the content of the event (other details - like the sender are still included). If `event_id_only` is enabled, it - has no effect. - For modern android devices the notification content will still appear - because it is loaded by the app. iPhone, however will send a - notification saying only that a message arrived and who it came from. - Defaults to true. Set to false to only include the event ID and room ID in push notification payloads. -* `group_unread_count_by_room: false`: When a push notification is received, an unread count is also sent. - This number can either be calculated as the number of unread messages for the user, or the number of *rooms* the - user has unread messages in. Defaults to true, meaning push clients will see the number of - rooms with unread messages in them. Set to false to instead send the number - of unread messages. -* `jitter_delay`: Delays push notifications by a random amount up to the given - duration. Useful for mitigating timing attacks. Optional, defaults to no - delay. _Added in Synapse 1.84.0._ +This setting has the following sub-options: + +* `enabled` (boolean): Enables or disables push notification calculation. Note, disabling this will also stop unread counts being calculated for rooms. This mode of operation is intended for homeservers which may only have bots or appservice users connected, or are otherwise not interested in push/unread counters. Defaults to `true`. + +* `include_content` (boolean): Clients requesting push notifications can either have the body of the message sent in the notification poke along with other details like the sender, or just the event ID and room ID (`event_id_only`). If clients choose to have the body sent, this option controls whether the notification request includes the content of the event (other details like the sender are still included). If `event_id_only` is enabled, it has no effect. For modern Android devices the notification content will still appear because it is loaded by the app. iPhone, however will send a notification saying only that a message arrived and who it came from. Set to false to only include the event ID and room ID in push notification payloads. Defaults to `true`. + +* `group_unread_count_by_room` (boolean): When a push notification is received, an unread count is also sent. This number can either be calculated as the number of unread messages for the user, or the number of *rooms* the user has unread messages in. If true, push clients will see the number of rooms with unread messages in them. Set to false to instead send the number of unread messages. Defaults to `true`. + +* `jitter_delay` (duration): Delays push notifications by a random amount up to the given duration. Useful for mitigating timing attacks. Optional. + + _Added in Synapse 1.84.0._ + + Defaults to `"0s"`. Example configuration: ```yaml @@ -3857,29 +3857,27 @@ push: enabled: true include_content: false group_unread_count_by_room: false - jitter_delay: "10s" + jitter_delay: 10s ``` --- ## Rooms + Config options relating to rooms. --- ### `encryption_enabled_by_default_for_room_type` -Controls whether locally-created rooms should be end-to-end encrypted by -default. +*(string)* Controls whether locally-created rooms should be end-to-end encrypted by default. Possible options are "all", "invite", and "off". They are defined as: * "all": any locally-created room -* "invite": any room created with the `private_chat` or `trusted_private_chat` - room creation presets +* "invite": any room created with the `private_chat` or `trusted_private_chat` room creation presets * "off": this option will take no effect -The default value is "off". +Note that this option will only affect rooms created after it is set. It will also not affect rooms created by other servers. -Note that this option will only affect rooms created after it is set. It -will also not affect rooms created by other servers. +Defaults to `"off"`. Example configuration: ```yaml @@ -3888,72 +3886,67 @@ encryption_enabled_by_default_for_room_type: invite --- ### `user_directory` -This setting defines options related to the user directory. +*(object)* This setting defines options related to the user directory. -This option has the following sub-options: -* `enabled`: Defines whether users can search the user directory. If false then - empty responses are returned to all queries. Defaults to true. -* `search_all_users`: Defines whether to search all users visible to your homeserver at the time the search is performed. - If set to true, will return all users known to the homeserver matching the search query. - If false, search results will only contain users - visible in public rooms and users sharing a room with the requester. - Defaults to false. +This setting has the following sub-options: - NB. If you set this to true, and the last time the user_directory search - indexes were (re)built was before Synapse 1.44, you'll have to - rebuild the indexes in order to search through all known users. +* `enabled` (boolean): Defines whether users can search the user directory. If `false` then empty responses are returned to all queries. - These indexes are built the first time Synapse starts; admins can - manually trigger a rebuild via the API following the instructions - [for running background updates](../administration/admin_api/background_updates.md#run), - set to true to return search results containing all known users, even if that - user does not share a room with the requester. -* `prefer_local_users`: Defines whether to prefer local users in search query results. - If set to true, local users are more likely to appear above remote users when searching the - user directory. Defaults to false. -* `show_locked_users`: Defines whether to show locked users in search query results. Defaults to false. + *Warning: While the homeserver may determine which subset of users are searched, the Matrix specification requires homeservers to include (at minimum) users visible in public rooms and users sharing a room with the requester. Using `false` improves performance but violates this requirement.* + + Defaults to `true`. + +* `search_all_users` (boolean): Defines whether to search all users visible to your homeserver at the time the search is performed. If set to true, will return all users known to the homeserver matching the search query. If false, search results will only contain users visible in public rooms and users sharing a room with the requester. + + NB. If you set this to true, and the last time the user_directory search indexes were (re)built was before Synapse 1.44, you'll have to rebuild the indexes in order to search through all known users. + + These indexes are built the first time Synapse starts; admins can manually trigger a rebuild via the API following the instructions [for running background updates](../administration/admin_api/background_updates.md#run), set to true to return search results containing all known users, even if that user does not share a room with the requester. + + Defaults to `false`. + +* `prefer_local_users` (boolean): Defines whether to prefer local users in search query results. If set to true, local users are more likely to appear above remote users when searching the user directory. Defaults to `false`. + +* `exclude_remote_users` (boolean): If set to true, the search will only return local users. Defaults to `false`. + +* `show_locked_users` (boolean): Defines whether to show locked users in search query results. Defaults to `false`. Example configuration: ```yaml user_directory: - enabled: false - search_all_users: true - prefer_local_users: true - show_locked_users: true + enabled: false + search_all_users: true + prefer_local_users: true + exclude_remote_users: false + show_locked_users: true ``` --- ### `user_consent` -For detailed instructions on user consent configuration, see [here](../../consent_tracking.md). +*(object)* For detailed instructions on user consent configuration, see [here](../../consent_tracking.md). -Parts of this section are required if enabling the `consent` resource under -[`listeners`](#listeners), in particular `template_dir` and `version`. +Parts of this section are required if enabling the `consent` resource under [`listeners`](#listeners), in particular `template_dir` and `version`. -* `template_dir`: gives the location of the templates for the HTML forms. - This directory should contain one subdirectory per language (eg, `en`, `fr`), - and each language directory should contain the policy document (named as - .html) and a success page (success.html). +This setting has the following sub-options: -* `version`: specifies the 'current' version of the policy document. It defines - the version to be served by the consent resource if there is no 'v' - parameter. +* `template_dir` (string): Gives the location of the templates for the HTML forms. This directory should contain one subdirectory per language (eg, `en`, `fr`), and each language directory should contain the policy document (named as .html) and a success page (success.html). -* `server_notice_content`: if enabled, will send a user a "Server Notice" - asking them to consent to the privacy policy. The [`server_notices` section](#server_notices) - must also be configured for this to work. Notices will *not* be sent to - guest users unless `send_server_notice_to_guests` is set to true. +* `version` (number): Specifies the "current" version of the policy document. It defines the version to be served by the consent resource if there is no `v` parameter. -* `block_events_error`, if set, will block any attempts to send events - until the user consents to the privacy policy. The value of the setting is - used as the text of the error. +* `server_notice_content` (object): If enabled, will send a user a "Server Notice" asking them to consent to the privacy policy. The [`server_notices` section](#server_notices) must also be configured for this to work. Notices will *not* be sent to guest users unless `send_server_notice_to_guests` is set to true. -* `require_at_registration`, if enabled, will add a step to the registration - process, similar to how captcha works. Users will be required to accept the - policy before their account is created. + This setting has the following sub-options: -* `policy_name` is the display name of the policy users will see when registering - for an account. Has no effect unless `require_at_registration` is enabled. - Defaults to "Privacy Policy". + * `msgtype` (string): Message type of the notice event. + + * `body` (string): Message template for the server notice event body. + +* `send_server_notice_to_guests` (boolean): Send server notices to guest users, too. Defaults to `false`. + +* `block_events_error` (string|null): If set, will block any attempts to send events until the user consents to the privacy policy. The value of the setting is used as the text of the error. Defaults to `null`. + +* `require_at_registration` (boolean): If enabled, will add a step to the registration process, similar to how captcha works. Users will be required to accept the policy before their account is created. + +* `policy_name` (string): Human-readable name of the privacy policy. Defaults to `"Privacy Policy"`. Example configuration: ```yaml @@ -3962,25 +3955,22 @@ user_consent: version: 1.0 server_notice_content: msgtype: m.text - body: >- - To continue using this homeserver you must review and agree to the - terms and conditions at %(consent_uri)s + body: To continue using this homeserver you must review and agree to the terms + and conditions at %(consent_uri)s send_server_notice_to_guests: true - block_events_error: >- - To continue using this homeserver you must review and agree to the - terms and conditions at %(consent_uri)s + block_events_error: To continue using this homeserver you must review and agree + to the terms and conditions at %(consent_uri)s require_at_registration: false policy_name: Privacy Policy ``` --- ### `stats` -Settings for local room and user statistics collection. See [here](../../room_and_user_statistics.md) -for more. +*(object)* Settings for local room and user statistics collection. See [here](../../room_and_user_statistics.md) for more. -* `enabled`: Set to false to disable room and user statistics. Note that doing - so may cause certain features (such as the room directory) not to work - correctly. Defaults to true. +This setting has the following sub-options: + +* `enabled` (boolean): Set to false to disable room and user statistics. Note that doing so may cause certain features (such as the room directory) not to work correctly. Defaults to `true`. Example configuration: ```yaml @@ -3990,42 +3980,53 @@ stats: --- ### `server_notices` -Use this setting to enable a room which can be used to send notices -from the server to users. It is a special room which users cannot leave; notices -in the room come from a special "notices" user id. +*(object)* Use this setting to enable a room which can be used to send notices from the server to users. It is a special room which users cannot leave; notices in the room come from a special "notices" user id. -If you use this setting, you *must* define the `system_mxid_localpart` -sub-setting, which defines the id of the user which will be used to send the -notices. - -Sub-options for this setting include: -* `system_mxid_display_name`: set the display name of the "notices" user -* `system_mxid_avatar_url`: set the avatar for the "notices" user -* `room_name`: set the room name of the server notices room -* `room_avatar_url`: optional string. The room avatar to use for server notice rooms. If set to the empty string `""`, notice rooms will not be given an avatar. Defaults to the empty string. _Added in Synapse 1.99.0._ -* `room_topic`: optional string. The topic to use for server notice rooms. If set to the empty string `""`, notice rooms will not be given a topic. Defaults to the empty string. _Added in Synapse 1.99.0._ -* `auto_join`: boolean. If true, the user will be automatically joined to the room instead of being invited. - Defaults to false. _Added in Synapse 1.98.0._ +If you use this setting, you *must* define the `system_mxid_localpart` sub-setting, which defines the id of the user which will be used to send the notices. Note that the name, topic and avatar of existing server notice rooms will only be updated when a new notice event is sent. +This setting has the following sub-options: + +* `system_mxid_display_name` (string): Display name of the "notices" user. Defaults to `"Notices"`. + +* `system_mxid_avatar_url` (string|null): Avatar for the "notices" user. Defaults to `null`. + +* `room_name` (string): Room name of the server notices room. Defaults to `"Server Notices"`. + +* `room_avatar_url` (string|null): Room avatar to use for server notice rooms. If set to the empty string `""`, notice rooms will not be given an avatar. + + _Added in Synapse 1.99.0._ + + Defaults to `null`. + +* `room_topic` (string|null): Topic to use for server notice rooms. If set to the empty string `""`, notice rooms will not be given a topic. Defaults to the empty string. + + _Added in Synapse 1.99.0._ + + Defaults to `null`. + +* `auto_join` (boolean): If true, the user will be automatically joined to the room instead of being invited. + + _Added in Synapse 1.98.0._ + + Defaults to `false`. + Example configuration: ```yaml server_notices: system_mxid_localpart: notices - system_mxid_display_name: "Server Notices" - system_mxid_avatar_url: "mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ" - room_name: "Server Notices" - room_avatar_url: "mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ" - room_topic: "Room used by your server admin to notice you of important information" + system_mxid_display_name: Server Notices + system_mxid_avatar_url: mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ + room_name: Server Notices + room_avatar_url: mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ + room_topic: Room used by your server admin to notice you of important information auto_join: true ``` --- ### `enable_room_list_search` -Set to false to disable searching the public room list. When disabled -blocks searching local and remote room lists for local and remote -users by always returning an empty list for all queries. Defaults to true. +*(boolean)* Set to false to disable searching the public room list. When disabled blocks searching local and remote room lists for local and remote users by always returning an empty list for all queries. Defaults to `true`. Example configuration: ```yaml @@ -4034,326 +4035,287 @@ enable_room_list_search: false --- ### `alias_creation_rules` -The `alias_creation_rules` option allows server admins to prevent unwanted -alias creation on this server. +*(array|null)* The `alias_creation_rules` option allows server admins to prevent unwanted alias creation on this server. -This setting is an optional list of 0 or more rules. By default, no list is -provided, meaning that all alias creations are permitted. +This setting is an optional list of 0 or more rules. By default, no list is provided, meaning that all alias creations are permitted. -Otherwise, requests to create aliases are matched against each rule in order. -The first rule that matches decides if the request is allowed or denied. If no -rule matches, the request is denied. In particular, this means that configuring -an empty list of rules will deny every alias creation request. +Otherwise, requests to create aliases are matched against each rule in order. The first rule that matches decides if the request is allowed or denied. If no rule matches, the request is denied. In particular, this means that configuring an empty list of rules will deny every alias creation request. -Each rule is a YAML object containing four fields, each of which is an optional string: +Each of the glob patterns is optional, defaulting to `*` ("match anything"). Note that the patterns match against fully qualified IDs, e.g. against `@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead of `alice`, `room` and `abcedgghijk`. -* `user_id`: a glob pattern that matches against the creator of the alias. -* `alias`: a glob pattern that matches against the alias being created. -* `room_id`: a glob pattern that matches against the room ID the alias is being pointed at. -* `action`: either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. +Each rule is a YAML object containing four fields, each of which is an optional string -Each of the glob patterns is optional, defaulting to `*` ("match anything"). -Note that the patterns match against fully qualified IDs, e.g. against -`@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead -of `alice`, `room` and `abcedgghijk`. +Defaults to `null`. -Example configuration: +Options for each entry include: +* `user_id` (string|null): Glob pattern that matches against the creator of the alias. + +* `alias` (string|null): Glob pattern that matches against the alias being created. + +* `room_id` (string|null): Glob pattern that matches against the room ID the alias is being pointed at. + +* `action` (string): Either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. + +Example configurations: ```yaml -# No rule list specified. All alias creations are allowed. -# This is the default behaviour. -alias_creation_rules: +alias_creation_rules: null ``` ```yaml -# A list of one rule which allows everything. -# This has the same effect as the previous example. alias_creation_rules: - - "action": "allow" +- action: allow ``` ```yaml -# An empty list of rules. All alias creations are denied. alias_creation_rules: [] ``` ```yaml -# A list of one rule which denies everything. -# This has the same effect as the previous example. alias_creation_rules: - - "action": "deny" +- action: deny ``` ```yaml -# Prevent a specific user from creating aliases. -# Allow other users to create any alias alias_creation_rules: - - user_id: "@bad_user:example.com" - action: deny - - - action: allow +- user_id: '@bad_user:example.com' + action: deny +- action: allow ``` ```yaml -# Prevent aliases being created which point to a specific room. alias_creation_rules: - - room_id: "!forbiddenRoom:example.com" - action: deny - - - action: allow +- room_id: '!forbiddenRoom:example.com' + action: deny +- action: allow ``` - --- ### `room_list_publication_rules` -The `room_list_publication_rules` option allows server admins to prevent -unwanted entries from being published in the public room list. +*(array|null)* The `room_list_publication_rules` option allows server admins to prevent unwanted entries from being published in the public room list. -The format of this option is the same as that for -[`alias_creation_rules`](#alias_creation_rules): an optional list of 0 or more -rules. By default, no list is provided, meaning that all rooms may be -published to the room list. +The format of this option is the same as that for [`alias_creation_rules`](#alias_creation_rules): an optional list of 0 or more rules. By default, no list is provided, meaning that no one may publish to the room list (except server admins). -Otherwise, requests to publish a room are matched against each rule in order. -The first rule that matches decides if the request is allowed or denied. If no -rule matches, the request is denied. In particular, this means that configuring -an empty list of rules will deny every alias creation request. +Otherwise, requests to publish a room are matched against each rule in order. The first rule that matches decides if the request is allowed or denied. If no rule matches, the request is denied. In particular, this means that configuring an empty list of rules will deny every alias creation request. -Requests to create a public (public as in published to the room directory) room which violates -the configured rules will result in the room being created but not published to the room directory. +Requests to create a public (public as in published to the room directory) room which violates the configured rules will result in the room being created but not published to the room directory. -Each rule is a YAML object containing four fields, each of which is an optional string: +Each of the glob patterns is optional, defaulting to `*` ("match anything"). Note that the patterns match against fully qualified IDs, e.g. against `@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead of `alice`, `room` and `abcedgghijk`. -* `user_id`: a glob pattern that matches against the user publishing the room. -* `alias`: a glob pattern that matches against one of published room's aliases. +Each rule is a YAML object containing four fields, each of which is an optional string. + +_Changed in Synapse 1.126.0: The default was changed to deny publishing to the room list by default_ + +Defaults to `null`. + +Options for each entry include: + +* `user_id` (string|null): Glob pattern that matches against the user publishing the room. + +* `alias` (string|null): Glob pattern that matches against one of published room's aliases. - If the room has no aliases, the alias match fails unless `alias` is unspecified or `*`. - If the room has exactly one alias, the alias match succeeds if the `alias` pattern matches that alias. - If the room has two or more aliases, the alias match succeeds if the pattern matches at least one of the aliases. -* `room_id`: a glob pattern that matches against the room ID of the room being published. -* `action`: either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. -Each of the glob patterns is optional, defaulting to `*` ("match anything"). -Note that the patterns match against fully qualified IDs, e.g. against -`@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead -of `alice`, `room` and `abcedgghijk`. +* `room_id` (string|null): Glob pattern that matches against the room ID of the room being published. +* `action` (string): Either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. -Example configuration: - +Example configurations: ```yaml -# No rule list specified. Anyone may publish any room to the public list. -# This is the default behaviour. -room_list_publication_rules: +room_list_publication_rules: null ``` ```yaml -# A list of one rule which allows everything. -# This has the same effect as the previous example. room_list_publication_rules: - - "action": "allow" +- action: deny ``` ```yaml -# An empty list of rules. No-one may publish to the room list. room_list_publication_rules: [] ``` ```yaml -# A list of one rule which denies everything. -# This has the same effect as the previous example. room_list_publication_rules: - - "action": "deny" +- action: allow ``` ```yaml -# Prevent a specific user from publishing rooms. -# Allow other users to publish anything. room_list_publication_rules: - - user_id: "@bad_user:example.com" - action: deny - - - action: allow +- user_id: '@bad_user:example.com' + action: deny +- action: allow ``` ```yaml -# Prevent publication of a specific room. room_list_publication_rules: - - room_id: "!forbiddenRoom:example.com" - action: deny - - - action: allow +- room_id: '!forbiddenRoom:example.com' + action: deny +- action: allow ``` ```yaml -# Prevent publication of rooms with at least one alias containing the word "potato". room_list_publication_rules: - - alias: "#*potato*:example.com" - action: deny - - - action: allow +- alias: '#*potato*:example.com' + action: deny +- action: allow ``` - --- ### `default_power_level_content_override` -The `default_power_level_content_override` option controls the default power -levels for rooms. +*(object)* The `default_power_level_content_override` option controls the default power levels for rooms. -Useful if you know that your users need special permissions in rooms -that they create (e.g. to send particular types of state events without -needing an elevated power level). This takes the same shape as the -`power_level_content_override` parameter in the /createRoom API, but -is applied before that parameter. +Useful if you know that your users need special permissions in rooms that they create (e.g. to send particular types of state events without needing an elevated power level). This takes the same shape as the `power_level_content_override` parameter in the /createRoom API, but is applied before that parameter. -Note that each key provided inside a preset (for example `events` in the example -below) will overwrite all existing defaults inside that key. So in the example -below, newly-created private_chat rooms will have no rules for any event types -except `com.example.foo`. - -Example configuration: -```yaml -default_power_level_content_override: - private_chat: { "events": { "com.example.foo" : 0 } } - trusted_private_chat: null - public_chat: null -``` +Note that each key provided inside a preset (for example `events` in the example below) will overwrite all existing defaults inside that key. So in Example #1, newly-created private_chat rooms will have no rules for any event types except `com.example.foo`. The default power levels for each preset are: + ```yaml "m.room.name": 50 "m.room.power_levels": 100 "m.room.history_visibility": 100 "m.room.canonical_alias": 50 "m.room.avatar": 50 -"m.room.tombstone": 100 +"m.room.tombstone": 100 (150 if MSC4289 is used) "m.room.server_acl": 100 "m.room.encryption": 100 ``` -So a complete example where the default power-levels for a preset are maintained -but the power level for a new key is set is: +In Example #2 the default power-levels for a preset are maintained, but the power level for a new key is set. + +Defaults to `{}`. + +Example configurations: ```yaml default_power_level_content_override: - private_chat: + private_chat: events: - "com.example.foo": 0 - "m.room.name": 50 - "m.room.power_levels": 100 - "m.room.history_visibility": 100 - "m.room.canonical_alias": 50 - "m.room.avatar": 50 - "m.room.tombstone": 100 - "m.room.server_acl": 100 - "m.room.encryption": 100 - trusted_private_chat: null - public_chat: null + com.example.foo: 0 + trusted_private_chat: null + public_chat: null ``` +```yaml +default_power_level_content_override: + private_chat: + events: + com.example.foo: 0 + m.room.name: 50 + m.room.power_levels: 100 + m.room.history_visibility: 100 + m.room.canonical_alias: 50 + m.room.avatar: 50 + m.room.tombstone: 100 + m.room.server_acl: 100 + m.room.encryption: 100 + trusted_private_chat: null + public_chat: null +``` --- ### `forget_rooms_on_leave` -Set to true to automatically forget rooms for users when they leave them, either -normally or via a kick or ban. Defaults to false. +*(boolean)* Set to true to automatically forget rooms for users when they leave them, either normally or via a kick or ban. Defaults to `false`. Example configuration: ```yaml -forget_rooms_on_leave: false +forget_rooms_on_leave: true ``` --- ### `exclude_rooms_from_sync` -A list of rooms to exclude from sync responses. This is useful for server -administrators wishing to group users into a room without these users being able -to see it from their client. -By default, no room is excluded. +*(array)* A list of rooms to exclude from sync responses. This is useful for server administrators wishing to group users into a room without these users being able to see it from their client. Defaults to `[]`. Example configuration: ```yaml exclude_rooms_from_sync: - - "!foo:example.com" +- '!foo:example.com' ``` - --- ## Opentracing + Configuration options related to Opentracing support. --- ### `opentracing` -These settings enable and configure opentracing, which implements distributed tracing. -This allows you to observe the causal chains of events across servers -including requests, key lookups etc., across any server running -synapse or any other services which support opentracing -(specifically those implemented with Jaeger). +*(object)* These settings enable and configure opentracing, which implements distributed tracing. This allows you to observe the causal chains of events across servers including requests, key lookups etc., across any server running synapse or any other services which support opentracing (specifically those implemented with Jaeger). -Sub-options include: -* `enabled`: whether tracing is enabled. Set to true to enable. Disabled by default. -* `homeserver_whitelist`: The list of homeservers we wish to send and receive span contexts and span baggage. - See [here](../../opentracing.md) for more. - This is a list of regexes which are matched against the `server_name` of the homeserver. - By default, it is empty, so no servers are matched. -* `force_tracing_for_users`: # A list of the matrix IDs of users whose requests will always be traced, - even if the tracing system would otherwise drop the traces due to probabilistic sampling. - By default, the list is empty. -* `jaeger_config`: Jaeger can be configured to sample traces at different rates. - All configuration options provided by Jaeger can be set here. Jaeger's configuration is - mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/). +This setting has the following sub-options: + +* `enabled` (boolean): Whether tracing is enabled. Set to true to enable. Defaults to `false`. + +* `homeserver_whitelist` (array): The list of homeservers we wish to send and receive span contexts and span baggage. See [here](../../opentracing.md) for more. This is a list of regexes which are matched against the `server_name` of the homeserver. If the list is empty, no servers are matched. Defaults to `[]`. + +* `force_tracing_for_users` (array): A list of the matrix IDs of users whose requests will always be traced, even if the tracing system would otherwise drop the traces due to probabilistic sampling. Defaults to `[]`. + +* `jaeger_config` (object): Jaeger can be configured to sample traces at different rates. All configuration options provided by Jaeger can be set here. Jaeger's configuration is mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/). Defaults to `{}`. Example configuration: ```yaml opentracing: - enabled: true - homeserver_whitelist: - - ".*" - force_tracing_for_users: - - "@user1:server_name" - - "@user2:server_name" - - jaeger_config: - sampler: - type: const - param: 1 - logging: - false + enabled: true + homeserver_whitelist: + - .* + force_tracing_for_users: + - '@user1:server_name' + - '@user2:server_name' + jaeger_config: + sampler: + type: const + param: 1 + logging: false ``` --- ## Coordinating workers -Configuration options related to workers which belong in the main config file -(usually called `homeserver.yaml`). -A Synapse deployment can scale horizontally by running multiple Synapse processes -called _workers_. Incoming requests are distributed between workers to handle higher -loads. Some workers are privileged and can accept requests from other workers. + +Configuration options related to workers which belong in the main config file (usually called `homeserver.yaml`). A Synapse deployment can scale horizontally by running multiple Synapse processes called _workers_. Incoming requests are distributed between workers to handle higher loads. Some workers are privileged and can accept requests from other workers. As a result, the worker configuration is divided into two parts. -1. The first part (in this section of the manual) defines which shardable tasks - are delegated to privileged workers. This allows unprivileged workers to make - requests to a privileged worker to act on their behalf. -1. [The second part](#individual-worker-configuration) - controls the behaviour of individual workers in isolation. +1. The first part (in this section of the manual) defines which shardable tasks are delegated to privileged workers. This allows unprivileged workers to make requests to a privileged worker to act on their behalf. +2. [The second part](#individual-worker-configuration) controls the behaviour of individual workers in isolation. For guidance on setting up workers, see the [worker documentation](../../workers.md). --- ### `worker_replication_secret` -A shared secret used by the replication APIs on the main process to authenticate -HTTP requests from workers. +*(string|null)* A shared secret used by the replication APIs on the main process to authenticate HTTP requests from workers. -The default, this value is omitted (equivalently `null`), which means that -traffic between the workers and the main process is not authenticated. +If unset or null, traffic between the workers and the main process is not authenticated. + +Replacing an existing `worker_replication_secret` with a new one will break communication with all workers that have not yet updated their secret. + +Defaults to `null`. Example configuration: ```yaml -worker_replication_secret: "secret_secret" +worker_replication_secret: secret_secret +``` +--- +### `worker_replication_secret_path` + +*(string|null)* An alternative to [`worker_replication_secret`](#worker_replication_secret): allows the secret to be specified in an external file. + +The file should be a plain text file, containing only the secret. Synapse reads the secret from the given file once at startup. + +_Added in Synapse 1.126.0._ + +Defaults to `null`. + +Example configuration: +```yaml +worker_replication_secret_path: /path/to/secrets/file ``` --- ### `start_pushers` -Unnecessary to set if using [`pusher_instances`](#pusher_instances) with [`generic_workers`](../../workers.md#synapseappgeneric_worker). +*(boolean)* Unnecessary to set if using [`pusher_instances`](#pusher_instances) with [`generic_workers`](../../workers.md#synapseappgeneric_worker). -Controls sending of push notifications on the main process. Set to `false` -if using a [pusher worker](../../workers.md#synapseapppusher). Defaults to `true`. +Controls sending of push notifications on the main process. Set to `false` if using a [pusher worker](../../workers.md#synapseapppusher). + +Defaults to `true`. Example configuration: ```yaml @@ -4362,32 +4324,26 @@ start_pushers: false --- ### `pusher_instances` -It is possible to scale the processes that handle sending push notifications to [sygnal](https://github.com/matrix-org/sygnal) -and email by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding it's [`worker_name`](#worker_name) to -a `pusher_instances` map. Doing so will remove handling of this function from the main -process. Multiple workers can be added to this map, in which case the work is balanced -across them. Ensure the main process and all pusher workers are restarted after changing -this option. +*(array)* It is possible to scale the processes that handle sending push notifications to [sygnal](https://github.com/matrix-org/sygnal) and email by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding it's [`worker_name`](#worker_name) to a `pusher_instances` map. Doing so will remove handling of this function from the main process. Multiple workers can be added to this map, in which case the work is balanced across them. Ensure the main process and all pusher workers are restarted after changing this option. Defaults to `[]`. -Example configuration for a single worker: +Example configurations: ```yaml pusher_instances: - - pusher_worker1 -``` -And for multiple workers: -```yaml -pusher_instances: - - pusher_worker1 - - pusher_worker2 +- pusher_worker1 ``` +```yaml +pusher_instances: +- pusher_worker1 +- pusher_worker2 +``` --- ### `send_federation` -Unnecessary to set if using [`federation_sender_instances`](#federation_sender_instances) with [`generic_workers`](../../workers.md#synapseappgeneric_worker). +*(boolean)* Unnecessary to set if using [`federation_sender_instances`](#federation_sender_instances) with [`generic_workers`](../../workers.md#synapseappgeneric_worker). + +Controls sending of outbound federation transactions on the main process. Set to `false` if using a [federation sender worker](../../workers.md#synapseappfederation_sender). -Controls sending of outbound federation transactions on the main process. -Set to `false` if using a [federation sender worker](../../workers.md#synapseappfederation_sender). Defaults to `true`. Example configuration: @@ -4397,48 +4353,31 @@ send_federation: false --- ### `federation_sender_instances` -It is possible to scale the processes that handle sending outbound federation requests -by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding it's [`worker_name`](#worker_name) to -a `federation_sender_instances` map. Doing so will remove handling of this function from -the main process. Multiple workers can be added to this map, in which case the work is -balanced across them. +*(array)* It is possible to scale the processes that handle sending outbound federation requests by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding it's [`worker_name`](#worker_name) to a `federation_sender_instances` map. Doing so will remove handling of this function from the main process. Multiple workers can be added to this map, in which case the work is balanced across them. -The way that the load balancing works is any outbound federation request will be assigned -to a federation sender worker based on the hash of the destination server name. This -means that all requests being sent to the same destination will be processed by the same -worker instance. Multiple `federation_sender_instances` are useful if there is a federation -with multiple servers. +The way that the load balancing works is any outbound federation request will be assigned to a federation sender worker based on the hash of the destination server name. This means that all requests being sent to the same destination will be processed by the same worker instance. Multiple `federation_sender_instances` are useful if there is a federation with multiple servers. -This configuration setting must be shared between all workers handling federation -sending, and if changed all federation sender workers must be stopped at the same time -and then started, to ensure that all instances are running with the same config (otherwise -events may be dropped). +This configuration setting must be shared between all workers handling federation sending, and if changed all federation sender workers must be stopped at the same time and then started, to ensure that all instances are running with the same config (otherwise events may be dropped). -Example configuration for a single worker: +Defaults to `[]`. + +Example configurations: ```yaml federation_sender_instances: - - federation_sender1 +- federation_sender1 ``` -And for multiple workers: + ```yaml federation_sender_instances: - - federation_sender1 - - federation_sender2 +- federation_sender1 +- federation_sender2 ``` --- ### `instance_map` -When using workers this should be a map from [`worker_name`](#worker_name) to the HTTP -replication listener of the worker, if configured, and to the main process. Each worker -declared under [`stream_writers`](../../workers.md#stream-writers) and -[`outbound_federation_restricted_to`](#outbound_federation_restricted_to) needs a HTTP -replication listener, and that listener should be included in the `instance_map`. The -main process also needs an entry on the `instance_map`, and it should be listed under -`main` **if even one other worker exists**. Ensure the port matches with what is -declared inside the `listener` block for a `replication` listener. +*(object)* When using workers this should be a map from [`worker_name`](#worker_name) to the HTTP replication listener of the worker, if configured, and to the main process. Each worker declared under [`stream_writers`](../../workers.md#stream-writers) and [`outbound_federation_restricted_to`](#outbound_federation_restricted_to) needs a HTTP replication listener, and that listener should be included in the `instance_map`. The main process also needs an entry on the `instance_map`, and it should be listed under `main` **if even one other worker exists**. Ensure the port matches with what is declared inside the `listener` block for a `replication` listener. Defaults to `{}`. - -Example configuration: +Example configurations: ```yaml instance_map: main: @@ -4447,8 +4386,12 @@ instance_map: worker1: host: localhost port: 8034 + other: + host: localhost + port: 8035 + tls: true ``` -Example configuration(#2, for UNIX sockets): + ```yaml instance_map: main: @@ -4459,12 +4402,29 @@ instance_map: --- ### `stream_writers` -Experimental: When using workers you can define which workers should -handle writing to streams such as event persistence and typing notifications. -Any worker specified here must also be in the [`instance_map`](#instance_map). +*(object)* Experimental: When using workers you can define which workers should handle writing to streams such as event persistence and typing notifications. Any worker specified here must also be in the [`instance_map`](#instance_map). -See the list of available streams in the -[worker documentation](../../workers.md#stream-writers). +See the list of available streams in the [worker documentation](../../workers.md#stream-writers). + +Defaults to `{}`. + +This setting has the following sub-options: + +* `events` (string): Name of a worker assigned to the `events` stream. + +* `typing` (string): Name of a worker assigned to the `typing` stream. + +* `to_device` (string): Name of a worker assigned to the `to_device` stream. + +* `account_data` (string): Name of a worker assigned to the `account_data` stream. + +* `receipts` (string): Name of a worker assigned to the `receipts` stream. + +* `presence` (string): Name of a worker assigned to the `presence` stream. + +* `push_rules` (string): Name of a worker assigned to the `push_rules` stream. + +* `device_lists` (string): Name of a worker assigned to the `device_lists` stream. Example configuration: ```yaml @@ -4475,30 +4435,24 @@ stream_writers: --- ### `outbound_federation_restricted_to` -When using workers, you can restrict outbound federation traffic to only go through a -specific subset of workers. Any worker specified here must also be in the -[`instance_map`](#instance_map). -[`worker_replication_secret`](#worker_replication_secret) must also be configured to -authorize inter-worker communication. +*(array)* When using workers, you can restrict outbound federation traffic to only go through a specific subset of workers. Any worker specified here must also be in the [`instance_map`](#instance_map). [`worker_replication_secret`](#worker_replication_secret) must also be configured to authorize inter-worker communication. -```yaml -outbound_federation_restricted_to: - - federation_sender1 - - federation_sender2 -``` - -Also see the [worker -documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers) -for more info. +Also see the [worker documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers) for more info. _Added in Synapse 1.89.0._ +Defaults to `[]`. + +Example configuration: +```yaml +outbound_federation_restricted_to: +- federation_sender1 +- federation_sender2 +``` --- ### `run_background_tasks_on` -The [worker](../../workers.md#background-tasks) that is used to run -background tasks (e.g. cleaning up expired data). If not provided this -defaults to the main process. +*(string|null)* The [worker](../../workers.md#background-tasks) that is used to run background tasks (e.g. cleaning up expired data). If not provided this defaults to the main process. Defaults to `null`. Example configuration: ```yaml @@ -4507,73 +4461,80 @@ run_background_tasks_on: worker1 --- ### `update_user_directory_from_worker` -The [worker](../../workers.md#updating-the-user-directory) that is used to -update the user directory. If not provided this defaults to the main process. +*(string|null)* The [worker](../../workers.md#updating-the-user-directory) that is used to update the user directory. If not provided this defaults to the main process. + +_Added in Synapse 1.59.0._ + +Defaults to `null`. Example configuration: ```yaml update_user_directory_from_worker: worker1 ``` - -_Added in Synapse 1.59.0._ - --- ### `notify_appservices_from_worker` -The [worker](../../workers.md#notifying-application-services) that is used to -send output traffic to Application Services. If not provided this defaults -to the main process. +*(string|null)* The [worker](../../workers.md#notifying-application-services) that is used to send output traffic to Application Services. If not provided this defaults to the main process. + +_Added in Synapse 1.59.0._ + +Defaults to `null`. Example configuration: ```yaml notify_appservices_from_worker: worker1 ``` - -_Added in Synapse 1.59.0._ - --- ### `media_instance_running_background_jobs` -The [worker](../../workers.md#synapseappmedia_repository) that is used to run -background tasks for media repository. If running multiple media repositories -you must configure a single instance to run the background tasks. If not provided -this defaults to the main process or your single `media_repository` worker. +*(string|null)* The [worker](../../workers.md#synapseappmedia_repository) that is used to run background tasks for media repository. If running multiple media repositories you must configure a single instance to run the background tasks. If not provided this defaults to the main process or your single `media_repository` worker. + +_Added in Synapse 1.16.0._ + +Defaults to `null`. Example configuration: ```yaml media_instance_running_background_jobs: worker1 ``` - -_Added in Synapse 1.16.0._ - --- ### `redis` -Configuration for Redis when using workers. This *must* be enabled when using workers. +*(object)* Configuration for Redis when using workers. This *must* be enabled when using workers. + +_Added in Synapse 1.78.0._ + +_Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, private\_key\_file, ca\_file and ca\_path attributes_ + +_Changed in Synapse 1.85.0: Added path option to use a local Unix socket_ + +_Changed in Synapse 1.116.0: Added password\_path_ + This setting has the following sub-options: -* `enabled`: whether to use Redis support. Defaults to false. -* `host` and `port`: Optional host and port to use to connect to redis. Defaults to - localhost and 6379 -* `path`: The full path to a local Unix socket file. **If this is used, `host` and - `port` are ignored.** Defaults to `/tmp/redis.sock' -* `password`: Optional password if configured on the Redis instance. -* `password_path`: Alternative to `password`, reading the password from an - external file. The file should be a plain text file, containing only the - password. Synapse reads the password from the given file once at startup. -* `dbid`: Optional redis dbid if needs to connect to specific redis logical db. -* `use_tls`: Whether to use tls connection. Defaults to false. -* `certificate_file`: Optional path to the certificate file -* `private_key_file`: Optional path to the private key file -* `ca_file`: Optional path to the CA certificate file. Use this one or: -* `ca_path`: Optional path to the folder containing the CA certificate file - _Added in Synapse 1.78.0._ +* `enabled` (boolean): Whether to use Redis support. Defaults to `false`. - _Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, private\_key\_file, ca\_file and ca\_path attributes_ +* `host` (string): Optional host to use to connect to Redis. Defaults to `"localhost"`. - _Changed in Synapse 1.85.0: Added path option to use a local Unix socket_ +* `port` (integer): Optional port to use to connect to Redis. Defaults to `6379`. - _Changed in Synapse 1.116.0: Added password\_path_ +* `path` (string): The full path to a local Unix socket file. **If this is used, `host` and `port` are ignored.** Defaults to `"/tmp/redis.sock"`. + +* `password` (string|null): Optional password if configured on the Redis instance. Defaults to `null`. + +* `password_path` (string|null): Alternative to `password`, reading the password from an external file. The file should be a plain text file, containing only the password. Synapse reads the password from the given file once at startup. Defaults to `null`. + +* `dbid` (string|null): Optional redis dbid if needs to connect to specific redis logical db. Defaults to `null`. + +* `use_tls` (boolean): Whether to use a TLS connection. Defaults to `false`. + +* `certificate_file` (string|null): Optional path to the certificate file. Defaults to `null`. + +* `private_key_file` (string|null): Optional path to the private key file. Defaults to `null`. + +* `ca_file` (string|null): Optional path to the CA certificate file. Use this one or `ca_path` Defaults to `null`. + +* `ca_path` (string|null): Optional path to the folder containing the CA certificate file. Use this one or `ca_file` Defaults to `null`. Example configuration: ```yaml @@ -4582,31 +4543,25 @@ redis: host: localhost port: 6379 password_path: - # OR password: dbid: - #use_tls: True - #certificate_file: - #private_key_file: - #ca_file: ``` --- ## Individual worker configuration -These options configure an individual worker, in its worker configuration file. -They should be not be provided when configuring the main process. -Note also the configuration above for -[coordinating a cluster of workers](#coordinating-workers). +These options configure an individual worker, in its worker configuration file. They should be not be provided when configuring the main process. + +Note also the configuration above for [coordinating a cluster of workers](#coordinating-workers). For guidance on setting up workers, see the [worker documentation](../../workers.md). --- ### `worker_app` -The type of worker. The currently available worker applications are listed -in [worker documentation](../../workers.md#available-worker-applications). +*(string)* The type of worker. The currently available worker applications are listed in [worker documentation](../../workers.md#available-worker-applications). -The most common worker is the -[`synapse.app.generic_worker`](../../workers.md#synapseappgeneric_worker). +The most common worker is the [`synapse.app.generic_worker`](../../workers.md#synapseappgeneric_worker). + +There is no default for this option. Example configuration: ```yaml @@ -4615,9 +4570,7 @@ worker_app: synapse.app.generic_worker --- ### `worker_name` -A unique name for the worker. The worker needs a name to be addressed in -further parameters and identification in log files. We strongly recommend -giving each worker a unique `worker_name`. +*(string)* A unique name for the worker. The worker needs a name to be addressed in further parameters and identification in log files. We strongly recommend giving each worker a unique `worker_name`. There is no default for this option. Example configuration: ```yaml @@ -4626,46 +4579,45 @@ worker_name: generic_worker1 --- ### `worker_listeners` -A worker can handle HTTP requests. To do so, a `worker_listeners` option -must be declared, in the same way as the [`listeners` option](#listeners) -in the shared config. +*(array)* A worker can handle HTTP requests. To do so, a `worker_listeners` option must be declared, in the same way as the [`listeners` option](#listeners) in the shared config. -Workers declared in [`stream_writers`](#stream_writers) and [`instance_map`](#instance_map) - will need to include a `replication` listener here, in order to accept internal HTTP -requests from other workers. +Workers declared in [`stream_writers`](#stream_writers) and [`instance_map`](#instance_map) will need to include a `replication` listener here, in order to accept internal HTTP requests from other workers. -Example configuration: +Example #2 is using UNIX sockets with a `replication` listener. + +Defaults to `[]`. + +Example configurations: ```yaml worker_listeners: - - type: http - port: 8083 - resources: - - names: [client, federation] +- type: http + port: 8083 + resources: + - names: + - client + - federation ``` -Example configuration(#2, using UNIX sockets with a `replication` listener): + ```yaml worker_listeners: - - type: http - path: /run/synapse/worker_replication.sock - resources: - - names: [replication] - - type: http - path: /run/synapse/worker_public.sock - resources: - - names: [client, federation] +- type: http + path: /run/synapse/worker_replication.sock + resources: + - names: + - replication +- type: http + path: /run/synapse/worker_public.sock + resources: + - names: + - client + - federation ``` --- ### `worker_manhole` -A worker may have a listener for [`manhole`](../../manhole.md). -It allows server administrators to access a Python shell on the worker. +*(integer|null)* A worker may have a listener for [`manhole`](../../manhole.md). It allows server administrators to access a Python shell on the worker. -Example configuration: -```yaml -worker_manhole: 9000 -``` - -This is a short form for: +The example below is a short form for ```yaml worker_listeners: - port: 9000 @@ -4675,14 +4627,16 @@ worker_listeners: It needs also an additional [`manhole_settings`](#manhole_settings) configuration. +Defaults to `null`. + +Example configuration: +```yaml +worker_manhole: 9000 +``` --- ### `worker_daemonize` -Specifies whether the worker should be started as a daemon process. -If Synapse is being managed by [systemd](../../systemd-with-workers/), this option -must be omitted or set to `false`. - -Defaults to `false`. +*(boolean)* Specifies whether the worker should be started as a daemon process. If Synapse is being managed by [systemd](../../systemd-with-workers/), this option must be omitted or set to `false`. Defaults to `false`. Example configuration: ```yaml @@ -4691,15 +4645,14 @@ worker_daemonize: true --- ### `worker_pid_file` -When running a worker as a daemon, we need a place to store the -[PID](https://en.wikipedia.org/wiki/Process_identifier) of the worker. -This option defines the location of that "pid file". +*(string|null)* When running a worker as a daemon, we need a place to store the [PID](https://en.wikipedia.org/wiki/Process_identifier) of the worker. This option defines the location of that "pid file". -This option is required if `worker_daemonize` is `true` and ignored -otherwise. It has no default. +This option is required if `worker_daemonize` is `true` and ignored otherwise. See also the [`pid_file` option](#pid_file) option for the main Synapse process. +Defaults to `null`. + Example configuration: ```yaml worker_pid_file: DATADIR/generic_worker1.pid @@ -4707,9 +4660,7 @@ worker_pid_file: DATADIR/generic_worker1.pid --- ### `worker_log_config` -This option specifies a yaml python logging config file as described -[here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). -See also the [`log_config` option](#log_config) option for the main Synapse process. +*(string|null)* This option specifies a yaml python logging config file as described [here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). See also the [`log_config` option](#log_config) option for the main Synapse process. Defaults to `null`. Example configuration: ```yaml @@ -4717,62 +4668,62 @@ worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml ``` --- ## Background Updates + Configuration settings related to background updates. --- ### `background_updates` -Background updates are database updates that are run in the background in batches. -The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to -sleep can all be configured. This is helpful to speed up or slow down the updates. +*(object)* Background updates are database updates that are run in the background in batches. The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to sleep can all be configured. This is helpful to speed up or slow down the updates. + This setting has the following sub-options: -* `background_update_duration_ms`: How long in milliseconds to run a batch of background updates for. Defaults to 100. - Set a different time to change the default. -* `sleep_enabled`: Whether to sleep between updates. Defaults to true. Set to false to change the default. -* `sleep_duration_ms`: If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. - Set a duration to change the default. -* `min_batch_size`: Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. - Set a size to change the default. -* `default_batch_size`: The batch size to use for the first iteration of a new background update. The default is 100. - Set a size to change the default. + +* `background_update_duration_ms` (integer): How long in milliseconds to run a batch of background updates for. Defaults to `100`. + +* `sleep_enabled` (boolean): Whether to sleep between updates. Defaults to `true`. + +* `sleep_duration_ms` (integer): If sleeping between updates, how long in milliseconds to sleep for. Defaults to `1000`. + +* `min_batch_size` (integer): Minimum size a batch of background updates can be. Must be greater than 0. Defaults to `1`. + +* `default_batch_size` (integer): The batch size to use for the first iteration of a new background update. Defaults to `100`. Example configuration: ```yaml background_updates: - background_update_duration_ms: 500 - sleep_enabled: false - sleep_duration_ms: 300 - min_batch_size: 10 - default_batch_size: 50 + background_update_duration_ms: 500 + sleep_enabled: false + sleep_duration_ms: 300 + min_batch_size: 10 + default_batch_size: 50 ``` --- ## Auto Accept Invites + Configuration settings related to automatically accepting invites. --- ### `auto_accept_invites` -Automatically accepting invites controls whether users are presented with an invite request or if they -are instead automatically joined to a room when receiving an invite. Set the `enabled` sub-option to true to -enable auto-accepting invites. Defaults to false. -This setting has the following sub-options: -* `enabled`: Whether to run the auto-accept invites logic. Defaults to false. -* `only_for_direct_messages`: Whether invites should be automatically accepted for all room types, or only - for direct messages. Defaults to false. -* `only_from_local_users`: Whether to only automatically accept invites from users on this homeserver. Defaults to false. -* `worker_to_run_on`: Which worker to run this module on. This must match - the "worker_name". If not set or `null`, invites will be accepted on the - main process. +*(object)* Automatically accepting invites controls whether users are presented with an invite request or if they are instead automatically joined to a room when receiving an invite. Set the `enabled` sub-option to true to enable auto-accepting invites. -NOTE: Care should be taken not to enable this setting if the `synapse_auto_accept_invite` module is enabled and installed. -The two modules will compete to perform the same task and may result in undesired behaviour. For example, multiple join -events could be generated from a single invite. +NOTE: Care should be taken not to enable this setting if the `synapse_auto_accept_invite` module is enabled and installed. The two modules will compete to perform the same task and may result in undesired behaviour. For example, multiple join events could be generated from a single invite. + +This setting has the following sub-options: + +* `enabled` (boolean): Whether to run the auto-accept invites logic. Defaults to `false`. + +* `only_for_direct_messages` (boolean): Whether invites should be automatically accepted for all room types, or only for direct messages. Defaults to `false`. + +* `only_from_local_users` (boolean): Whether to only automatically accept invites from users on this homeserver. Defaults to `false`. + +* `worker_to_run_on` (string|null): Which worker to run this module on. This must match the "worker_name". If not set or `null`, invites will be accepted on the main process. Defaults to `null`. Example configuration: ```yaml auto_accept_invites: - enabled: true - only_for_direct_messages: true - only_from_local_users: true - worker_to_run_on: "worker_1" + enabled: true + only_for_direct_messages: true + only_from_local_users: true + worker_to_run_on: worker_1 ``` diff --git a/docs/user_directory.md b/docs/user_directory.md index be8664a016..75d32af44f 100644 --- a/docs/user_directory.md +++ b/docs/user_directory.md @@ -77,14 +77,11 @@ The user provided search term is lowercased and normalized using [NFKC](https:// this treats the string as case-insensitive, canonicalizes different forms of the same text, and maps some "roughly equivalent" characters together. -The search term is then split into words: - -* If [ICU](https://en.wikipedia.org/wiki/International_Components_for_Unicode) is - available, then the system's [default locale](https://unicode-org.github.io/icu/userguide/locale/#default-locales) - will be used to break the search term into words. (See the - [installation instructions](setup/installation.md) for how to install ICU.) -* If unavailable, then runs of ASCII characters, numbers, underscores, and hyphens - are considered words. +The search term is then split into segments using the [`icu_segmenter` +Rust crate](https://crates.io/crates/icu_segmenter). This crate ships with its +own dictionary and Long Short Term-Memory (LSTM) machine learning models +per-language to segment words. Read more [in the crate's +documentation](https://docs.rs/icu/latest/icu/segmenter/struct.WordSegmenter.html#method.new_auto). The queries for PostgreSQL and SQLite are detailed below, but their overall goal is to find matching users, preferring users who are "real" (e.g. not bots, diff --git a/docs/workers.md b/docs/workers.md index 0116c455bc..18bb0b76f6 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -200,6 +200,7 @@ information. ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ # Federation requests + ^/_matrix/federation/v1/version$ ^/_matrix/federation/v1/event/ ^/_matrix/federation/v1/state/ ^/_matrix/federation/v1/state_ids/ @@ -237,7 +238,9 @@ information. ^/_matrix/client/unstable/im.nheko.summary/summary/.*$ ^/_matrix/client/(r0|v3|unstable)/account/3pid$ ^/_matrix/client/(r0|v3|unstable)/account/whoami$ - ^/_matrix/client/(r0|v3|unstable)/devices$ + ^/_matrix/client/(r0|v3|unstable)/account/deactivate$ + ^/_matrix/client/(r0|v3)/delete_devices$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/devices(/|$) ^/_matrix/client/versions$ ^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$ ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/ @@ -249,13 +252,16 @@ information. ^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$ ^/_matrix/client/(r0|v3|unstable)/capabilities$ ^/_matrix/client/(r0|v3|unstable)/notifications$ + ^/_synapse/admin/v1/rooms/[^/]+$ # Encryption requests ^/_matrix/client/(r0|v3|unstable)/keys/query$ ^/_matrix/client/(r0|v3|unstable)/keys/changes$ ^/_matrix/client/(r0|v3|unstable)/keys/claim$ ^/_matrix/client/(r0|v3|unstable)/room_keys/ - ^/_matrix/client/(r0|v3|unstable)/keys/upload/ + ^/_matrix/client/(r0|v3|unstable)/keys/upload + ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/device_signing/upload$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$ # Registration/login requests ^/_matrix/client/(api/v1|r0|v3|unstable)/login$ @@ -273,17 +279,6 @@ information. ^/_matrix/client/(api/v1|r0|v3|unstable)/knock/ ^/_matrix/client/(api/v1|r0|v3|unstable)/profile/ - # Account data requests - ^/_matrix/client/(r0|v3|unstable)/.*/tags - ^/_matrix/client/(r0|v3|unstable)/.*/account_data - - # Receipts requests - ^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt - ^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers - - # Presence requests - ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ - # User directory search requests ^/_matrix/client/(r0|v3|unstable)/user_directory/search$ @@ -292,6 +287,13 @@ Additionally, the following REST endpoints can be handled for GET requests: ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events + # Account data requests + ^/_matrix/client/(r0|v3|unstable)/.*/tags + ^/_matrix/client/(r0|v3|unstable)/.*/account_data + + # Presence requests + ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ + Pagination requests can also be handled, but all requests for a given room must be routed to the same instance. Additionally, care must be taken to ensure that the purge history admin API is not used while pagination requests @@ -324,6 +326,14 @@ For multiple workers not handling the SSO endpoints properly, see [#7530](https://github.com/matrix-org/synapse/issues/7530) and [#9427](https://github.com/matrix-org/synapse/issues/9427). +Additionally, when MSC3861 is enabled (`experimental_features.msc3861.enabled` +set to `true`), the following endpoints can be handled by the worker: + + ^/_synapse/admin/v2/users/[^/]+$ + ^/_synapse/admin/v1/username_available$ + ^/_synapse/admin/v1/users/[^/]+/_allow_cross_signing_replacement_without_uia$ + ^/_synapse/admin/v1/users/[^/]+/devices$ + Note that a [HTTP listener](usage/configuration/config_documentation.md#listeners) with `client` and `federation` `resources` must be configured in the [`worker_listeners`](usage/configuration/config_documentation.md#worker_listeners) @@ -522,8 +532,9 @@ the stream writer for the `account_data` stream: ##### The `receipts` stream -The following endpoints should be routed directly to the worker configured as -the stream writer for the `receipts` stream: +The `receipts` stream supports multiple writers. The following endpoints +can be handled by any worker, but should be routed directly to one of the workers +configured as stream writer for the `receipts` stream: ^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt ^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers @@ -542,6 +553,18 @@ the stream writer for the `push_rules` stream: ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ +##### The `device_lists` stream + +The `device_lists` stream supports multiple writers. The following endpoints +can be handled by any worker, but should be routed directly to one of the workers +configured as stream writer for the `device_lists` stream: + + ^/_matrix/client/(r0|v3)/delete_devices$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/devices(/|$) + ^/_matrix/client/(r0|v3|unstable)/keys/upload + ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/device_signing/upload$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$ + #### Restrict outbound federation traffic to a specific set of workers The diff --git a/flake.nix b/flake.nix index 749c10da1d..4ff6518aed 100644 --- a/flake.nix +++ b/flake.nix @@ -96,7 +96,6 @@ gnumake # Native dependencies for running Synapse. - icu libffi libjpeg libpqxx diff --git a/mypy.ini b/mypy.ini index cf64248cc5..ae903f858a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,17 @@ [mypy] namespace_packages = True -plugins = pydantic.mypy, mypy_zope:plugin, scripts-dev/mypy_synapse_plugin.py +# Our custom mypy plugin should remain first in this list. +# +# mypy has a limitation where it only chooses the first plugin that returns a non-None +# value for each hook (known-limitation, c.f. +# https://github.com/python/mypy/issues/19524). We workaround this by putting our custom +# plugin first in the plugin order and then manually calling any other conflicting +# plugin hooks in our own plugin followed by our own checks. +# +# If you add a new plugin, make sure to check whether the hooks being used conflict with +# our custom plugin hooks and if so, manually call the other plugin's hooks in our +# custom plugin. (also applies to if the plugin is updated in the future) +plugins = scripts-dev/mypy_synapse_plugin.py, pydantic.mypy, mypy_zope:plugin follow_imports = normal show_error_codes = True show_traceback = True @@ -99,3 +110,6 @@ ignore_missing_imports = True [mypy-multipart.*] ignore_missing_imports = True + +[mypy-mypy_zope.*] +ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index eece221095..4eedeea4e7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -6,6 +6,7 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -13,32 +14,35 @@ files = [ [[package]] name = "attrs" -version = "24.2.0" +version = "25.3.0" description = "Classes Without Boilerplate" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["main", "dev"] files = [ - {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, - {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, + {file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"}, + {file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"}, ] [package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] [[package]] name = "authlib" -version = "1.3.2" +version = "1.6.3" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ - {file = "Authlib-1.3.2-py2.py3-none-any.whl", hash = "sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc"}, - {file = "authlib-1.3.2.tar.gz", hash = "sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2"}, + {file = "authlib-1.6.3-py2.py3-none-any.whl", hash = "sha256:7ea0f082edd95a03b7b72edac65ec7f8f68d703017d7e37573aee4fc603f2a48"}, + {file = "authlib-1.6.3.tar.gz", hash = "sha256:9f7a982cc395de719e4c2215c5707e7ea690ecf84f1ab126f28c053f4219e610"}, ] [package.dependencies] @@ -46,56 +50,81 @@ cryptography = "*" [[package]] name = "automat" -version = "22.10.0" +version = "25.4.16" description = "Self-service finite-state machines for the programmer on the go." optional = false -python-versions = "*" +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "Automat-22.10.0-py2.py3-none-any.whl", hash = "sha256:c3164f8742b9dc440f3682482d32aaff7bb53f71740dd018533f9de286b64180"}, - {file = "Automat-22.10.0.tar.gz", hash = "sha256:e56beb84edad19dcc11d30e8d9b895f75deeb5ef5e96b84a467066b3b84bb04e"}, + {file = "automat-25.4.16-py3-none-any.whl", hash = "sha256:04e9bce696a8d5671ee698005af6e5a9fa15354140a87f4870744604dcdd3ba1"}, + {file = "automat-25.4.16.tar.gz", hash = "sha256:0017591a5477066e90d26b0e696ddc143baafd87b588cfac8100bc6be9634de0"}, ] [package.dependencies] -attrs = ">=19.2.0" -six = "*" +typing_extensions = {version = "*", markers = "python_version < \"3.10\""} [package.extras] visualize = ["Twisted (>=16.1.1)", "graphviz (>0.5.1)"] [[package]] name = "bcrypt" -version = "4.2.0" +version = "4.3.0" description = "Modern password hashing for your software and your servers" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "bcrypt-4.2.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7"}, - {file = "bcrypt-4.2.0-cp37-abi3-win32.whl", hash = "sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458"}, - {file = "bcrypt-4.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5"}, - {file = "bcrypt-4.2.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8"}, - {file = "bcrypt-4.2.0-cp39-abi3-win32.whl", hash = "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34"}, - {file = "bcrypt-4.2.0-cp39-abi3-win_amd64.whl", hash = "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9"}, - {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:39e1d30c7233cfc54f5c3f2c825156fe044efdd3e0b9d309512cc514a263ec2a"}, - {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f4f4acf526fcd1c34e7ce851147deedd4e26e6402369304220250598b26448db"}, - {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:1ff39b78a52cf03fdf902635e4c81e544714861ba3f0efc56558979dd4f09170"}, - {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:373db9abe198e8e2c70d12b479464e0d5092cc122b20ec504097b5f2297ed184"}, - {file = "bcrypt-4.2.0.tar.gz", hash = "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221"}, + {file = "bcrypt-4.3.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f01e060f14b6b57bbb72fc5b4a83ac21c443c9a2ee708e04a10e9192f90a6281"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5eeac541cefd0bb887a371ef73c62c3cd78535e4887b310626036a7c0a817bb"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59e1aa0e2cd871b08ca146ed08445038f42ff75968c7ae50d2fdd7860ade2180"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:0042b2e342e9ae3d2ed22727c1262f76cc4f345683b5c1715f0250cf4277294f"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74a8d21a09f5e025a9a23e7c0fd2c7fe8e7503e4d356c0a2c1486ba010619f09"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:0142b2cb84a009f8452c8c5a33ace5e3dfec4159e7735f5afe9a4d50a8ea722d"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_34_aarch64.whl", hash = "sha256:12fa6ce40cde3f0b899729dbd7d5e8811cb892d31b6f7d0334a1f37748b789fd"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_34_x86_64.whl", hash = "sha256:5bd3cca1f2aa5dbcf39e2aa13dd094ea181f48959e1071265de49cc2b82525af"}, + {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:335a420cfd63fc5bc27308e929bee231c15c85cc4c496610ffb17923abf7f231"}, + {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:0e30e5e67aed0187a1764911af023043b4542e70a7461ad20e837e94d23e1d6c"}, + {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:3b8d62290ebefd49ee0b3ce7500f5dbdcf13b81402c05f6dafab9a1e1b27212f"}, + {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef6630e0ec01376f59a006dc72918b1bf436c3b571b80fa1968d775fa02fe7d"}, + {file = "bcrypt-4.3.0-cp313-cp313t-win32.whl", hash = "sha256:7a4be4cbf241afee43f1c3969b9103a41b40bcb3a3f467ab19f891d9bc4642e4"}, + {file = "bcrypt-4.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5c1949bf259a388863ced887c7861da1df681cb2388645766c89fdfd9004c669"}, + {file = "bcrypt-4.3.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:f81b0ed2639568bf14749112298f9e4e2b28853dab50a8b357e31798686a036d"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:864f8f19adbe13b7de11ba15d85d4a428c7e2f344bac110f667676a0ff84924b"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e36506d001e93bffe59754397572f21bb5dc7c83f54454c990c74a468cd589e"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:842d08d75d9fe9fb94b18b071090220697f9f184d4547179b60734846461ed59"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7c03296b85cb87db865d91da79bf63d5609284fc0cab9472fdd8367bbd830753"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:62f26585e8b219cdc909b6a0069efc5e4267e25d4a3770a364ac58024f62a761"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:beeefe437218a65322fbd0069eb437e7c98137e08f22c4660ac2dc795c31f8bb"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:97eea7408db3a5bcce4a55d13245ab3fa566e23b4c67cd227062bb49e26c585d"}, + {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:191354ebfe305e84f344c5964c7cd5f924a3bfc5d405c75ad07f232b6dffb49f"}, + {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:41261d64150858eeb5ff43c753c4b216991e0ae16614a308a15d909503617732"}, + {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:33752b1ba962ee793fa2b6321404bf20011fe45b9afd2a842139de3011898fef"}, + {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:50e6e80a4bfd23a25f5c05b90167c19030cf9f87930f7cb2eacb99f45d1c3304"}, + {file = "bcrypt-4.3.0-cp38-abi3-win32.whl", hash = "sha256:67a561c4d9fb9465ec866177e7aebcad08fe23aaf6fbd692a6fab69088abfc51"}, + {file = "bcrypt-4.3.0-cp38-abi3-win_amd64.whl", hash = "sha256:584027857bc2843772114717a7490a37f68da563b3620f78a849bcb54dc11e62"}, + {file = "bcrypt-4.3.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0d3efb1157edebfd9128e4e46e2ac1a64e0c1fe46fb023158a407c7892b0f8c3"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08bacc884fd302b611226c01014eca277d48f0a05187666bca23aac0dad6fe24"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6746e6fec103fcd509b96bacdfdaa2fbde9a553245dbada284435173a6f1aef"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:afe327968aaf13fc143a56a3360cb27d4ad0345e34da12c7290f1b00b8fe9a8b"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d9af79d322e735b1fc33404b5765108ae0ff232d4b54666d46730f8ac1a43676"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f1e3ffa1365e8702dc48c8b360fef8d7afeca482809c5e45e653af82ccd088c1"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:3004df1b323d10021fda07a813fd33e0fd57bef0e9a480bb143877f6cba996fe"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:531457e5c839d8caea9b589a1bcfe3756b0547d7814e9ce3d437f17da75c32b0"}, + {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:17a854d9a7a476a89dcef6c8bd119ad23e0f82557afbd2c442777a16408e614f"}, + {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6fb1fd3ab08c0cbc6826a2e0447610c6f09e983a281b919ed721ad32236b8b23"}, + {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e965a9c1e9a393b8005031ff52583cedc15b7884fce7deb8b0346388837d6cfe"}, + {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:79e70b8342a33b52b55d93b3a59223a844962bef479f6a0ea318ebbcadf71505"}, + {file = "bcrypt-4.3.0-cp39-abi3-win32.whl", hash = "sha256:b4d4e57f0a63fd0b358eb765063ff661328f69a04494427265950c71b992a39a"}, + {file = "bcrypt-4.3.0-cp39-abi3-win_amd64.whl", hash = "sha256:e53e074b120f2877a35cc6c736b8eb161377caae8925c17688bd46ba56daaa5b"}, + {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c950d682f0952bafcceaf709761da0a32a942272fad381081b51096ffa46cea1"}, + {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:107d53b5c67e0bbc3f03ebf5b030e0403d24dda980f8e244795335ba7b4a027d"}, + {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b693dbb82b3c27a1604a3dff5bfc5418a7e6a781bb795288141e5f80cf3a3492"}, + {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:b6354d3760fcd31994a14c89659dee887f1351a06e5dac3c1142307172a79f90"}, + {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a839320bf27d474e52ef8cb16449bb2ce0ba03ca9f44daba6d93fa1d8828e48a"}, + {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:bdc6a24e754a555d7316fa4774e64c6c3997d27ed2d1964d55920c7c227bc4ce"}, + {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:55a935b8e9a1d2def0626c4269db3fcd26728cbff1e84f0341465c31c4ee56d8"}, + {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:57967b7a28d855313a963aaea51bf6df89f833db4320da458e5b3c5ab6d4c938"}, + {file = "bcrypt-4.3.0.tar.gz", hash = "sha256:3a3fd2204178b6d2adcf09cb4f6426ffef54762577a7c9b54c159008cb288c18"}, ] [package.extras] @@ -108,6 +137,7 @@ version = "6.2.0" description = "An easy safelist-based HTML-sanitizing tool." optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "bleach-6.2.0-py3-none-any.whl", hash = "sha256:117d9c6097a7c3d22fd578fcd8d35ff1e125df6736f554da4e432fdd63f31e5e"}, {file = "bleach-6.2.0.tar.gz", hash = "sha256:123e894118b8a599fd80d3ec1a6d4cc7ce4e5882b1317a7e1ba69b56e95f991f"}, @@ -125,6 +155,7 @@ version = "2.0.0" description = "Canonical JSON" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "canonicaljson-2.0.0-py3-none-any.whl", hash = "sha256:c38a315de3b5a0532f1ec1f9153cd3d716abfc565a558d00a4835428a34fca5b"}, {file = "canonicaljson-2.0.0.tar.gz", hash = "sha256:e2fdaef1d7fadc5d9cb59bd3d0d41b064ddda697809ac4325dced721d12f113f"}, @@ -136,6 +167,7 @@ version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main", "dev"] files = [ {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, @@ -147,6 +179,7 @@ version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -226,6 +259,7 @@ version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" +groups = ["main", "dev"] files = [ {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, @@ -306,13 +340,14 @@ files = [ [[package]] name = "click" -version = "8.1.7" +version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ - {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, - {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, ] [package.dependencies] @@ -324,31 +359,20 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] +markers = "platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -[[package]] -name = "commonmark" -version = "0.9.1" -description = "Python parser for the CommonMark Markdown spec" -optional = false -python-versions = "*" -files = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] - -[package.extras] -test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] - [[package]] name = "constantly" version = "15.1.0" description = "Symbolic constants in Python" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "constantly-15.1.0-py2.py3-none-any.whl", hash = "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"}, {file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"}, @@ -360,6 +384,7 @@ version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, @@ -409,34 +434,20 @@ version = "0.7.1" description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, ] -[[package]] -name = "deprecated" -version = "1.2.13" -description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, - {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, -] - -[package.dependencies] -wrapt = ">=1.10,<2" - -[package.extras] -dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version (<1)", "configparser (<5)", "importlib-metadata (<3)", "importlib-resources (<4)", "sphinx (<2)", "sphinxcontrib-websupport (<2)", "tox", "zipp (<2)"] - [[package]] name = "docutils" version = "0.19" description = "Docutils -- Python Documentation Utilities" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, @@ -448,6 +459,8 @@ version = "4.1.5" description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and lxml" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -462,6 +475,7 @@ version = "4.0.10" description = "Git Object Database" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, @@ -472,123 +486,142 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.43" +version = "3.1.45" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, + {file = "gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77"}, + {file = "gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" +typing-extensions = {version = ">=3.10.0.2", markers = "python_version < \"3.10\""} [package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] +doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] name = "hiredis" -version = "3.0.0" +version = "3.2.1" description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"all\" or extra == \"redis\"" files = [ - {file = "hiredis-3.0.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:4b182791c41c5eb1d9ed736f0ff81694b06937ca14b0d4dadde5dadba7ff6dae"}, - {file = "hiredis-3.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:13c275b483a052dd645eb2cb60d6380f1f5215e4c22d6207e17b86be6dd87ffa"}, - {file = "hiredis-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1018cc7f12824506f165027eabb302735b49e63af73eb4d5450c66c88f47026"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83a29cc7b21b746cb6a480189e49f49b2072812c445e66a9e38d2004d496b81c"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e241fab6332e8fb5f14af00a4a9c6aefa22f19a336c069b7ddbf28ef8341e8d6"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1fb8de899f0145d6c4d5d4bd0ee88a78eb980a7ffabd51e9889251b8f58f1785"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b23291951959141173eec10f8573538e9349fa27f47a0c34323d1970bf891ee5"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e421ac9e4b5efc11705a0d5149e641d4defdc07077f748667f359e60dc904420"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:77c8006c12154c37691b24ff293c077300c22944018c3ff70094a33e10c1d795"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:41afc0d3c18b59eb50970479a9c0e5544fb4b95e3a79cf2fbaece6ddefb926fe"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:04ccae6dcd9647eae6025425ab64edb4d79fde8b9e6e115ebfabc6830170e3b2"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:fe91d62b0594db5ea7d23fc2192182b1a7b6973f628a9b8b2e0a42a2be721ac6"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:99516d99316062824a24d145d694f5b0d030c80da693ea6f8c4ecf71a251d8bb"}, - {file = "hiredis-3.0.0-cp310-cp310-win32.whl", hash = "sha256:562eaf820de045eb487afaa37e6293fe7eceb5b25e158b5a1974b7e40bf04543"}, - {file = "hiredis-3.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:a1c81c89ed765198da27412aa21478f30d54ef69bf5e4480089d9c3f77b8f882"}, - {file = "hiredis-3.0.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:4664dedcd5933364756d7251a7ea86d60246ccf73a2e00912872dacbfcef8978"}, - {file = "hiredis-3.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:47de0bbccf4c8a9f99d82d225f7672b9dd690d8fd872007b933ef51a302c9fa6"}, - {file = "hiredis-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e43679eca508ba8240d016d8cca9d27342d70184773c15bea78a23c87a1922f1"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13c345e7278c210317e77e1934b27b61394fee0dec2e8bd47e71570900f75823"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00018f22f38530768b73ea86c11f47e8d4df65facd4e562bd78773bd1baef35e"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ea3a86405baa8eb0d3639ced6926ad03e07113de54cb00fd7510cb0db76a89d"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c073848d2b1d5561f3903879ccf4e1a70c9b1e7566c7bdcc98d082fa3e7f0a1d"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a8dffb5f5b3415a4669d25de48b617fd9d44b0bccfc4c2ab24b06406ecc9ecb"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:22c17c96143c2a62dfd61b13803bc5de2ac526b8768d2141c018b965d0333b66"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c3ece960008dab66c6b8bb3a1350764677ee7c74ccd6270aaf1b1caf9ccebb46"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f75999ae00a920f7dce6ecae76fa5e8674a3110e5a75f12c7a2c75ae1af53396"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e069967cbd5e1900aafc4b5943888f6d34937fc59bf8918a1a546cb729b4b1e4"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0aacc0a78e1d94d843a6d191f224a35893e6bdfeb77a4a89264155015c65f126"}, - {file = "hiredis-3.0.0-cp311-cp311-win32.whl", hash = "sha256:719c32147ba29528cb451f037bf837dcdda4ff3ddb6cdb12c4216b0973174718"}, - {file = "hiredis-3.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:bdc144d56333c52c853c31b4e2e52cfbdb22d3da4374c00f5f3d67c42158970f"}, - {file = "hiredis-3.0.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:484025d2eb8f6348f7876fc5a2ee742f568915039fcb31b478fd5c242bb0fe3a"}, - {file = "hiredis-3.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:fcdb552ffd97151dab8e7bc3ab556dfa1512556b48a367db94b5c20253a35ee1"}, - {file = "hiredis-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bb6f9fd92f147ba11d338ef5c68af4fd2908739c09e51f186e1d90958c68cc1"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa86bf9a0ed339ec9e8a9a9d0ae4dccd8671625c83f9f9f2640729b15e07fbfd"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e194a0d5df9456995d8f510eab9f529213e7326af6b94770abf8f8b7952ddcaa"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8a1df39d74ec507d79c7a82c8063eee60bf80537cdeee652f576059b9cdd15c"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f91456507427ba36fd81b2ca11053a8e112c775325acc74e993201ea912d63e9"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9862db92ef67a8a02e0d5370f07d380e14577ecb281b79720e0d7a89aedb9ee5"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d10fcd9e0eeab835f492832b2a6edb5940e2f1230155f33006a8dfd3bd2c94e4"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:48727d7d405d03977d01885f317328dc21d639096308de126c2c4e9950cbd3c9"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e0bb6102ebe2efecf8a3292c6660a0e6fac98176af6de67f020bea1c2343717"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:df274e3abb4df40f4c7274dd3e587dfbb25691826c948bc98d5fead019dfb001"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:034925b5fb514f7b11aac38cd55b3fd7e9d3af23bd6497f3f20aa5b8ba58e232"}, - {file = "hiredis-3.0.0-cp312-cp312-win32.whl", hash = "sha256:120f2dda469b28d12ccff7c2230225162e174657b49cf4cd119db525414ae281"}, - {file = "hiredis-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:e584fe5f4e6681d8762982be055f1534e0170f6308a7a90f58d737bab12ff6a8"}, - {file = "hiredis-3.0.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:122171ff47d96ed8dd4bba6c0e41d8afaba3e8194949f7720431a62aa29d8895"}, - {file = "hiredis-3.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:ba9fc605ac558f0de67463fb588722878641e6fa1dabcda979e8e69ff581d0bd"}, - {file = "hiredis-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a631e2990b8be23178f655cae8ac6c7422af478c420dd54e25f2e26c29e766f1"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63482db3fadebadc1d01ad33afa6045ebe2ea528eb77ccaabd33ee7d9c2bad48"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f669212c390eebfbe03c4e20181f5970b82c5d0a0ad1df1785f7ffbe7d61150"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a49ef161739f8018c69b371528bdb47d7342edfdee9ddc75a4d8caddf45a6e"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98a152052b8878e5e43a2e3a14075218adafc759547c98668a21e9485882696c"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50a196af0ce657fcde9bf8a0bbe1032e22c64d8fcec2bc926a35e7ff68b3a166"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f2f312eef8aafc2255e3585dcf94d5da116c43ef837db91db9ecdc1bc930072d"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:6ca41fa40fa019cde42c21add74aadd775e71458051a15a352eabeb12eb4d084"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:6eecb343c70629f5af55a8b3e53264e44fa04e155ef7989de13668a0cb102a90"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:c3fdad75e7837a475900a1d3a5cc09aa024293c3b0605155da2d42f41bc0e482"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8854969e7480e8d61ed7549eb232d95082a743e94138d98d7222ba4e9f7ecacd"}, - {file = "hiredis-3.0.0-cp38-cp38-win32.whl", hash = "sha256:f114a6c86edbf17554672b050cce72abf489fe58d583c7921904d5f1c9691605"}, - {file = "hiredis-3.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:7d99b91e42217d7b4b63354b15b41ce960e27d216783e04c4a350224d55842a4"}, - {file = "hiredis-3.0.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:4c6efcbb5687cf8d2aedcc2c3ed4ac6feae90b8547427d417111194873b66b06"}, - {file = "hiredis-3.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5b5cff42a522a0d81c2ae7eae5e56d0ee7365e0c4ad50c4de467d8957aff4414"}, - {file = "hiredis-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:82f794d564f4bc76b80c50b03267fe5d6589e93f08e66b7a2f674faa2fa76ebc"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7a4c1791d7aa7e192f60fe028ae409f18ccdd540f8b1e6aeb0df7816c77e4a4"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2537b2cd98192323fce4244c8edbf11f3cac548a9d633dbbb12b48702f379f4"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fed69bbaa307040c62195a269f82fc3edf46b510a17abb6b30a15d7dab548df"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:869f6d5537d243080f44253491bb30aa1ec3c21754003b3bddeadedeb65842b0"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d435ae89073d7cd51e6b6bf78369c412216261c9c01662e7008ff00978153729"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:204b79b30a0e6be0dc2301a4d385bb61472809f09c49f400497f1cdd5a165c66"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3ea635101b739c12effd189cc19b2671c268abb03013fd1f6321ca29df3ca625"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:f359175197fd833c8dd7a8c288f1516be45415bb5c939862ab60c2918e1e1943"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac6d929cb33dd12ad3424b75725975f0a54b5b12dbff95f2a2d660c510aa106d"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:100431e04d25a522ef2c3b94f294c4219c4de3bfc7d557b6253296145a144c11"}, - {file = "hiredis-3.0.0-cp39-cp39-win32.whl", hash = "sha256:e1a9c14ae9573d172dc050a6f63a644457df5d01ec4d35a6a0f097f812930f83"}, - {file = "hiredis-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:54a6dd7b478e6eb01ce15b3bb5bf771e108c6c148315bf194eb2ab776a3cac4d"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:50da7a9edf371441dfcc56288d790985ee9840d982750580710a9789b8f4a290"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9b285ef6bf1581310b0d5e8f6ce64f790a1c40e89c660e1320b35f7515433672"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dcfa684966f25b335072115de2f920228a3c2caf79d4bfa2b30f6e4f674a948"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a41be8af1fd78ca97bc948d789a09b730d1e7587d07ca53af05758f31f4b985d"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:038756db735e417ab36ee6fd7725ce412385ed2bd0767e8179a4755ea11b804f"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fcecbd39bd42cef905c0b51c9689c39d0cc8b88b1671e7f40d4fb213423aef3a"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:a131377493a59fb0f5eaeb2afd49c6540cafcfba5b0b3752bed707be9e7c4eaf"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d22c53f0ec5c18ecb3d92aa9420563b1c5d657d53f01356114978107b00b860"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8a91e9520fbc65a799943e5c970ffbcd67905744d8becf2e75f9f0a5e8414f0"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3dc8043959b50141df58ab4f398e8ae84c6f9e673a2c9407be65fc789138f4a6"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51b99cfac514173d7b8abdfe10338193e8a0eccdfe1870b646009d2fb7cbe4b5"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:fa1fcad89d8a41d8dc10b1e54951ec1e161deabd84ed5a2c95c3c7213bdb3514"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:898636a06d9bf575d2c594129085ad6b713414038276a4bfc5db7646b8a5be78"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:466f836dbcf86de3f9692097a7a01533dc9926986022c6617dc364a402b265c5"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23142a8af92a13fc1e3f2ca1d940df3dcf2af1d176be41fe8d89e30a837a0b60"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:793c80a3d6b0b0e8196a2d5de37a08330125668c8012922685e17aa9108c33ac"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:467d28112c7faa29b7db743f40803d927c8591e9da02b6ce3d5fadc170a542a2"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:dc384874a719c767b50a30750f937af18842ee5e288afba95a5a3ed703b1515a"}, - {file = "hiredis-3.0.0.tar.gz", hash = "sha256:fed8581ae26345dea1f1e0d1a96e05041a727a45e7d8d459164583e23c6ac441"}, + {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:add17efcbae46c5a6a13b244ff0b4a8fa079602ceb62290095c941b42e9d5dec"}, + {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:5fe955cc4f66c57df1ae8e5caf4de2925d43b5efab4e40859662311d1bcc5f54"}, + {file = "hiredis-3.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f9ad63cd9065820a43fb1efb8ed5ae85bb78f03ef5eb53f6bde47914708f5718"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e7f9e5fdba08841d78d4e1450cae03a4dbed2eda8a4084673cafa5615ce24a"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1dce2508eca5d4e47ef38bc7c0724cb45abcdb0089f95a2ef49baf52882979a8"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:186428bf353e4819abae15aa2ad64c3f40499d596ede280fe328abb9e98e72ce"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74f2500d90a0494843aba7abcdc3e77f859c502e0892112d708c02e1dcae8f90"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32822a94d2fdd1da96c05b22fdeef6d145d8fdbd865ba2f273f45eb949e4a805"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ead809fb08dd4fdb5b4b6e2999c834e78c3b0c450a07c3ed88983964432d0c64"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b90fada20301c3a257e868dd6a4694febc089b2b6d893fa96a3fc6c1f9ab4340"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6d8bff53f526da3d9db86c8668011e4f7ca2958ee3a46c648edab6fe2cd1e709"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:043d929ae262d03e1db0f08616e14504a9119c1ff3de13d66f857d85cd45caff"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8d470fef39d02dbe5c541ec345cc4ffd7d2baec7d6e59c92bd9d9545dc221829"}, + {file = "hiredis-3.2.1-cp310-cp310-win32.whl", hash = "sha256:efa4c76c45cc8c42228c7989b279fa974580e053b5e6a4a834098b5324b9eafa"}, + {file = "hiredis-3.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:cbac5ec3a620b095c46ef3a8f1f06da9c86c1cdc411d44a5f538876c39a2b321"}, + {file = "hiredis-3.2.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:e4ae0be44cab5e74e6e4c4a93d04784629a45e781ff483b136cc9e1b9c23975c"}, + {file = "hiredis-3.2.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:24647e84c9f552934eb60b7f3d2116f8b64a7020361da9369e558935ca45914d"}, + {file = "hiredis-3.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6fb3e92d1172da8decc5f836bf8b528c0fc9b6d449f1353e79ceeb9dc1801132"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38ba7a32e51e518b6b3e470142e52ed2674558e04d7d73d86eb19ebcb37d7d40"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4fc632be73174891d6bb71480247e57b2fd8f572059f0a1153e4d0339e919779"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f03e6839ff21379ad3c195e0700fc9c209e7f344946dea0f8a6d7b5137a2a141"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99983873e37c71bb71deb544670ff4f9d6920dab272aaf52365606d87a4d6c73"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffd982c419f48e3a57f592678c72474429465bb4bfc96472ec805f5d836523f0"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bc993f4aa4abc029347f309e722f122e05a3b8a0c279ae612849b5cc9dc69f2d"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:dde790d420081f18b5949227649ccb3ed991459df33279419a25fcae7f97cd92"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b0c8cae7edbef860afcf3177b705aef43e10b5628f14d5baf0ec69668247d08d"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e8a90eaca7e1ce7f175584f07a2cdbbcab13f4863f9f355d7895c4d28805f65b"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:476031958fa44e245e803827e0787d49740daa4de708fe514370293ce519893a"}, + {file = "hiredis-3.2.1-cp311-cp311-win32.whl", hash = "sha256:eb3f5df2a9593b4b4b676dce3cea53b9c6969fc372875188589ddf2bafc7f624"}, + {file = "hiredis-3.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:1402e763d8a9fdfcc103bbf8b2913971c0a3f7b8a73deacbda3dfe5f3a9d1e0b"}, + {file = "hiredis-3.2.1-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:3742d8b17e73c198cabeab11da35f2e2a81999d406f52c6275234592256bf8e8"}, + {file = "hiredis-3.2.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9c2f3176fb617a79f6cccf22cb7d2715e590acb534af6a82b41f8196ad59375d"}, + {file = "hiredis-3.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a8bd46189c7fa46174e02670dc44dfecb60f5bd4b67ed88cb050d8f1fd842f09"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f86ee4488c8575b58139cdfdddeae17f91e9a893ffee20260822add443592e2f"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3717832f4a557b2fe7060b9d4a7900e5de287a15595e398c3f04df69019ca69d"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5cb12c21fb9e2403d28c4e6a38120164973342d34d08120f2d7009b66785644"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:080fda1510bbd389af91f919c11a4f2aa4d92f0684afa4709236faa084a42cac"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1252e10a1f3273d1c6bf2021e461652c2e11b05b83e0915d6eb540ec7539afe2"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d9e320e99ab7d2a30dc91ff6f745ba38d39b23f43d345cdee9881329d7b511d6"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:641668f385f16550fdd6fdc109b0af6988b94ba2acc06770a5e06a16e88f320c"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1e1f44208c39d6c345ff451f82f21e9eeda6fe9af4ac65972cc3eeb58d41f7cb"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f882a0d6415fffe1ffcb09e6281d0ba8b1ece470e866612bbb24425bf76cf397"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b4e78719a0730ebffe335528531d154bc8867a246418f74ecd88adbc4d938c49"}, + {file = "hiredis-3.2.1-cp312-cp312-win32.whl", hash = "sha256:33c4604d9f79a13b84da79950a8255433fca7edaf292bbd3364fd620864ed7b2"}, + {file = "hiredis-3.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7b9749375bf9d171aab8813694f379f2cff0330d7424000f5e92890ad4932dc9"}, + {file = "hiredis-3.2.1-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:7cabf7f1f06be221e1cbed1f34f00891a7bdfad05b23e4d315007dd42148f3d4"}, + {file = "hiredis-3.2.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:db85cb86f8114c314d0ec6d8de25b060a2590b4713135240d568da4f7dea97ac"}, + {file = "hiredis-3.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c9a592a49b7b8497e4e62c3ff40700d0c7f1a42d145b71e3e23c385df573c964"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0079ef1e03930b364556b78548e67236ab3def4e07e674f6adfc52944aa972dd"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d6a290ed45d9c14f4c50b6bda07afb60f270c69b5cb626fd23a4c2fde9e3da1"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79dd5fe8c0892769f82949adeb021342ca46871af26e26945eb55d044fcdf0d0"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:998a82281a159f4aebbfd4fb45cfe24eb111145206df2951d95bc75327983b58"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41fc3cd52368ffe7c8e489fb83af5e99f86008ed7f9d9ba33b35fec54f215c0a"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8d10df3575ce09b0fa54b8582f57039dcbdafde5de698923a33f601d2e2a246c"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1ab010d04be33735ad8e643a40af0d68a21d70a57b1d0bff9b6a66b28cca9dbf"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ec3b5f9ea34f70aaba3e061cbe1fa3556fea401d41f5af321b13e326792f3017"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:158dfb505fff6bffd17f823a56effc0c2a7a8bc4fb659d79a52782f22eefc697"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d632cd0ddd7895081be76748e6fb9286f81d2a51c371b516541c6324f2fdac9"}, + {file = "hiredis-3.2.1-cp313-cp313-win32.whl", hash = "sha256:e9726d03e7df068bf755f6d1ecc61f7fc35c6b20363c7b1b96f39a14083df940"}, + {file = "hiredis-3.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:b5b1653ad7263a001f2e907e81a957d6087625f9700fa404f1a2268c0a4f9059"}, + {file = "hiredis-3.2.1-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:ef27728a8ceaa038ef4b6efc0e4473b7643b5c873c2fff5475e2c8b9c8d2e0d5"}, + {file = "hiredis-3.2.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:1039d8d2e1d2a1528ad9f9e289e8aa8eec9bf4b4759be4d453a2ab406a70a800"}, + {file = "hiredis-3.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83a8cd0eb6e535c93aad9c21e3e85bcb7dd26d3ff9b8ab095287be86e8af2f59"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6fc1e8f78bcdc7e25651b7d96d19b983b843b575904d96642f97ae157797ae4"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ddfa9a10fda3bea985a3b371a64553731141aaa0a20cbcc62a0e659f05e6c01"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e789ee008752b9be82a7bed82e36b62053c7cc06a0179a5a403ba5b2acba5bd8"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bf271877947a0f3eb9dc331688404a2e4cc246bca61bc5a1e2d62da9a1caad8"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9ad404fd0fdbdfe74e55ebb0592ab4169eecfe70ccf0db80eedc1d9943dd6d7"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:979572c602bdea0c3df255545c8c257f2163dd6c10d1f172268ffa7a6e1287d6"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:f74e3d899be057fb00444ea5f7ae1d7389d393bddf0f3ed698997aa05563483b"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:a015666d5fdc3ca704f68db9850d0272ddcfb27e9f26a593013383f565ed2ad7"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:259a3389dfe3390e356c2796b6bc96a778695e9d7d40c82121096a6b8a2dd3c6"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:39f469891d29f0522712265de76018ab83a64b85ac4b4f67e1f692cbd42a03f9"}, + {file = "hiredis-3.2.1-cp38-cp38-win32.whl", hash = "sha256:73aa0508f26cd6cb4dfdbe189b28fb3162fd171532e526e90a802363b88027f8"}, + {file = "hiredis-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:2b910f12d7bcaf5ffc056087fc7b2d23e688f166462c31b73a0799d12891378d"}, + {file = "hiredis-3.2.1-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:523a241d9f268bc0c7306792f58f9c633185f939a19abc0356c55f078d3901c5"}, + {file = "hiredis-3.2.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:fec453a038c262e18d7de4919220b2916e0b17d1eadd12e7a800f09f78f84f39"}, + {file = "hiredis-3.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e75a49c5927453c316665cfa39f4274081d00ce69b137b393823eb90c66a8371"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd974cbe8b3ae8d3e7f60675e6da10383da69f029147c2c93d1a7e44b36d1290"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12d3b8fff9905e44f357417159d64138a32500dbd0d5cffaddbb2600d3ce33b1"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e21985804a40cb91e69e35ae321eb4e3610cd61a2cbc0328ab73a245f608fa1c"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e26e2b49a9569f44a2a2d743464ff0786b46fb1124ed33d2a1bd8b1c660c25b"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ef1ebf9ee8e0b4a895b86a02a8b7e184b964c43758393532966ecb8a256f37c"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c936b690dd31d7af74f707fc9003c500315b4c9ad70fa564aff73d1283b3b37a"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4909666bcb73270bb806aa00d0eee9e81f7a1aca388aafb4ba7dfcf5d344d23a"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d74a2ad25bc91ca9639e4485099852e6263b360b2c3650fdd3cc47762c5db3fa"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:e99910088df446ee64d64b160835f592fb4d36189fcc948dd204e903d91fffa3"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:54423bd7af93a773edc6f166341cfb0e5f35ef42ca07b93f568f672a6f445e40"}, + {file = "hiredis-3.2.1-cp39-cp39-win32.whl", hash = "sha256:4a5365cb6d7be82d3c6d523b369bc0bc1a64987e88ed6ecfabadda2aa1cf4fa4"}, + {file = "hiredis-3.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a2eb02b6aaf4f1425a408e892c0378ba6cb6b45b1412c30dd258df1322d88c0"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:73913d2fa379e722d17ba52f21ce12dd578140941a08efd73e73b6fab1dea4d8"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:15a3dff3eca31ecbf3d7d6d104cf1b318dc2b013bad3f4bdb2839cb9ea2e1584"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c78258032c2f9fc6f39fee7b07882ce26de281e09178266ce535992572132d95"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:578d6a881e64e46db065256355594e680202c3bacf3270be3140057171d2c23e"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b7f34b170093c077c972b8cc0ceb15d8ff88ad0079751a8ae9733e94d77e733"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:291a18b228fc90f6720d178de2fac46522082c96330b4cc2d3dd8cb2c1cb2815"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f53d2af5a7cd33a4b4d7ba632dce80c17823df6814ef5a8d328ed44c815a68e7"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:20bdf6dbdf77eb43b98bc53950f7711983042472199245d4c36448e6b4cb460f"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f43e5c50d76da15118c72b757216cf26c643d55bb1b3c86cad1ae49173971780"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e5bb5fe9834851d56c8543e52dcd2ac5275fb6772ebc97876e18c2e05a3300b"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e348438b6452e3d14dddb95d071fe8eaf6f264f641cba999c10bf6359cf1d2"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e305f6c63a2abcbde6ce28958de2bb4dd0fd34c6ab3bde5a4410befd5df8c6b2"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:33f24b1152f684b54d6b9d09135d849a6df64b6982675e8cf972f8adfa2de9aa"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:01dd8ea88bf8363751857ca2eb8f13faad0c7d57a6369663d4d1160f225ab449"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b16946533535cbb5cc7d4b6fc009d32d22b0f9ac58e8eb6f144637b64f9a61d"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9a03886cad1076e9f7e9e411c402826a8eac6f56ba426ee84b88e6515574b7b"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a4f6340f1c378bce17c195d46288a796fcf213dd3e2a008c2c942b33ab58993"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:9d64ddf29016d34e7e3bc4b3d36ca9ac8a94f9b2c13ac4b9d8a486862d91b95c"}, + {file = "hiredis-3.2.1.tar.gz", hash = "sha256:5a5f64479bf04dd829fe7029fad0ea043eac4023abc6e946668cbbec3493a78d"}, ] [[package]] @@ -597,6 +630,7 @@ version = "21.0.0" description = "A featureful, immutable, and correct URL for Python." optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] files = [ {file = "hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4"}, {file = "hyperlink-21.0.0.tar.gz", hash = "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b"}, @@ -605,12 +639,33 @@ files = [ [package.dependencies] idna = ">=2.5" +[[package]] +name = "id" +version = "1.5.0" +description = "A tool for generating OIDC identities" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "id-1.5.0-py3-none-any.whl", hash = "sha256:f1434e1cef91f2cbb8a4ec64663d5a23b9ed43ef44c4c957d02583d61714c658"}, + {file = "id-1.5.0.tar.gz", hash = "sha256:292cb8a49eacbbdbce97244f47a97b4c62540169c976552e497fd57df0734c1d"}, +] + +[package.dependencies] +requests = "*" + +[package.extras] +dev = ["build", "bump (>=1.3.2)", "id[lint,test]"] +lint = ["bandit", "interrogate", "mypy", "ruff (<0.8.2)", "types-requests"] +test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"] + [[package]] name = "idna" version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main", "dev"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -621,105 +676,97 @@ all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2 [[package]] name = "ijson" -version = "3.3.0" +version = "3.4.0" description = "Iterative JSON parser with standard Python iterator interfaces" optional = false -python-versions = "*" +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "ijson-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7f7a5250599c366369fbf3bc4e176f5daa28eb6bc7d6130d02462ed335361675"}, - {file = "ijson-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f87a7e52f79059f9c58f6886c262061065eb6f7554a587be7ed3aa63e6b71b34"}, - {file = "ijson-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b73b493af9e947caed75d329676b1b801d673b17481962823a3e55fe529c8b8b"}, - {file = "ijson-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5576415f3d76290b160aa093ff968f8bf6de7d681e16e463a0134106b506f49"}, - {file = "ijson-3.3.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e9ffe358d5fdd6b878a8a364e96e15ca7ca57b92a48f588378cef315a8b019e"}, - {file = "ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8643c255a25824ddd0895c59f2319c019e13e949dc37162f876c41a283361527"}, - {file = "ijson-3.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:df3ab5e078cab19f7eaeef1d5f063103e1ebf8c26d059767b26a6a0ad8b250a3"}, - {file = "ijson-3.3.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3dc1fb02c6ed0bae1b4bf96971258bf88aea72051b6e4cebae97cff7090c0607"}, - {file = "ijson-3.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e9afd97339fc5a20f0542c971f90f3ca97e73d3050cdc488d540b63fae45329a"}, - {file = "ijson-3.3.0-cp310-cp310-win32.whl", hash = "sha256:844c0d1c04c40fd1b60f148dc829d3f69b2de789d0ba239c35136efe9a386529"}, - {file = "ijson-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:d654d045adafdcc6c100e8e911508a2eedbd2a1b5f93f930ba13ea67d7704ee9"}, - {file = "ijson-3.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:501dce8eaa537e728aa35810656aa00460a2547dcb60937c8139f36ec344d7fc"}, - {file = "ijson-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:658ba9cad0374d37b38c9893f4864f284cdcc7d32041f9808fba8c7bcaadf134"}, - {file = "ijson-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2636cb8c0f1023ef16173f4b9a233bcdb1df11c400c603d5f299fac143ca8d70"}, - {file = "ijson-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd174b90db68c3bcca273e9391934a25d76929d727dc75224bf244446b28b03b"}, - {file = "ijson-3.3.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:97a9aea46e2a8371c4cf5386d881de833ed782901ac9f67ebcb63bb3b7d115af"}, - {file = "ijson-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c594c0abe69d9d6099f4ece17763d53072f65ba60b372d8ba6de8695ce6ee39e"}, - {file = "ijson-3.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8e0ff16c224d9bfe4e9e6bd0395826096cda4a3ef51e6c301e1b61007ee2bd24"}, - {file = "ijson-3.3.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0015354011303175eae7e2ef5136414e91de2298e5a2e9580ed100b728c07e51"}, - {file = "ijson-3.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:034642558afa57351a0ffe6de89e63907c4cf6849070cc10a3b2542dccda1afe"}, - {file = "ijson-3.3.0-cp311-cp311-win32.whl", hash = "sha256:192e4b65495978b0bce0c78e859d14772e841724d3269fc1667dc6d2f53cc0ea"}, - {file = "ijson-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:72e3488453754bdb45c878e31ce557ea87e1eb0f8b4fc610373da35e8074ce42"}, - {file = "ijson-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:988e959f2f3d59ebd9c2962ae71b97c0df58323910d0b368cc190ad07429d1bb"}, - {file = "ijson-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b2f73f0d0fce5300f23a1383d19b44d103bb113b57a69c36fd95b7c03099b181"}, - {file = "ijson-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0ee57a28c6bf523d7cb0513096e4eb4dac16cd935695049de7608ec110c2b751"}, - {file = "ijson-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0155a8f079c688c2ccaea05de1ad69877995c547ba3d3612c1c336edc12a3a5"}, - {file = "ijson-3.3.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ab00721304af1ae1afa4313ecfa1bf16b07f55ef91e4a5b93aeaa3e2bd7917c"}, - {file = "ijson-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40ee3821ee90be0f0e95dcf9862d786a7439bd1113e370736bfdf197e9765bfb"}, - {file = "ijson-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:da3b6987a0bc3e6d0f721b42c7a0198ef897ae50579547b0345f7f02486898f5"}, - {file = "ijson-3.3.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:63afea5f2d50d931feb20dcc50954e23cef4127606cc0ecf7a27128ed9f9a9e6"}, - {file = "ijson-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b5c3e285e0735fd8c5a26d177eca8b52512cdd8687ca86ec77a0c66e9c510182"}, - {file = "ijson-3.3.0-cp312-cp312-win32.whl", hash = "sha256:907f3a8674e489abdcb0206723e5560a5cb1fa42470dcc637942d7b10f28b695"}, - {file = "ijson-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f890d04ad33262d0c77ead53c85f13abfb82f2c8f078dfbf24b78f59534dfdd"}, - {file = "ijson-3.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b9d85a02e77ee8ea6d9e3fd5d515bcc3d798d9c1ea54817e5feb97a9bc5d52fe"}, - {file = "ijson-3.3.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6576cdc36d5a09b0c1a3d81e13a45d41a6763188f9eaae2da2839e8a4240bce"}, - {file = "ijson-3.3.0-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5589225c2da4bb732c9c370c5961c39a6db72cf69fb2a28868a5413ed7f39e6"}, - {file = "ijson-3.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad04cf38164d983e85f9cba2804566c0160b47086dcca4cf059f7e26c5ace8ca"}, - {file = "ijson-3.3.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:a3b730ef664b2ef0e99dec01b6573b9b085c766400af363833e08ebc1e38eb2f"}, - {file = "ijson-3.3.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:4690e3af7b134298055993fcbea161598d23b6d3ede11b12dca6815d82d101d5"}, - {file = "ijson-3.3.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:aaa6bfc2180c31a45fac35d40e3312a3d09954638ce0b2e9424a88e24d262a13"}, - {file = "ijson-3.3.0-cp36-cp36m-win32.whl", hash = "sha256:44367090a5a876809eb24943f31e470ba372aaa0d7396b92b953dda953a95d14"}, - {file = "ijson-3.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7e2b3e9ca957153557d06c50a26abaf0d0d6c0ddf462271854c968277a6b5372"}, - {file = "ijson-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:47c144117e5c0e2babb559bc8f3f76153863b8dd90b2d550c51dab5f4b84a87f"}, - {file = "ijson-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29ce02af5fbf9ba6abb70765e66930aedf73311c7d840478f1ccecac53fefbf3"}, - {file = "ijson-3.3.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ac6c3eeed25e3e2cb9b379b48196413e40ac4e2239d910bb33e4e7f6c137745"}, - {file = "ijson-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d92e339c69b585e7b1d857308ad3ca1636b899e4557897ccd91bb9e4a56c965b"}, - {file = "ijson-3.3.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:8c85447569041939111b8c7dbf6f8fa7a0eb5b2c4aebb3c3bec0fb50d7025121"}, - {file = "ijson-3.3.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:542c1e8fddf082159a5d759ee1412c73e944a9a2412077ed00b303ff796907dc"}, - {file = "ijson-3.3.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:30cfea40936afb33b57d24ceaf60d0a2e3d5c1f2335ba2623f21d560737cc730"}, - {file = "ijson-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:6b661a959226ad0d255e49b77dba1d13782f028589a42dc3172398dd3814c797"}, - {file = "ijson-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:0b003501ee0301dbf07d1597482009295e16d647bb177ce52076c2d5e64113e0"}, - {file = "ijson-3.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3e8d8de44effe2dbd0d8f3eb9840344b2d5b4cc284a14eb8678aec31d1b6bea8"}, - {file = "ijson-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9cd5c03c63ae06d4f876b9844c5898d0044c7940ff7460db9f4cd984ac7862b5"}, - {file = "ijson-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04366e7e4a4078d410845e58a2987fd9c45e63df70773d7b6e87ceef771b51ee"}, - {file = "ijson-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de7c1ddb80fa7a3ab045266dca169004b93f284756ad198306533b792774f10a"}, - {file = "ijson-3.3.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8851584fb931cffc0caa395f6980525fd5116eab8f73ece9d95e6f9c2c326c4c"}, - {file = "ijson-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdcfc88347fd981e53c33d832ce4d3e981a0d696b712fbcb45dcc1a43fe65c65"}, - {file = "ijson-3.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3917b2b3d0dbbe3296505da52b3cb0befbaf76119b2edaff30bd448af20b5400"}, - {file = "ijson-3.3.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:e10c14535abc7ddf3fd024aa36563cd8ab5d2bb6234a5d22c77c30e30fa4fb2b"}, - {file = "ijson-3.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3aba5c4f97f4e2ce854b5591a8b0711ca3b0c64d1b253b04ea7b004b0a197ef6"}, - {file = "ijson-3.3.0-cp38-cp38-win32.whl", hash = "sha256:b325f42e26659df1a0de66fdb5cde8dd48613da9c99c07d04e9fb9e254b7ee1c"}, - {file = "ijson-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:ff835906f84451e143f31c4ce8ad73d83ef4476b944c2a2da91aec8b649570e1"}, - {file = "ijson-3.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3c556f5553368dff690c11d0a1fb435d4ff1f84382d904ccc2dc53beb27ba62e"}, - {file = "ijson-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e4396b55a364a03ff7e71a34828c3ed0c506814dd1f50e16ebed3fc447d5188e"}, - {file = "ijson-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e6850ae33529d1e43791b30575070670070d5fe007c37f5d06aebc1dd152ab3f"}, - {file = "ijson-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36aa56d68ea8def26778eb21576ae13f27b4a47263a7a2581ab2ef58b8de4451"}, - {file = "ijson-3.3.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7ec759c4a0fc820ad5dc6a58e9c391e7b16edcb618056baedbedbb9ea3b1524"}, - {file = "ijson-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b51bab2c4e545dde93cb6d6bb34bf63300b7cd06716f195dd92d9255df728331"}, - {file = "ijson-3.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:92355f95a0e4da96d4c404aa3cff2ff033f9180a9515f813255e1526551298c1"}, - {file = "ijson-3.3.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8795e88adff5aa3c248c1edce932db003d37a623b5787669ccf205c422b91e4a"}, - {file = "ijson-3.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:8f83f553f4cde6d3d4eaf58ec11c939c94a0ec545c5b287461cafb184f4b3a14"}, - {file = "ijson-3.3.0-cp39-cp39-win32.whl", hash = "sha256:ead50635fb56577c07eff3e557dac39533e0fe603000684eea2af3ed1ad8f941"}, - {file = "ijson-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:c8a9befb0c0369f0cf5c1b94178d0d78f66d9cebb9265b36be6e4f66236076b8"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2af323a8aec8a50fa9effa6d640691a30a9f8c4925bd5364a1ca97f1ac6b9b5c"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f64f01795119880023ba3ce43072283a393f0b90f52b66cc0ea1a89aa64a9ccb"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a716e05547a39b788deaf22725490855337fc36613288aa8ae1601dc8c525553"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:473f5d921fadc135d1ad698e2697025045cd8ed7e5e842258295012d8a3bc702"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dd26b396bc3a1e85f4acebeadbf627fa6117b97f4c10b177d5779577c6607744"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:25fd49031cdf5fd5f1fd21cb45259a64dad30b67e64f745cc8926af1c8c243d3"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b72178b1e565d06ab19319965022b36ef41bcea7ea153b32ec31194bec032a2"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d0b6b637d05dbdb29d0bfac2ed8425bb369e7af5271b0cc7cf8b801cb7360c2"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5378d0baa59ae422905c5f182ea0fd74fe7e52a23e3821067a7d58c8306b2191"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:99f5c8ab048ee4233cc4f2b461b205cbe01194f6201018174ac269bf09995749"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:45ff05de889f3dc3d37a59d02096948ce470699f2368b32113954818b21aa74a"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1efb521090dd6cefa7aafd120581947b29af1713c902ff54336b7c7130f04c47"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87c727691858fd3a1c085d9980d12395517fcbbf02c69fbb22dede8ee03422da"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0420c24e50389bc251b43c8ed379ab3e3ba065ac8262d98beb6735ab14844460"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8fdf3721a2aa7d96577970f5604bd81f426969c1822d467f07b3d844fa2fecc7"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:891f95c036df1bc95309951940f8eea8537f102fa65715cdc5aae20b8523813b"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed1336a2a6e5c427f419da0154e775834abcbc8ddd703004108121c6dd9eba9d"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0c819f83e4f7b7f7463b2dc10d626a8be0c85fbc7b3db0edc098c2b16ac968e"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33afc25057377a6a43c892de34d229a86f89ea6c4ca3dd3db0dcd17becae0dbb"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7914d0cf083471856e9bc2001102a20f08e82311dfc8cf1a91aa422f9414a0d6"}, - {file = "ijson-3.3.0.tar.gz", hash = "sha256:7f172e6ba1bee0d4c8f8ebd639577bfe429dee0f3f96775a067b8bae4492d8a0"}, + {file = "ijson-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e27e50f6dcdee648f704abc5d31b976cd2f90b4642ed447cf03296d138433d09"}, + {file = "ijson-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a753be681ac930740a4af9c93cfb4edc49a167faed48061ea650dc5b0f406f1"}, + {file = "ijson-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a07c47aed534e0ec198e6a2d4360b259d32ac654af59c015afc517ad7973b7fb"}, + {file = "ijson-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c55f48181e11c597cd7146fb31edc8058391201ead69f8f40d2ecbb0b3e4fc6"}, + {file = "ijson-3.4.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd5669f96f79d8a2dd5ae81cbd06770a4d42c435fd4a75c74ef28d9913b697d"}, + {file = "ijson-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e3ddd46d16b8542c63b1b8af7006c758d4e21cc1b86122c15f8530fae773461"}, + {file = "ijson-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1504cec7fe04be2bb0cc33b50c9dd3f83f98c0540ad4991d4017373b7853cfe6"}, + {file = "ijson-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2f2ff456adeb216603e25d7915f10584c1b958b6eafa60038d76d08fc8a5fb06"}, + {file = "ijson-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0ab00d75d61613a125fbbb524551658b1ad6919a52271ca16563ca5bc2737bb1"}, + {file = "ijson-3.4.0-cp310-cp310-win32.whl", hash = "sha256:ada421fd59fe2bfa4cfa64ba39aeba3f0753696cdcd4d50396a85f38b1d12b01"}, + {file = "ijson-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:8c75e82cec05d00ed3a4af5f4edf08f59d536ed1a86ac7e84044870872d82a33"}, + {file = "ijson-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9e369bf5a173ca51846c243002ad8025d32032532523b06510881ecc8723ee54"}, + {file = "ijson-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:26e7da0a3cd2a56a1fde1b34231867693f21c528b683856f6691e95f9f39caec"}, + {file = "ijson-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c28c7f604729be22aa453e604e9617b665fa0c24cd25f9f47a970e8130c571a"}, + {file = "ijson-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bed8bcb84d3468940f97869da323ba09ae3e6b950df11dea9b62e2b231ca1e3"}, + {file = "ijson-3.4.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:296bc824f4088f2af814aaf973b0435bc887ce3d9f517b1577cc4e7d1afb1cb7"}, + {file = "ijson-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8145f8f40617b6a8aa24e28559d0adc8b889e56a203725226a8a60fa3501073f"}, + {file = "ijson-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b674a97bd503ea21bc85103e06b6493b1b2a12da3372950f53e1c664566a33a4"}, + {file = "ijson-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8bc731cf1c3282b021d3407a601a5a327613da9ad3c4cecb1123232623ae1826"}, + {file = "ijson-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:42ace5e940e0cf58c9de72f688d6829ddd815096d07927ee7e77df2648006365"}, + {file = "ijson-3.4.0-cp311-cp311-win32.whl", hash = "sha256:5be39a0df4cd3f02b304382ea8885391900ac62e95888af47525a287c50005e9"}, + {file = "ijson-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:0b1be1781792291e70d2e177acf564ec672a7907ba74f313583bdf39fe81f9b7"}, + {file = "ijson-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:956b148f88259a80a9027ffbe2d91705fae0c004fbfba3e5a24028fbe72311a9"}, + {file = "ijson-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:06b89960f5c721106394c7fba5760b3f67c515b8eb7d80f612388f5eca2f4621"}, + {file = "ijson-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9a0bb591cf250dd7e9dfab69d634745a7f3272d31cfe879f9156e0a081fd97ee"}, + {file = "ijson-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72e92de999977f4c6b660ffcf2b8d59604ccd531edcbfde05b642baf283e0de8"}, + {file = "ijson-3.4.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e9602157a5b869d44b6896e64f502c712a312fcde044c2e586fccb85d3e316e"}, + {file = "ijson-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e83660edb931a425b7ff662eb49db1f10d30ca6d4d350e5630edbed098bc01"}, + {file = "ijson-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:49bf8eac1c7b7913073865a859c215488461f7591b4fa6a33c14b51cb73659d0"}, + {file = "ijson-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:160b09273cb42019f1811469508b0a057d19f26434d44752bde6f281da6d3f32"}, + {file = "ijson-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2019ff4e6f354aa00c76c8591bd450899111c61f2354ad55cc127e2ce2492c44"}, + {file = "ijson-3.4.0-cp312-cp312-win32.whl", hash = "sha256:931c007bf6bb8330705429989b2deed6838c22b63358a330bf362b6e458ba0bf"}, + {file = "ijson-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:71523f2b64cb856a820223e94d23e88369f193017ecc789bb4de198cc9d349eb"}, + {file = "ijson-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e8d96f88d75196a61c9d9443de2b72c2d4a7ba9456ff117b57ae3bba23a54256"}, + {file = "ijson-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c45906ce2c1d3b62f15645476fc3a6ca279549127f01662a39ca5ed334a00cf9"}, + {file = "ijson-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4ab4bc2119b35c4363ea49f29563612237cae9413d2fbe54b223be098b97bc9e"}, + {file = "ijson-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97b0a9b5a15e61dfb1f14921ea4e0dba39f3a650df6d8f444ddbc2b19b479ff1"}, + {file = "ijson-3.4.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3047bb994dabedf11de11076ed1147a307924b6e5e2df6784fb2599c4ad8c60"}, + {file = "ijson-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68c83161b052e9f5dc8191acbc862bb1e63f8a35344cb5cd0db1afd3afd487a6"}, + {file = "ijson-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1eebd9b6c20eb1dffde0ae1f0fbb4aeacec2eb7b89adb5c7c0449fc9fd742760"}, + {file = "ijson-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:13fb6d5c35192c541421f3ee81239d91fc15a8d8f26c869250f941f4b346a86c"}, + {file = "ijson-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:28b7196ff7b37c4897c547a28fa4876919696739fc91c1f347651c9736877c69"}, + {file = "ijson-3.4.0-cp313-cp313-win32.whl", hash = "sha256:3c2691d2da42629522140f77b99587d6f5010440d58d36616f33bc7bdc830cc3"}, + {file = "ijson-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:c4554718c275a044c47eb3874f78f2c939f300215d9031e785a6711cc51b83fc"}, + {file = "ijson-3.4.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:915a65e3f3c0eee2ea937bc62aaedb6c14cc1e8f0bb9f3f4fb5a9e2bbfa4b480"}, + {file = "ijson-3.4.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:afbe9748707684b6c5adc295c4fdcf27765b300aec4d484e14a13dca4e5c0afa"}, + {file = "ijson-3.4.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d823f8f321b4d8d5fa020d0a84f089fec5d52b7c0762430476d9f8bf95bbc1a9"}, + {file = "ijson-3.4.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8a0a2c54f3becf76881188beefd98b484b1d3bd005769a740d5b433b089fa23"}, + {file = "ijson-3.4.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ced19a83ab09afa16257a0b15bc1aa888dbc555cb754be09d375c7f8d41051f2"}, + {file = "ijson-3.4.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8100f9885eff1f38d35cef80ef759a1bbf5fc946349afa681bd7d0e681b7f1a0"}, + {file = "ijson-3.4.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d7bcc3f7f21b0f703031ecd15209b1284ea51b2a329d66074b5261de3916c1eb"}, + {file = "ijson-3.4.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:2dcb190227b09dd171bdcbfe4720fddd574933c66314818dfb3960c8a6246a77"}, + {file = "ijson-3.4.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:eda4cfb1d49c6073a901735aaa62e39cb7ab47f3ad7bb184862562f776f1fa8a"}, + {file = "ijson-3.4.0-cp313-cp313t-win32.whl", hash = "sha256:0772638efa1f3b72b51736833404f1cbd2f5beeb9c1a3d392e7d385b9160cba7"}, + {file = "ijson-3.4.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3d8a0d67f36e4fb97c61a724456ef0791504b16ce6f74917a31c2e92309bbeb9"}, + {file = "ijson-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8a990401dc7350c1739f42187823e68d2ef6964b55040c6e9f3a29461f9929e2"}, + {file = "ijson-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:80f50e0f5da4cd6b65e2d8ff38cb61b26559608a05dd3a3f9cfa6f19848e6f22"}, + {file = "ijson-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2d9ca52f5650d820a2e7aa672dea1c560f609e165337e5b3ed7cf56d696bf309"}, + {file = "ijson-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:940c8c5fd20fb89b56dde9194a4f1c7b779149f1ab26af6d8dc1da51a95d26dd"}, + {file = "ijson-3.4.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:41dbb525666017ad856ac9b4f0f4b87d3e56b7dfde680d5f6d123556b22e2172"}, + {file = "ijson-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9f84f5e2eea5c2d271c97221c382db005534294d1175ddd046a12369617c41c"}, + {file = "ijson-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c0cd126c11835839bba8ac0baaba568f67d701fc4f717791cf37b10b74a2ebd7"}, + {file = "ijson-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f9a9d3bbc6d91c24a2524a189d2aca703cb5f7e8eb34ad0aff3c91702404a983"}, + {file = "ijson-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:56679ee133470d0f1f598a8ad109d760fcfebeef4819531e29335aefb7e4cb1a"}, + {file = "ijson-3.4.0-cp39-cp39-win32.whl", hash = "sha256:583c15ded42ba80104fa1d0fa0dfdd89bb47922f3bb893a931bb843aeb55a3f3"}, + {file = "ijson-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:4563e603e56f4451572d96b47311dffef5b933d825f3417881d4d3630c6edac2"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:54e989c35dba9cf163d532c14bcf0c260897d5f465643f0cd1fba9c908bed7ef"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:494eeb8e87afef22fbb969a4cb81ac2c535f30406f334fb6136e9117b0bb5380"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81603de95de1688958af65cd2294881a4790edae7de540b70c65c8253c5dc44a"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8524be12c1773e1be466034cc49c1ecbe3d5b47bb86217bd2a57f73f970a6c19"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17994696ec895d05e0cfa21b11c68c920c82634b4a3d8b8a1455d6fe9fdee8f7"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0b67727aaee55d43b2e82b6a866c3cbcb2b66a5e9894212190cbd8773d0d9857"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cdc8c5ca0eec789ed99db29c68012dda05027af0860bb360afd28d825238d69d"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:8e6b44b6ec45d5b1a0ee9d97e0e65ab7f62258727004cbbe202bf5f198bc21f7"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b51e239e4cb537929796e840d349fc731fdc0d58b1a0683ce5465ad725321e0f"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed05d43ec02be8ddb1ab59579761f6656b25d241a77fd74f4f0f7ec09074318a"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfeca1aaa59d93fd0a3718cbe5f7ef0effff85cf837e0bceb71831a47f39cc14"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7ca72ca12e9a1dd4252c97d952be34282907f263f7e28fcdff3a01b83981e837"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0f79b2cd52bd220fff83b3ee4ef89b54fd897f57cc8564a6d8ab7ac669de3930"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d16eed737610ad5ad8989b5864fbe09c64133129734e840c29085bb0d497fb03"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b3aac1d7a27e1e3bdec5bd0689afe55c34aa499baa06a80852eda31f1ffa6dc"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:784ae654aa9851851e87f323e9429b20b58a5399f83e6a7e348e080f2892081f"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d05bd8fa6a8adefb32bbf7b993d2a2f4507db08453dd1a444c281413a6d9685"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b5a05fd935cc28786b88c16976313086cd96414c6a3eb0a3822c47ab48b1793e"}, + {file = "ijson-3.4.0.tar.gz", hash = "sha256:5f74dcbad9d592c428d3ca3957f7115a42689ee7ee941458860900236ae9bb13"}, ] [[package]] @@ -728,6 +775,7 @@ version = "4.2.1" description = "Immutable wrapper around dictionaries (a fork of frozendict)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "immutabledict-4.2.1-py3-none-any.whl", hash = "sha256:c56a26ced38c236f79e74af3ccce53772827cef5c3bce7cab33ff2060f756373"}, {file = "immutabledict-4.2.1.tar.gz", hash = "sha256:d91017248981c72eb66c8ff9834e99c2f53562346f23e7f51e7a5ebcf66a3bcc"}, @@ -739,6 +787,8 @@ version = "6.7.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\" or python_version < \"3.10\"" files = [ {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"}, {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"}, @@ -750,7 +800,7 @@ zipp = ">=0.5" [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +testing = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\"", "pytest-perf (>=0.9.2)", "pytest-ruff"] [[package]] name = "importlib-resources" @@ -758,6 +808,8 @@ version = "5.12.0" description = "Read resources from Python packages" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version < \"3.10\"" files = [ {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, @@ -768,7 +820,7 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8 ; python_version < \"3.12\"", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "incremental" @@ -776,6 +828,7 @@ version = "24.7.2" description = "A small library that versions your Python projects." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "incremental-24.7.2-py3-none-any.whl", hash = "sha256:8cb2c3431530bec48ad70513931a760f446ad6c25e8333ca5d95e24b0ed7b8fe"}, {file = "incremental-24.7.2.tar.gz", hash = "sha256:fb4f1d47ee60efe87d4f6f0ebb5f70b9760db2b2574c59c8e8912be4ebd464c9"}, @@ -794,6 +847,8 @@ version = "4.8.0" description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -813,6 +868,8 @@ version = "3.2.3" description = "Utility functions for Python class constructs" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "jaraco.classes-3.2.3-py3-none-any.whl", hash = "sha256:2353de3288bc6b82120752201c6b1c1a14b058267fa424ed5ce5984e3b922158"}, {file = "jaraco.classes-3.2.3.tar.gz", hash = "sha256:89559fa5c1d3c34eff6f631ad80bb21f378dbcbb35dd161fd2c6b93f5be2f98a"}, @@ -823,7 +880,7 @@ more-itertools = "*" [package.extras] docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "jeepney" @@ -831,6 +888,8 @@ version = "0.8.0" description = "Low-level, pure Python DBus protocol wrapper." optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and sys_platform == \"linux\"" files = [ {file = "jeepney-0.8.0-py3-none-any.whl", hash = "sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755"}, {file = "jeepney-0.8.0.tar.gz", hash = "sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806"}, @@ -838,17 +897,18 @@ files = [ [package.extras] test = ["async-timeout", "pytest", "pytest-asyncio (>=0.17)", "pytest-trio", "testpath", "trio"] -trio = ["async_generator", "trio"] +trio = ["async_generator ; python_version == \"3.6\"", "trio"] [[package]] name = "jinja2" -version = "3.1.4" +version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, ] [package.dependencies] @@ -859,13 +919,14 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jsonschema" -version = "4.23.0" +version = "4.25.1" description = "An implementation of JSON Schema validation for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, - {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, + {file = "jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63"}, + {file = "jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85"}, ] [package.dependencies] @@ -876,7 +937,7 @@ rpds-py = ">=0.7.1" [package.extras] format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] -format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=24.6.0)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "rfc3987-syntax (>=1.1.0)", "uri-template", "webcolors (>=24.6.0)"] [[package]] name = "jsonschema-specifications" @@ -884,6 +945,7 @@ version = "2023.6.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "jsonschema_specifications-2023.6.1-py3-none-any.whl", hash = "sha256:3d2b82663aff01815f744bb5c7887e2121a63399b49b104a3c96145474d091d7"}, {file = "jsonschema_specifications-2023.6.1.tar.gz", hash = "sha256:ca1c4dd059a9e7b34101cf5b3ab7ff1d18b139f35950d598d629837ef66e8f28"}, @@ -898,6 +960,8 @@ version = "23.13.1" description = "Store and access your passwords safely." optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "keyring-23.13.1-py3-none-any.whl", hash = "sha256:771ed2a91909389ed6148631de678f82ddc73737d85a927f382a8a1b157898cd"}, {file = "keyring-23.13.1.tar.gz", hash = "sha256:ba2e15a9b35e21908d0aaf4e0a47acc52d6ae33444df0da2b49d41a46ef6d678"}, @@ -913,7 +977,7 @@ SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} [package.extras] completion = ["shtab"] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8 ; python_version < \"3.12\"", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "ldap3" @@ -921,6 +985,8 @@ version = "2.9.1" description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -931,157 +997,114 @@ pyasn1 = ">=0.4.6" [[package]] name = "lxml" -version = "5.3.0" +version = "6.0.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = true -python-versions = ">=3.6" +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ - {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, - {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32"}, - {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86"}, - {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5"}, - {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03"}, - {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7"}, - {file = "lxml-5.3.0-cp310-cp310-win32.whl", hash = "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80"}, - {file = "lxml-5.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3"}, - {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b"}, - {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654"}, - {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d"}, - {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763"}, - {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec"}, - {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be"}, - {file = "lxml-5.3.0-cp311-cp311-win32.whl", hash = "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9"}, - {file = "lxml-5.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1"}, - {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859"}, - {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99"}, - {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff"}, - {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a"}, - {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8"}, - {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d"}, - {file = "lxml-5.3.0-cp312-cp312-win32.whl", hash = "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30"}, - {file = "lxml-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f"}, - {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a"}, - {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832"}, - {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff"}, - {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd"}, - {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb"}, - {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b"}, - {file = "lxml-5.3.0-cp313-cp313-win32.whl", hash = "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957"}, - {file = "lxml-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d"}, - {file = "lxml-5.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e"}, - {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f"}, - {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94"}, - {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99"}, - {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237"}, - {file = "lxml-5.3.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577"}, - {file = "lxml-5.3.0-cp36-cp36m-win32.whl", hash = "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70"}, - {file = "lxml-5.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c"}, - {file = "lxml-5.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b"}, - {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5"}, - {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11"}, - {file = "lxml-5.3.0-cp37-cp37m-win32.whl", hash = "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84"}, - {file = "lxml-5.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e"}, - {file = "lxml-5.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920"}, - {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945"}, - {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42"}, - {file = "lxml-5.3.0-cp38-cp38-win32.whl", hash = "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e"}, - {file = "lxml-5.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903"}, - {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de"}, - {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2"}, - {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71"}, - {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3"}, - {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727"}, - {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a"}, - {file = "lxml-5.3.0-cp39-cp39-win32.whl", hash = "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff"}, - {file = "lxml-5.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c"}, - {file = "lxml-5.3.0.tar.gz", hash = "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f"}, + {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35bc626eec405f745199200ccb5c6b36f202675d204aa29bb52e27ba2b71dea8"}, + {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:246b40f8a4aec341cbbf52617cad8ab7c888d944bfe12a6abd2b1f6cfb6f6082"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:2793a627e95d119e9f1e19720730472f5543a6d84c50ea33313ce328d870f2dd"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:46b9ed911f36bfeb6338e0b482e7fe7c27d362c52fde29f221fddbc9ee2227e7"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b4790b558bee331a933e08883c423f65bbcd07e278f91b2272489e31ab1e2b4"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e2030956cf4886b10be9a0285c6802e078ec2391e1dd7ff3eb509c2c95a69b76"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d23854ecf381ab1facc8f353dcd9adeddef3652268ee75297c1164c987c11dc"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:43fe5af2d590bf4691531b1d9a2495d7aab2090547eaacd224a3afec95706d76"}, + {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74e748012f8c19b47f7d6321ac929a9a94ee92ef12bc4298c47e8b7219b26541"}, + {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:43cfbb7db02b30ad3926e8fceaef260ba2fb7df787e38fa2df890c1ca7966c3b"}, + {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:34190a1ec4f1e84af256495436b2d196529c3f2094f0af80202947567fdbf2e7"}, + {file = "lxml-6.0.0-cp310-cp310-win32.whl", hash = "sha256:5967fe415b1920a3877a4195e9a2b779249630ee49ece22021c690320ff07452"}, + {file = "lxml-6.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:f3389924581d9a770c6caa4df4e74b606180869043b9073e2cec324bad6e306e"}, + {file = "lxml-6.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:522fe7abb41309e9543b0d9b8b434f2b630c5fdaf6482bee642b34c8c70079c8"}, + {file = "lxml-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ee56288d0df919e4aac43b539dd0e34bb55d6a12a6562038e8d6f3ed07f9e36"}, + {file = "lxml-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8dd6dd0e9c1992613ccda2bcb74fc9d49159dbe0f0ca4753f37527749885c25"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:d7ae472f74afcc47320238b5dbfd363aba111a525943c8a34a1b657c6be934c3"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5592401cdf3dc682194727c1ddaa8aa0f3ddc57ca64fd03226a430b955eab6f6"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58ffd35bd5425c3c3b9692d078bf7ab851441434531a7e517c4984d5634cd65b"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f720a14aa102a38907c6d5030e3d66b3b680c3e6f6bc95473931ea3c00c59967"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2a5e8d207311a0170aca0eb6b160af91adc29ec121832e4ac151a57743a1e1e"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:2dd1cc3ea7e60bfb31ff32cafe07e24839df573a5e7c2d33304082a5019bcd58"}, + {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2cfcf84f1defed7e5798ef4f88aa25fcc52d279be731ce904789aa7ccfb7e8d2"}, + {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:a52a4704811e2623b0324a18d41ad4b9fabf43ce5ff99b14e40a520e2190c851"}, + {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c16304bba98f48a28ae10e32a8e75c349dd742c45156f297e16eeb1ba9287a1f"}, + {file = "lxml-6.0.0-cp311-cp311-win32.whl", hash = "sha256:f8d19565ae3eb956d84da3ef367aa7def14a2735d05bd275cd54c0301f0d0d6c"}, + {file = "lxml-6.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:b2d71cdefda9424adff9a3607ba5bbfc60ee972d73c21c7e3c19e71037574816"}, + {file = "lxml-6.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:8a2e76efbf8772add72d002d67a4c3d0958638696f541734304c7f28217a9cab"}, + {file = "lxml-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78718d8454a6e928470d511bf8ac93f469283a45c354995f7d19e77292f26108"}, + {file = "lxml-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:84ef591495ffd3f9dcabffd6391db7bb70d7230b5c35ef5148354a134f56f2be"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:2930aa001a3776c3e2601cb8e0a15d21b8270528d89cc308be4843ade546b9ab"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:219e0431ea8006e15005767f0351e3f7f9143e793e58519dc97fe9e07fae5563"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bd5913b4972681ffc9718bc2d4c53cde39ef81415e1671ff93e9aa30b46595e7"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:390240baeb9f415a82eefc2e13285016f9c8b5ad71ec80574ae8fa9605093cd7"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d6e200909a119626744dd81bae409fc44134389e03fbf1d68ed2a55a2fb10991"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca50bd612438258a91b5b3788c6621c1f05c8c478e7951899f492be42defc0da"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:c24b8efd9c0f62bad0439283c2c795ef916c5a6b75f03c17799775c7ae3c0c9e"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afd27d8629ae94c5d863e32ab0e1d5590371d296b87dae0a751fb22bf3685741"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:54c4855eabd9fc29707d30141be99e5cd1102e7d2258d2892314cf4c110726c3"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c907516d49f77f6cd8ead1322198bdfd902003c3c330c77a1c5f3cc32a0e4d16"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36531f81c8214e293097cd2b7873f178997dae33d3667caaae8bdfb9666b76c0"}, + {file = "lxml-6.0.0-cp312-cp312-win32.whl", hash = "sha256:690b20e3388a7ec98e899fd54c924e50ba6693874aa65ef9cb53de7f7de9d64a"}, + {file = "lxml-6.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:310b719b695b3dd442cdfbbe64936b2f2e231bb91d998e99e6f0daf991a3eba3"}, + {file = "lxml-6.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:8cb26f51c82d77483cdcd2b4a53cda55bbee29b3c2f3ddeb47182a2a9064e4eb"}, + {file = "lxml-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6da7cd4f405fd7db56e51e96bff0865b9853ae70df0e6720624049da76bde2da"}, + {file = "lxml-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b34339898bb556a2351a1830f88f751679f343eabf9cf05841c95b165152c9e7"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:51a5e4c61a4541bd1cd3ba74766d0c9b6c12d6a1a4964ef60026832aac8e79b3"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d18a25b19ca7307045581b18b3ec9ead2b1db5ccd8719c291f0cd0a5cec6cb81"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d4f0c66df4386b75d2ab1e20a489f30dc7fd9a06a896d64980541506086be1f1"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f4b481b6cc3a897adb4279216695150bbe7a44c03daba3c894f49d2037e0a24"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a78d6c9168f5bcb20971bf3329c2b83078611fbe1f807baadc64afc70523b3a"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae06fbab4f1bb7db4f7c8ca9897dc8db4447d1a2b9bee78474ad403437bcc29"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:1fa377b827ca2023244a06554c6e7dc6828a10aaf74ca41965c5d8a4925aebb4"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1676b56d48048a62ef77a250428d1f31f610763636e0784ba67a9740823988ca"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:0e32698462aacc5c1cf6bdfebc9c781821b7e74c79f13e5ffc8bfe27c42b1abf"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4d6036c3a296707357efb375cfc24bb64cd955b9ec731abf11ebb1e40063949f"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7488a43033c958637b1a08cddc9188eb06d3ad36582cebc7d4815980b47e27ef"}, + {file = "lxml-6.0.0-cp313-cp313-win32.whl", hash = "sha256:5fcd7d3b1d8ecb91445bd71b9c88bdbeae528fefee4f379895becfc72298d181"}, + {file = "lxml-6.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:2f34687222b78fff795feeb799a7d44eca2477c3d9d3a46ce17d51a4f383e32e"}, + {file = "lxml-6.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:21db1ec5525780fd07251636eb5f7acb84003e9382c72c18c542a87c416ade03"}, + {file = "lxml-6.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4eb114a0754fd00075c12648d991ec7a4357f9cb873042cc9a77bf3a7e30c9db"}, + {file = "lxml-6.0.0-cp38-cp38-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:7da298e1659e45d151b4028ad5c7974917e108afb48731f4ed785d02b6818994"}, + {file = "lxml-6.0.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bf61bc4345c1895221357af8f3e89f8c103d93156ef326532d35c707e2fb19d"}, + {file = "lxml-6.0.0-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63b634facdfbad421d4b61c90735688465d4ab3a8853ac22c76ccac2baf98d97"}, + {file = "lxml-6.0.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:e380e85b93f148ad28ac15f8117e2fd8e5437aa7732d65e260134f83ce67911b"}, + {file = "lxml-6.0.0-cp38-cp38-win32.whl", hash = "sha256:185efc2fed89cdd97552585c624d3c908f0464090f4b91f7d92f8ed2f3b18f54"}, + {file = "lxml-6.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:f97487996a39cb18278ca33f7be98198f278d0bc3c5d0fd4d7b3d63646ca3c8a"}, + {file = "lxml-6.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:85b14a4689d5cff426c12eefe750738648706ea2753b20c2f973b2a000d3d261"}, + {file = "lxml-6.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f64ccf593916e93b8d36ed55401bb7fe9c7d5de3180ce2e10b08f82a8f397316"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:b372d10d17a701b0945f67be58fae4664fd056b85e0ff0fbc1e6c951cdbc0512"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a674c0948789e9136d69065cc28009c1b1874c6ea340253db58be7622ce6398f"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:edf6e4c8fe14dfe316939711e3ece3f9a20760aabf686051b537a7562f4da91a"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:048a930eb4572829604982e39a0c7289ab5dc8abc7fc9f5aabd6fbc08c154e93"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0b5fa5eda84057a4f1bbb4bb77a8c28ff20ae7ce211588d698ae453e13c6281"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:c352fc8f36f7e9727db17adbf93f82499457b3d7e5511368569b4c5bd155a922"}, + {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8db5dc617cb937ae17ff3403c3a70a7de9df4852a046f93e71edaec678f721d0"}, + {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:2181e4b1d07dde53986023482673c0f1fba5178ef800f9ab95ad791e8bdded6a"}, + {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b3c98d5b24c6095e89e03d65d5c574705be3d49c0d8ca10c17a8a4b5201b72f5"}, + {file = "lxml-6.0.0-cp39-cp39-win32.whl", hash = "sha256:04d67ceee6db4bcb92987ccb16e53bef6b42ced872509f333c04fb58a3315256"}, + {file = "lxml-6.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:e0b1520ef900e9ef62e392dd3d7ae4f5fa224d1dd62897a792cf353eb20b6cae"}, + {file = "lxml-6.0.0-cp39-cp39-win_arm64.whl", hash = "sha256:e35e8aaaf3981489f42884b59726693de32dabfc438ac10ef4eb3409961fd402"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:dbdd7679a6f4f08152818043dbb39491d1af3332128b3752c3ec5cebc0011a72"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:40442e2a4456e9910875ac12951476d36c0870dcb38a68719f8c4686609897c4"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db0efd6bae1c4730b9c863fc4f5f3c0fa3e8f05cae2c44ae141cb9dfc7d091dc"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ab542c91f5a47aaa58abdd8ea84b498e8e49fe4b883d67800017757a3eb78e8"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:013090383863b72c62a702d07678b658fa2567aa58d373d963cca245b017e065"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c86df1c9af35d903d2b52d22ea3e66db8058d21dc0f59842ca5deb0595921141"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4337e4aec93b7c011f7ee2e357b0d30562edd1955620fdd4aeab6aacd90d43c5"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ae74f7c762270196d2dda56f8dd7309411f08a4084ff2dfcc0b095a218df2e06"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:059c4cbf3973a621b62ea3132934ae737da2c132a788e6cfb9b08d63a0ef73f9"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f090a9bc0ce8da51a5632092f98a7e7f84bca26f33d161a98b57f7fb0004ca"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9da022c14baeec36edfcc8daf0e281e2f55b950249a455776f0d1adeeada4734"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a55da151d0b0c6ab176b4e761670ac0e2667817a1e0dadd04a01d0561a219349"}, + {file = "lxml-6.0.0.tar.gz", hash = "sha256:032e65120339d44cdc3efc326c9f660f5f7205f3a535c1fdbf898b29ea01fb72"}, ] [package.extras] cssselect = ["cssselect (>=0.7)"] -html-clean = ["lxml-html-clean"] +html-clean = ["lxml_html_clean"] html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] -source = ["Cython (>=3.0.11)"] [[package]] name = "lxml-stubs" @@ -1089,6 +1112,7 @@ version = "0.5.1" description = "Type annotations for the lxml package" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "lxml-stubs-0.5.1.tar.gz", hash = "sha256:e0ec2aa1ce92d91278b719091ce4515c12adc1d564359dfaf81efa7d4feab79d"}, {file = "lxml_stubs-0.5.1-py3-none-any.whl", hash = "sha256:1f689e5dbc4b9247cb09ae820c7d34daeb1fdbd1db06123814b856dae7787272"}, @@ -1099,13 +1123,14 @@ test = ["coverage[toml] (>=7.2.5)", "mypy (>=1.2.0)", "pytest (>=7.3.0)", "pytes [[package]] name = "markdown-it-py" -version = "2.2.0" +version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["dev"] files = [ - {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, - {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, ] [package.dependencies] @@ -1118,66 +1143,78 @@ compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0 linkify = ["linkify-it-py (>=1,<3)"] plugins = ["mdit-py-plugins"] profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] [[package]] name = "markupsafe" -version = "2.1.2" +version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" +groups = ["main", "dev"] files = [ - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, - {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] [[package]] @@ -1186,6 +1223,7 @@ version = "1.3.0" description = "Common utilities for Synapse, Sydent and Sygnal" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "matrix_common-1.3.0-py3-none-any.whl", hash = "sha256:524e2785b9b03be4d15f3a8a6b857c5b6af68791ffb1b9918f0ad299abc4db20"}, {file = "matrix_common-1.3.0.tar.gz", hash = "sha256:62e121cccd9f243417b57ec37a76dc44aeb198a7a5c67afd6b8275992ff2abd1"}, @@ -1204,6 +1242,8 @@ version = "0.3.0" description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1223,6 +1263,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -1234,6 +1275,8 @@ version = "9.1.0" description = "More routines for operating on iterables, beyond itertools" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "more-itertools-9.1.0.tar.gz", hash = "sha256:cabaa341ad0389ea83c17a94566a53ae4c9d07349861ecb14dc6d0345cf9ac5d"}, {file = "more_itertools-9.1.0-py3-none-any.whl", hash = "sha256:d2bc7f02446e86a68911e58ded76d6561eea00cddfb2a91e7019bbb586c799f3"}, @@ -1241,120 +1284,146 @@ files = [ [[package]] name = "msgpack" -version = "1.1.0" +version = "1.1.1" description = "MessagePack serializer" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7ad442d527a7e358a469faf43fda45aaf4ac3249c8310a82f0ccff9164e5dccd"}, - {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:74bed8f63f8f14d75eec75cf3d04ad581da6b914001b474a5d3cd3372c8cc27d"}, - {file = "msgpack-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:914571a2a5b4e7606997e169f64ce53a8b1e06f2cf2c3a7273aa106236d43dd5"}, - {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c921af52214dcbb75e6bdf6a661b23c3e6417f00c603dd2070bccb5c3ef499f5"}, - {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8ce0b22b890be5d252de90d0e0d119f363012027cf256185fc3d474c44b1b9e"}, - {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73322a6cc57fcee3c0c57c4463d828e9428275fb85a27aa2aa1a92fdc42afd7b"}, - {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e1f3c3d21f7cf67bcf2da8e494d30a75e4cf60041d98b3f79875afb5b96f3a3f"}, - {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64fc9068d701233effd61b19efb1485587560b66fe57b3e50d29c5d78e7fef68"}, - {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:42f754515e0f683f9c79210a5d1cad631ec3d06cea5172214d2176a42e67e19b"}, - {file = "msgpack-1.1.0-cp310-cp310-win32.whl", hash = "sha256:3df7e6b05571b3814361e8464f9304c42d2196808e0119f55d0d3e62cd5ea044"}, - {file = "msgpack-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:685ec345eefc757a7c8af44a3032734a739f8c45d1b0ac45efc5d8977aa4720f"}, - {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d364a55082fb2a7416f6c63ae383fbd903adb5a6cf78c5b96cc6316dc1cedc7"}, - {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79ec007767b9b56860e0372085f8504db5d06bd6a327a335449508bbee9648fa"}, - {file = "msgpack-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6ad622bf7756d5a497d5b6836e7fc3752e2dd6f4c648e24b1803f6048596f701"}, - {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e59bca908d9ca0de3dc8684f21ebf9a690fe47b6be93236eb40b99af28b6ea6"}, - {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1da8f11a3dd397f0a32c76165cf0c4eb95b31013a94f6ecc0b280c05c91b59"}, - {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452aff037287acb1d70a804ffd022b21fa2bb7c46bee884dbc864cc9024128a0"}, - {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8da4bf6d54ceed70e8861f833f83ce0814a2b72102e890cbdfe4b34764cdd66e"}, - {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:41c991beebf175faf352fb940bf2af9ad1fb77fd25f38d9142053914947cdbf6"}, - {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a52a1f3a5af7ba1c9ace055b659189f6c669cf3657095b50f9602af3a3ba0fe5"}, - {file = "msgpack-1.1.0-cp311-cp311-win32.whl", hash = "sha256:58638690ebd0a06427c5fe1a227bb6b8b9fdc2bd07701bec13c2335c82131a88"}, - {file = "msgpack-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd2906780f25c8ed5d7b323379f6138524ba793428db5d0e9d226d3fa6aa1788"}, - {file = "msgpack-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d"}, - {file = "msgpack-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2"}, - {file = "msgpack-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420"}, - {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2"}, - {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39"}, - {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f"}, - {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247"}, - {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c"}, - {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b"}, - {file = "msgpack-1.1.0-cp312-cp312-win32.whl", hash = "sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b"}, - {file = "msgpack-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f"}, - {file = "msgpack-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:071603e2f0771c45ad9bc65719291c568d4edf120b44eb36324dcb02a13bfddf"}, - {file = "msgpack-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0f92a83b84e7c0749e3f12821949d79485971f087604178026085f60ce109330"}, - {file = "msgpack-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1964df7b81285d00a84da4e70cb1383f2e665e0f1f2a7027e683956d04b734"}, - {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59caf6a4ed0d164055ccff8fe31eddc0ebc07cf7326a2aaa0dbf7a4001cd823e"}, - {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0907e1a7119b337971a689153665764adc34e89175f9a34793307d9def08e6ca"}, - {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65553c9b6da8166e819a6aa90ad15288599b340f91d18f60b2061f402b9a4915"}, - {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a946a8992941fea80ed4beae6bff74ffd7ee129a90b4dd5cf9c476a30e9708d"}, - {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4b51405e36e075193bc051315dbf29168d6141ae2500ba8cd80a522964e31434"}, - {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4c01941fd2ff87c2a934ee6055bda4ed353a7846b8d4f341c428109e9fcde8c"}, - {file = "msgpack-1.1.0-cp313-cp313-win32.whl", hash = "sha256:7c9a35ce2c2573bada929e0b7b3576de647b0defbd25f5139dcdaba0ae35a4cc"}, - {file = "msgpack-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:bce7d9e614a04d0883af0b3d4d501171fbfca038f12c77fa838d9f198147a23f"}, - {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c40ffa9a15d74e05ba1fe2681ea33b9caffd886675412612d93ab17b58ea2fec"}, - {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1ba6136e650898082d9d5a5217d5906d1e138024f836ff48691784bbe1adf96"}, - {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0856a2b7e8dcb874be44fea031d22e5b3a19121be92a1e098f46068a11b0870"}, - {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:471e27a5787a2e3f974ba023f9e265a8c7cfd373632247deb225617e3100a3c7"}, - {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:646afc8102935a388ffc3914b336d22d1c2d6209c773f3eb5dd4d6d3b6f8c1cb"}, - {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:13599f8829cfbe0158f6456374e9eea9f44eee08076291771d8ae93eda56607f"}, - {file = "msgpack-1.1.0-cp38-cp38-win32.whl", hash = "sha256:8a84efb768fb968381e525eeeb3d92857e4985aacc39f3c47ffd00eb4509315b"}, - {file = "msgpack-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:879a7b7b0ad82481c52d3c7eb99bf6f0645dbdec5134a4bddbd16f3506947feb"}, - {file = "msgpack-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:53258eeb7a80fc46f62fd59c876957a2d0e15e6449a9e71842b6d24419d88ca1"}, - {file = "msgpack-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e7b853bbc44fb03fbdba34feb4bd414322180135e2cb5164f20ce1c9795ee48"}, - {file = "msgpack-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3e9b4936df53b970513eac1758f3882c88658a220b58dcc1e39606dccaaf01c"}, - {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46c34e99110762a76e3911fc923222472c9d681f1094096ac4102c18319e6468"}, - {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a706d1e74dd3dea05cb54580d9bd8b2880e9264856ce5068027eed09680aa74"}, - {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:534480ee5690ab3cbed89d4c8971a5c631b69a8c0883ecfea96c19118510c846"}, - {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8cf9e8c3a2153934a23ac160cc4cba0ec035f6867c8013cc6077a79823370346"}, - {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3180065ec2abbe13a4ad37688b61b99d7f9e012a535b930e0e683ad6bc30155b"}, - {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c5a91481a3cc573ac8c0d9aace09345d989dc4a0202b7fcb312c88c26d4e71a8"}, - {file = "msgpack-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f80bc7d47f76089633763f952e67f8214cb7b3ee6bfa489b3cb6a84cfac114cd"}, - {file = "msgpack-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:4d1b7ff2d6146e16e8bd665ac726a89c74163ef8cd39fa8c1087d4e52d3a2325"}, - {file = "msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e"}, + {file = "msgpack-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:353b6fc0c36fde68b661a12949d7d49f8f51ff5fa019c1e47c87c4ff34b080ed"}, + {file = "msgpack-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:79c408fcf76a958491b4e3b103d1c417044544b68e96d06432a189b43d1215c8"}, + {file = "msgpack-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78426096939c2c7482bf31ef15ca219a9e24460289c00dd0b94411040bb73ad2"}, + {file = "msgpack-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b17ba27727a36cb73aabacaa44b13090feb88a01d012c0f4be70c00f75048b4"}, + {file = "msgpack-1.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a17ac1ea6ec3c7687d70201cfda3b1e8061466f28f686c24f627cae4ea8efd0"}, + {file = "msgpack-1.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:88d1e966c9235c1d4e2afac21ca83933ba59537e2e2727a999bf3f515ca2af26"}, + {file = "msgpack-1.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f6d58656842e1b2ddbe07f43f56b10a60f2ba5826164910968f5933e5178af75"}, + {file = "msgpack-1.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96decdfc4adcbc087f5ea7ebdcfd3dee9a13358cae6e81d54be962efc38f6338"}, + {file = "msgpack-1.1.1-cp310-cp310-win32.whl", hash = "sha256:6640fd979ca9a212e4bcdf6eb74051ade2c690b862b679bfcb60ae46e6dc4bfd"}, + {file = "msgpack-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:8b65b53204fe1bd037c40c4148d00ef918eb2108d24c9aaa20bc31f9810ce0a8"}, + {file = "msgpack-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:71ef05c1726884e44f8b1d1773604ab5d4d17729d8491403a705e649116c9558"}, + {file = "msgpack-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:36043272c6aede309d29d56851f8841ba907a1a3d04435e43e8a19928e243c1d"}, + {file = "msgpack-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a32747b1b39c3ac27d0670122b57e6e57f28eefb725e0b625618d1b59bf9d1e0"}, + {file = "msgpack-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a8b10fdb84a43e50d38057b06901ec9da52baac6983d3f709d8507f3889d43f"}, + {file = "msgpack-1.1.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0c325c3f485dc54ec298d8b024e134acf07c10d494ffa24373bea729acf704"}, + {file = "msgpack-1.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:88daaf7d146e48ec71212ce21109b66e06a98e5e44dca47d853cbfe171d6c8d2"}, + {file = "msgpack-1.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8b55ea20dc59b181d3f47103f113e6f28a5e1c89fd5b67b9140edb442ab67f2"}, + {file = "msgpack-1.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4a28e8072ae9779f20427af07f53bbb8b4aa81151054e882aee333b158da8752"}, + {file = "msgpack-1.1.1-cp311-cp311-win32.whl", hash = "sha256:7da8831f9a0fdb526621ba09a281fadc58ea12701bc709e7b8cbc362feabc295"}, + {file = "msgpack-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fd1b58e1431008a57247d6e7cc4faa41c3607e8e7d4aaf81f7c29ea013cb458"}, + {file = "msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238"}, + {file = "msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157"}, + {file = "msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce"}, + {file = "msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a"}, + {file = "msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c"}, + {file = "msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b"}, + {file = "msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef"}, + {file = "msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a"}, + {file = "msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c"}, + {file = "msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4"}, + {file = "msgpack-1.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3765afa6bd4832fc11c3749be4ba4b69a0e8d7b728f78e68120a157a4c5d41f0"}, + {file = "msgpack-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8ddb2bcfd1a8b9e431c8d6f4f7db0773084e107730ecf3472f1dfe9ad583f3d9"}, + {file = "msgpack-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:196a736f0526a03653d829d7d4c5500a97eea3648aebfd4b6743875f28aa2af8"}, + {file = "msgpack-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d592d06e3cc2f537ceeeb23d38799c6ad83255289bb84c2e5792e5a8dea268a"}, + {file = "msgpack-1.1.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4df2311b0ce24f06ba253fda361f938dfecd7b961576f9be3f3fbd60e87130ac"}, + {file = "msgpack-1.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4141c5a32b5e37905b5940aacbc59739f036930367d7acce7a64e4dec1f5e0b"}, + {file = "msgpack-1.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b1ce7f41670c5a69e1389420436f41385b1aa2504c3b0c30620764b15dded2e7"}, + {file = "msgpack-1.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4147151acabb9caed4e474c3344181e91ff7a388b888f1e19ea04f7e73dc7ad5"}, + {file = "msgpack-1.1.1-cp313-cp313-win32.whl", hash = "sha256:500e85823a27d6d9bba1d057c871b4210c1dd6fb01fbb764e37e4e8847376323"}, + {file = "msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69"}, + {file = "msgpack-1.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bba1be28247e68994355e028dcd668316db30c1f758d3241a7b903ac78dcd285"}, + {file = "msgpack-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8f93dcddb243159c9e4109c9750ba5b335ab8d48d9522c5308cd05d7e3ce600"}, + {file = "msgpack-1.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fbbc0b906a24038c9958a1ba7ae0918ad35b06cb449d398b76a7d08470b0ed9"}, + {file = "msgpack-1.1.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:61e35a55a546a1690d9d09effaa436c25ae6130573b6ee9829c37ef0f18d5e78"}, + {file = "msgpack-1.1.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:1abfc6e949b352dadf4bce0eb78023212ec5ac42f6abfd469ce91d783c149c2a"}, + {file = "msgpack-1.1.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:996f2609ddf0142daba4cefd767d6db26958aac8439ee41db9cc0db9f4c4c3a6"}, + {file = "msgpack-1.1.1-cp38-cp38-win32.whl", hash = "sha256:4d3237b224b930d58e9d83c81c0dba7aacc20fcc2f89c1e5423aa0529a4cd142"}, + {file = "msgpack-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:da8f41e602574ece93dbbda1fab24650d6bf2a24089f9e9dbb4f5730ec1e58ad"}, + {file = "msgpack-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5be6b6bc52fad84d010cb45433720327ce886009d862f46b26d4d154001994b"}, + {file = "msgpack-1.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3a89cd8c087ea67e64844287ea52888239cbd2940884eafd2dcd25754fb72232"}, + {file = "msgpack-1.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d75f3807a9900a7d575d8d6674a3a47e9f227e8716256f35bc6f03fc597ffbf"}, + {file = "msgpack-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d182dac0221eb8faef2e6f44701812b467c02674a322c739355c39e94730cdbf"}, + {file = "msgpack-1.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b13fe0fb4aac1aa5320cd693b297fe6fdef0e7bea5518cbc2dd5299f873ae90"}, + {file = "msgpack-1.1.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:435807eeb1bc791ceb3247d13c79868deb22184e1fc4224808750f0d7d1affc1"}, + {file = "msgpack-1.1.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4835d17af722609a45e16037bb1d4d78b7bdf19d6c0128116d178956618c4e88"}, + {file = "msgpack-1.1.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8ef6e342c137888ebbfb233e02b8fbd689bb5b5fcc59b34711ac47ebd504478"}, + {file = "msgpack-1.1.1-cp39-cp39-win32.whl", hash = "sha256:61abccf9de335d9efd149e2fff97ed5974f2481b3353772e8e2dd3402ba2bd57"}, + {file = "msgpack-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:40eae974c873b2992fd36424a5d9407f93e97656d999f43fca9d29f820899084"}, + {file = "msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd"}, ] [[package]] -name = "mypy" -version = "1.11.2" -description = "Optional static typing for Python" +name = "multipart" +version = "1.2.1" +description = "Parser for multipart/form-data" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"}, - {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"}, - {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"}, - {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"}, - {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"}, - {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"}, - {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"}, - {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"}, - {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"}, - {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"}, - {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"}, - {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"}, - {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"}, - {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"}, - {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"}, - {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"}, - {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"}, - {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"}, - {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"}, - {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"}, - {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"}, - {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"}, - {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"}, - {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"}, - {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"}, - {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"}, - {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"}, + {file = "multipart-1.2.1-py3-none-any.whl", hash = "sha256:c03dc203bc2e67f6b46a599467ae0d87cf71d7530504b2c1ff4a9ea21d8b8c8c"}, + {file = "multipart-1.2.1.tar.gz", hash = "sha256:829b909b67bc1ad1c6d4488fcdc6391c2847842b08323addf5200db88dbe9480"}, +] + +[package.extras] +dev = ["build", "pytest", "pytest-cov", "twine"] +docs = ["sphinx (>=8,<9)", "sphinx-autobuild"] + +[[package]] +name = "mypy" +version = "1.17.1" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972"}, + {file = "mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7"}, + {file = "mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df"}, + {file = "mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390"}, + {file = "mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94"}, + {file = "mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b"}, + {file = "mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58"}, + {file = "mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5"}, + {file = "mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd"}, + {file = "mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b"}, + {file = "mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5"}, + {file = "mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b"}, + {file = "mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb"}, + {file = "mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403"}, + {file = "mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056"}, + {file = "mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341"}, + {file = "mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb"}, + {file = "mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19"}, + {file = "mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7"}, + {file = "mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81"}, + {file = "mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6"}, + {file = "mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849"}, + {file = "mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14"}, + {file = "mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a"}, + {file = "mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733"}, + {file = "mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd"}, + {file = "mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0"}, + {file = "mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a"}, + {file = "mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91"}, + {file = "mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed"}, + {file = "mypy-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5d1092694f166a7e56c805caaf794e0585cabdbf1df36911c414e4e9abb62ae9"}, + {file = "mypy-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79d44f9bfb004941ebb0abe8eff6504223a9c1ac51ef967d1263c6572bbebc99"}, + {file = "mypy-1.17.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b01586eed696ec905e61bd2568f48740f7ac4a45b3a468e6423a03d3788a51a8"}, + {file = "mypy-1.17.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43808d9476c36b927fbcd0b0255ce75efe1b68a080154a38ae68a7e62de8f0f8"}, + {file = "mypy-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:feb8cc32d319edd5859da2cc084493b3e2ce5e49a946377663cc90f6c15fb259"}, + {file = "mypy-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d7598cf74c3e16539d4e2f0b8d8c318e00041553d83d4861f87c7a72e95ac24d"}, + {file = "mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9"}, + {file = "mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01"}, ] [package.dependencies] -mypy-extensions = ">=1.0.0" +mypy_extensions = ">=1.0.0" +pathspec = ">=0.9.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=4.6.0" +typing_extensions = ">=4.6.0" [package.extras] dmypy = ["psutil (>=4.0)"] +faster-cache = ["orjson"] install-types = ["pip"] mypyc = ["setuptools (>=50)"] reports = ["lxml"] @@ -1365,6 +1434,7 @@ version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.5" +groups = ["dev"] files = [ {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, @@ -1372,17 +1442,18 @@ files = [ [[package]] name = "mypy-zope" -version = "1.0.8" +version = "1.0.13" description = "Plugin for mypy to support zope interfaces" optional = false python-versions = "*" +groups = ["dev"] files = [ - {file = "mypy_zope-1.0.8-py3-none-any.whl", hash = "sha256:8794a77dae0c7e2f28b8ac48569091310b3ee45bb9d6cd4797dcb837c40f9976"}, - {file = "mypy_zope-1.0.8.tar.gz", hash = "sha256:854303a95aefc4289e8a0796808e002c2c7ecde0a10a8f7b8f48092f94ef9b9f"}, + {file = "mypy_zope-1.0.13-py3-none-any.whl", hash = "sha256:13740c4cbc910cca2c143c6709e1c483c991abeeeb7b629ad6f73d8ac1edad15"}, + {file = "mypy_zope-1.0.13.tar.gz", hash = "sha256:63fb4d035ea874baf280dc69e714dcde4bd2a4a4837a0fd8d90ce91bea510f99"}, ] [package.dependencies] -mypy = ">=1.0.0,<1.13.0" +mypy = ">=1.0.0,<1.18.0" "zope.interface" = "*" "zope.schema" = "*" @@ -1395,6 +1466,7 @@ version = "1.3.0" description = "A network address manipulation library for Python" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "netaddr-1.3.0-py3-none-any.whl", hash = "sha256:c2c6a8ebe5554ce33b7d5b3a306b71bbb373e000bbbf2350dd5213cc56e3dbbe"}, {file = "netaddr-1.3.0.tar.gz", hash = "sha256:5c3c3d9895b551b763779ba7db7a03487dc1f8e3b385af819af341ae9ef6e48a"}, @@ -1409,6 +1481,8 @@ version = "2.4.0" description = "OpenTracing API for Python. See documentation at http://opentracing.io" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1418,13 +1492,14 @@ tests = ["Sphinx", "doubles", "flake8", "flake8-quotes", "gevent", "mock", "pyte [[package]] name = "packaging" -version = "24.2" +version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ - {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, - {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] [[package]] @@ -1433,6 +1508,7 @@ version = "0.9.0" description = "Parameterized testing with any Python test framework" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b"}, {file = "parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1"}, @@ -1441,137 +1517,165 @@ files = [ [package.extras] dev = ["jinja2"] +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + [[package]] name = "phonenumbers" -version = "8.13.50" +version = "9.0.13" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" +groups = ["main"] files = [ - {file = "phonenumbers-8.13.50-py2.py3-none-any.whl", hash = "sha256:bb95dbc0d9979c51f7ad94bcd780784938958861fbb4b75a2fe39ccd3d58954a"}, - {file = "phonenumbers-8.13.50.tar.gz", hash = "sha256:e05ac6fb7b98c6d719a87ea895b9fc153673b4a51f455ec9afaf557ef4629da6"}, + {file = "phonenumbers-9.0.13-py2.py3-none-any.whl", hash = "sha256:b97661e177773e7509c6d503e0f537cd0af22aa3746231654590876eb9430915"}, + {file = "phonenumbers-9.0.13.tar.gz", hash = "sha256:eca06e01382412c45316868f86a44bb217c02f9ee7196589041556a2f54a7639"}, ] [[package]] name = "pillow" -version = "10.4.0" +version = "11.3.0" description = "Python Imaging Library (Fork)" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "pillow-10.4.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e"}, - {file = "pillow-10.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d"}, - {file = "pillow-10.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7928ecbf1ece13956b95d9cbcfc77137652b02763ba384d9ab508099a2eca856"}, - {file = "pillow-10.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d49b85c4348ea0b31ea63bc75a9f3857869174e2bf17e7aba02945cd218e6f"}, - {file = "pillow-10.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6c762a5b0997f5659a5ef2266abc1d8851ad7749ad9a6a5506eb23d314e4f46b"}, - {file = "pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a985e028fc183bf12a77a8bbf36318db4238a3ded7fa9df1b9a133f1cb79f8fc"}, - {file = "pillow-10.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:812f7342b0eee081eaec84d91423d1b4650bb9828eb53d8511bcef8ce5aecf1e"}, - {file = "pillow-10.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ac1452d2fbe4978c2eec89fb5a23b8387aba707ac72810d9490118817d9c0b46"}, - {file = "pillow-10.4.0-cp310-cp310-win32.whl", hash = "sha256:bcd5e41a859bf2e84fdc42f4edb7d9aba0a13d29a2abadccafad99de3feff984"}, - {file = "pillow-10.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:ecd85a8d3e79cd7158dec1c9e5808e821feea088e2f69a974db5edf84dc53141"}, - {file = "pillow-10.4.0-cp310-cp310-win_arm64.whl", hash = "sha256:ff337c552345e95702c5fde3158acb0625111017d0e5f24bf3acdb9cc16b90d1"}, - {file = "pillow-10.4.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:0a9ec697746f268507404647e531e92889890a087e03681a3606d9b920fbee3c"}, - {file = "pillow-10.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe91cb65544a1321e631e696759491ae04a2ea11d36715eca01ce07284738be"}, - {file = "pillow-10.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dc6761a6efc781e6a1544206f22c80c3af4c8cf461206d46a1e6006e4429ff3"}, - {file = "pillow-10.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e84b6cc6a4a3d76c153a6b19270b3526a5a8ed6b09501d3af891daa2a9de7d6"}, - {file = "pillow-10.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:bbc527b519bd3aa9d7f429d152fea69f9ad37c95f0b02aebddff592688998abe"}, - {file = "pillow-10.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:76a911dfe51a36041f2e756b00f96ed84677cdeb75d25c767f296c1c1eda1319"}, - {file = "pillow-10.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59291fb29317122398786c2d44427bbd1a6d7ff54017075b22be9d21aa59bd8d"}, - {file = "pillow-10.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:416d3a5d0e8cfe4f27f574362435bc9bae57f679a7158e0096ad2beb427b8696"}, - {file = "pillow-10.4.0-cp311-cp311-win32.whl", hash = "sha256:7086cc1d5eebb91ad24ded9f58bec6c688e9f0ed7eb3dbbf1e4800280a896496"}, - {file = "pillow-10.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cbed61494057c0f83b83eb3a310f0bf774b09513307c434d4366ed64f4128a91"}, - {file = "pillow-10.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:f5f0c3e969c8f12dd2bb7e0b15d5c468b51e5017e01e2e867335c81903046a22"}, - {file = "pillow-10.4.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:673655af3eadf4df6b5457033f086e90299fdd7a47983a13827acf7459c15d94"}, - {file = "pillow-10.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:866b6942a92f56300012f5fbac71f2d610312ee65e22f1aa2609e491284e5597"}, - {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29dbdc4207642ea6aad70fbde1a9338753d33fb23ed6956e706936706f52dd80"}, - {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf2342ac639c4cf38799a44950bbc2dfcb685f052b9e262f446482afaf4bffca"}, - {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f5b92f4d70791b4a67157321c4e8225d60b119c5cc9aee8ecf153aace4aad4ef"}, - {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:86dcb5a1eb778d8b25659d5e4341269e8590ad6b4e8b44d9f4b07f8d136c414a"}, - {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:780c072c2e11c9b2c7ca37f9a2ee8ba66f44367ac3e5c7832afcfe5104fd6d1b"}, - {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37fb69d905be665f68f28a8bba3c6d3223c8efe1edf14cc4cfa06c241f8c81d9"}, - {file = "pillow-10.4.0-cp312-cp312-win32.whl", hash = "sha256:7dfecdbad5c301d7b5bde160150b4db4c659cee2b69589705b6f8a0c509d9f42"}, - {file = "pillow-10.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1d846aea995ad352d4bdcc847535bd56e0fd88d36829d2c90be880ef1ee4668a"}, - {file = "pillow-10.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:e553cad5179a66ba15bb18b353a19020e73a7921296a7979c4a2b7f6a5cd57f9"}, - {file = "pillow-10.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8bc1a764ed8c957a2e9cacf97c8b2b053b70307cf2996aafd70e91a082e70df3"}, - {file = "pillow-10.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6209bb41dc692ddfee4942517c19ee81b86c864b626dbfca272ec0f7cff5d9fb"}, - {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bee197b30783295d2eb680b311af15a20a8b24024a19c3a26431ff83eb8d1f70"}, - {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ef61f5dd14c300786318482456481463b9d6b91ebe5ef12f405afbba77ed0be"}, - {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:297e388da6e248c98bc4a02e018966af0c5f92dfacf5a5ca22fa01cb3179bca0"}, - {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e4db64794ccdf6cb83a59d73405f63adbe2a1887012e308828596100a0b2f6cc"}, - {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd2880a07482090a3bcb01f4265f1936a903d70bc740bfcb1fd4e8a2ffe5cf5a"}, - {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b35b21b819ac1dbd1233317adeecd63495f6babf21b7b2512d244ff6c6ce309"}, - {file = "pillow-10.4.0-cp313-cp313-win32.whl", hash = "sha256:551d3fd6e9dc15e4c1eb6fc4ba2b39c0c7933fa113b220057a34f4bb3268a060"}, - {file = "pillow-10.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:030abdbe43ee02e0de642aee345efa443740aa4d828bfe8e2eb11922ea6a21ea"}, - {file = "pillow-10.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:5b001114dd152cfd6b23befeb28d7aee43553e2402c9f159807bf55f33af8a8d"}, - {file = "pillow-10.4.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8d4d5063501b6dd4024b8ac2f04962d661222d120381272deea52e3fc52d3736"}, - {file = "pillow-10.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c1ee6f42250df403c5f103cbd2768a28fe1a0ea1f0f03fe151c8741e1469c8b"}, - {file = "pillow-10.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15e02e9bb4c21e39876698abf233c8c579127986f8207200bc8a8f6bb27acf2"}, - {file = "pillow-10.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a8d4bade9952ea9a77d0c3e49cbd8b2890a399422258a77f357b9cc9be8d680"}, - {file = "pillow-10.4.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:43efea75eb06b95d1631cb784aa40156177bf9dd5b4b03ff38979e048258bc6b"}, - {file = "pillow-10.4.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:950be4d8ba92aca4b2bb0741285a46bfae3ca699ef913ec8416c1b78eadd64cd"}, - {file = "pillow-10.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d7480af14364494365e89d6fddc510a13e5a2c3584cb19ef65415ca57252fb84"}, - {file = "pillow-10.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:73664fe514b34c8f02452ffb73b7a92c6774e39a647087f83d67f010eb9a0cf0"}, - {file = "pillow-10.4.0-cp38-cp38-win32.whl", hash = "sha256:e88d5e6ad0d026fba7bdab8c3f225a69f063f116462c49892b0149e21b6c0a0e"}, - {file = "pillow-10.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:5161eef006d335e46895297f642341111945e2c1c899eb406882a6c61a4357ab"}, - {file = "pillow-10.4.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:0ae24a547e8b711ccaaf99c9ae3cd975470e1a30caa80a6aaee9a2f19c05701d"}, - {file = "pillow-10.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:298478fe4f77a4408895605f3482b6cc6222c018b2ce565c2b6b9c354ac3229b"}, - {file = "pillow-10.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:134ace6dc392116566980ee7436477d844520a26a4b1bd4053f6f47d096997fd"}, - {file = "pillow-10.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:930044bb7679ab003b14023138b50181899da3f25de50e9dbee23b61b4de2126"}, - {file = "pillow-10.4.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c76e5786951e72ed3686e122d14c5d7012f16c8303a674d18cdcd6d89557fc5b"}, - {file = "pillow-10.4.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b2724fdb354a868ddf9a880cb84d102da914e99119211ef7ecbdc613b8c96b3c"}, - {file = "pillow-10.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dbc6ae66518ab3c5847659e9988c3b60dc94ffb48ef9168656e0019a93dbf8a1"}, - {file = "pillow-10.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:06b2f7898047ae93fad74467ec3d28fe84f7831370e3c258afa533f81ef7f3df"}, - {file = "pillow-10.4.0-cp39-cp39-win32.whl", hash = "sha256:7970285ab628a3779aecc35823296a7869f889b8329c16ad5a71e4901a3dc4ef"}, - {file = "pillow-10.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:961a7293b2457b405967af9c77dcaa43cc1a8cd50d23c532e62d48ab6cdd56f5"}, - {file = "pillow-10.4.0-cp39-cp39-win_arm64.whl", hash = "sha256:32cda9e3d601a52baccb2856b8ea1fc213c90b340c542dcef77140dfa3278a9e"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5b4815f2e65b30f5fbae9dfffa8636d992d49705723fe86a3661806e069352d4"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8f0aef4ef59694b12cadee839e2ba6afeab89c0f39a3adc02ed51d109117b8da"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f4727572e2918acaa9077c919cbbeb73bd2b3ebcfe033b72f858fc9fbef0026"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff25afb18123cea58a591ea0244b92eb1e61a1fd497bf6d6384f09bc3262ec3e"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dc3e2db6ba09ffd7d02ae9141cfa0ae23393ee7687248d46a7507b75d610f4f5"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:02a2be69f9c9b8c1e97cf2713e789d4e398c751ecfd9967c18d0ce304efbf885"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0755ffd4a0c6f267cccbae2e9903d95477ca2f77c4fcf3a3a09570001856c8a5"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:a02364621fe369e06200d4a16558e056fe2805d3468350df3aef21e00d26214b"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1b5dea9831a90e9d0721ec417a80d4cbd7022093ac38a568db2dd78363b00908"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b885f89040bb8c4a1573566bbb2f44f5c505ef6e74cec7ab9068c900047f04b"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87dd88ded2e6d74d31e1e0a99a726a6765cda32d00ba72dc37f0651f306daaa8"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:2db98790afc70118bd0255c2eeb465e9767ecf1f3c25f9a1abb8ffc8cfd1fe0a"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f7baece4ce06bade126fb84b8af1c33439a76d8a6fd818970215e0560ca28c27"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cfdd747216947628af7b259d274771d84db2268ca062dd5faf373639d00113a3"}, - {file = "pillow-10.4.0.tar.gz", hash = "sha256:166c1cd4d24309b30d61f79f4a9114b7b2313d7450912277855ff5dfd7cd4a06"}, + {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, + {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae"}, + {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9"}, + {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:040a5b691b0713e1f6cbe222e0f4f74cd233421e105850ae3b3c0ceda520f42e"}, + {file = "pillow-11.3.0-cp310-cp310-win32.whl", hash = "sha256:89bd777bc6624fe4115e9fac3352c79ed60f3bb18651420635f26e643e3dd1f6"}, + {file = "pillow-11.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:19d2ff547c75b8e3ff46f4d9ef969a06c30ab2d4263a9e287733aa8b2429ce8f"}, + {file = "pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f"}, + {file = "pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722"}, + {file = "pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f"}, + {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e"}, + {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:932c754c2d51ad2b2271fd01c3d121daaa35e27efae2a616f77bf164bc0b3e94"}, + {file = "pillow-11.3.0-cp311-cp311-win32.whl", hash = "sha256:b4b8f3efc8d530a1544e5962bd6b403d5f7fe8b9e08227c6b255f98ad82b4ba0"}, + {file = "pillow-11.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:1a992e86b0dd7aeb1f053cd506508c0999d710a8f07b4c791c63843fc6a807ac"}, + {file = "pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd"}, + {file = "pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4"}, + {file = "pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024"}, + {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809"}, + {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6be31e3fc9a621e071bc17bb7de63b85cbe0bfae91bb0363c893cbe67247780d"}, + {file = "pillow-11.3.0-cp312-cp312-win32.whl", hash = "sha256:7b161756381f0918e05e7cb8a371fff367e807770f8fe92ecb20d905d0e1c149"}, + {file = "pillow-11.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6444696fce635783440b7f7a9fc24b3ad10a9ea3f0ab66c5905be1c19ccf17d"}, + {file = "pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1c627742b539bba4309df89171356fcb3cc5a9178355b2727d1b74a6cf155fbd"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:30b7c02f3899d10f13d7a48163c8969e4e653f8b43416d23d13d1bbfdc93b9f8"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f"}, + {file = "pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c"}, + {file = "pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8"}, + {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2"}, + {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:45dfc51ac5975b938e9809451c51734124e73b04d0f0ac621649821a63852e7b"}, + {file = "pillow-11.3.0-cp313-cp313-win32.whl", hash = "sha256:a4d336baed65d50d37b88ca5b60c0fa9d81e3a87d4a7930d3880d1624d5b31f3"}, + {file = "pillow-11.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bce5c4fd0921f99d2e858dc4d4d64193407e1b99478bc5cacecba2311abde51"}, + {file = "pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580"}, + {file = "pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e"}, + {file = "pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59"}, + {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe"}, + {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83e1b0161c9d148125083a35c1c5a89db5b7054834fd4387499e06552035236c"}, + {file = "pillow-11.3.0-cp313-cp313t-win32.whl", hash = "sha256:2a3117c06b8fb646639dce83694f2f9eac405472713fcb1ae887469c0d4f6788"}, + {file = "pillow-11.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:857844335c95bea93fb39e0fa2726b4d9d758850b34075a7e3ff4f4fa3aa3b31"}, + {file = "pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e"}, + {file = "pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12"}, + {file = "pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77"}, + {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874"}, + {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:98a9afa7b9007c67ed84c57c9e0ad86a6000da96eaa638e4f8abe5b65ff83f0a"}, + {file = "pillow-11.3.0-cp314-cp314-win32.whl", hash = "sha256:02a723e6bf909e7cea0dac1b0e0310be9d7650cd66222a5f1c571455c0a45214"}, + {file = "pillow-11.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a418486160228f64dd9e9efcd132679b7a02a5f22c982c78b6fc7dab3fefb635"}, + {file = "pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6"}, + {file = "pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae"}, + {file = "pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477"}, + {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50"}, + {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a1bc6ba083b145187f648b667e05a2534ecc4b9f2784c2cbe3089e44868f2b9b"}, + {file = "pillow-11.3.0-cp314-cp314t-win32.whl", hash = "sha256:118ca10c0d60b06d006be10a501fd6bbdfef559251ed31b794668ed569c87e12"}, + {file = "pillow-11.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8924748b688aa210d79883357d102cd64690e56b923a186f35a82cbc10f997db"}, + {file = "pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa"}, + {file = "pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f"}, + {file = "pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:23cff760a9049c502721bdb743a7cb3e03365fafcdfc2ef9784610714166e5a4"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6359a3bc43f57d5b375d1ad54a0074318a0844d11b76abccf478c37c986d3cfc"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a"}, + {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978"}, + {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:97afb3a00b65cc0804d1c7abddbf090a81eaac02768af58cbdcaaa0a931e0b6d"}, + {file = "pillow-11.3.0-cp39-cp39-win32.whl", hash = "sha256:ea944117a7974ae78059fcc1800e5d3295172bb97035c0c1d9345fca1419da71"}, + {file = "pillow-11.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:e5c5858ad8ec655450a7c7df532e9842cf8df7cc349df7225c60d5d348c8aada"}, + {file = "pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8"}, + {file = "pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523"}, ] [package.extras] -docs = ["furo", "olefile", "sphinx (>=7.3)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] fpx = ["olefile"] mic = ["olefile"] -tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] -typing = ["typing-extensions"] +test-arrow = ["pyarrow"] +tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "trove-classifiers (>=2024.10.12)"] +typing = ["typing-extensions ; python_version < \"3.10\""] xmp = ["defusedxml"] -[[package]] -name = "pkginfo" -version = "1.9.6" -description = "Query metadata from sdists / bdists / installed packages." -optional = false -python-versions = ">=3.6" -files = [ - {file = "pkginfo-1.9.6-py3-none-any.whl", hash = "sha256:4b7a555a6d5a22169fcc9cf7bfd78d296b0361adad412a346c1226849af5e546"}, - {file = "pkginfo-1.9.6.tar.gz", hash = "sha256:8fd5896e8718a4372f0ea9cc9d96f6417c9b986e23a4d116dda26b62cc29d046"}, -] - -[package.extras] -testing = ["pytest", "pytest-cov"] - [[package]] name = "prometheus-client" -version = "0.21.0" +version = "0.22.1" description = "Python client for the Prometheus monitoring system." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "prometheus_client-0.21.0-py3-none-any.whl", hash = "sha256:4fa6b4dd0ac16d58bb587c04b1caae65b8c5043e85f778f42f5f632f6af2e166"}, - {file = "prometheus_client-0.21.0.tar.gz", hash = "sha256:96c83c606b71ff2b0a433c98889d275f51ffec6c5e267de37c7a2b5c9aa9233e"}, + {file = "prometheus_client-0.22.1-py3-none-any.whl", hash = "sha256:cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094"}, + {file = "prometheus_client-0.22.1.tar.gz", hash = "sha256:190f1331e783cf21eb60bca559354e0a4d4378facecf78f5428c39b675d20d28"}, ] [package.extras] @@ -1583,6 +1687,8 @@ version = "2.9.10" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1590,6 +1696,7 @@ files = [ {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, + {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, @@ -1601,6 +1708,8 @@ version = "2.9.0" description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=master" optional = true python-versions = "*" +groups = ["main"] +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1615,6 +1724,8 @@ version = "1.1" description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" +groups = ["main"] +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1628,6 +1739,7 @@ version = "0.6.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, @@ -1635,17 +1747,18 @@ files = [ [[package]] name = "pyasn1-modules" -version = "0.4.1" +version = "0.4.2" description = "A collection of ASN.1-based protocols modules" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"}, - {file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"}, + {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, + {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, ] [package.dependencies] -pyasn1 = ">=0.4.6,<0.7.0" +pyasn1 = ">=0.6.1,<0.7.0" [[package]] name = "pycparser" @@ -1653,6 +1766,7 @@ version = "2.21" description = "C parser in Python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main", "dev"] files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, @@ -1660,123 +1774,133 @@ files = [ [[package]] name = "pydantic" -version = "2.9.2" +version = "2.11.9" description = "Data validation using Python type hints" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main", "dev"] files = [ - {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, - {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, + {file = "pydantic-2.11.9-py3-none-any.whl", hash = "sha256:c42dd626f5cfc1c6950ce6205ea58c93efa406da65f479dcb4029d5934857da2"}, + {file = "pydantic-2.11.9.tar.gz", hash = "sha256:6b8ffda597a14812a7975c90b82a8a2e777d9257aba3453f973acd3c032a18e2"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.23.4" -typing-extensions = [ - {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, - {version = ">=4.6.1", markers = "python_version < \"3.13\""}, -] +pydantic-core = "2.33.2" +typing-extensions = ">=4.12.2" +typing-inspection = ">=0.4.0" [package.extras] email = ["email-validator (>=2.0.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] [[package]] name = "pydantic-core" -version = "2.23.4" +version = "2.33.2" description = "Core functionality for Pydantic validation and serialization" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main", "dev"] files = [ - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, - {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, - {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, - {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, - {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, - {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, - {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, - {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, - {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, - {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, - {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, - {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, - {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, - {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, + {file = "pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8"}, + {file = "pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a"}, + {file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac"}, + {file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a"}, + {file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b"}, + {file = "pydantic_core-2.33.2-cp310-cp310-win32.whl", hash = "sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22"}, + {file = "pydantic_core-2.33.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640"}, + {file = "pydantic_core-2.33.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7"}, + {file = "pydantic_core-2.33.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e"}, + {file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d"}, + {file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30"}, + {file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf"}, + {file = "pydantic_core-2.33.2-cp311-cp311-win32.whl", hash = "sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51"}, + {file = "pydantic_core-2.33.2-cp311-cp311-win_amd64.whl", hash = "sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab"}, + {file = "pydantic_core-2.33.2-cp311-cp311-win_arm64.whl", hash = "sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65"}, + {file = "pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc"}, + {file = "pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b"}, + {file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1"}, + {file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6"}, + {file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea"}, + {file = "pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290"}, + {file = "pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2"}, + {file = "pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab"}, + {file = "pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f"}, + {file = "pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56"}, + {file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5"}, + {file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e"}, + {file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162"}, + {file = "pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849"}, + {file = "pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9"}, + {file = "pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9"}, + {file = "pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac"}, + {file = "pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5"}, + {file = "pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9"}, + {file = "pydantic_core-2.33.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d"}, + {file = "pydantic_core-2.33.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a"}, + {file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782"}, + {file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9"}, + {file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e"}, + {file = "pydantic_core-2.33.2-cp39-cp39-win32.whl", hash = "sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9"}, + {file = "pydantic_core-2.33.2-cp39-cp39-win_amd64.whl", hash = "sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27"}, + {file = "pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc"}, ] [package.dependencies] @@ -1784,21 +1908,21 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pygithub" -version = "2.5.0" +version = "2.7.0" description = "Use the full Github API v3" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ - {file = "PyGithub-2.5.0-py3-none-any.whl", hash = "sha256:b0b635999a658ab8e08720bdd3318893ff20e2275f6446fcf35bf3f44f2c0fd2"}, - {file = "pygithub-2.5.0.tar.gz", hash = "sha256:e1613ac508a9be710920d26eb18b1905ebd9926aa49398e88151c1b526aad3cf"}, + {file = "pygithub-2.7.0-py3-none-any.whl", hash = "sha256:40ecbfe26dc55cc34ab4b0ffa1d455e6f816ef9a2bc8d6f5ad18ce572f163700"}, + {file = "pygithub-2.7.0.tar.gz", hash = "sha256:7cd6eafabb09b5369afba3586d86b1f1ad6f1326d2ff01bc47bb26615dce4cbb"}, ] [package.dependencies] -Deprecated = "*" pyjwt = {version = ">=2.4.0", extras = ["crypto"]} pynacl = ">=1.4.0" requests = ">=2.14.0" -typing-extensions = ">=4.0.0" +typing-extensions = ">=4.5.0" urllib3 = ">=1.26.0" [[package]] @@ -1807,23 +1931,14 @@ version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, ] [package.extras] -plugins = ["importlib-metadata"] - -[[package]] -name = "pyicu" -version = "2.13.1" -description = "Python extension wrapping the ICU C++ API" -optional = true -python-versions = "*" -files = [ - {file = "PyICU-2.13.1.tar.gz", hash = "sha256:d4919085eaa07da12bade8ee721e7bbf7ade0151ca0f82946a26c8f4b98cdceb"}, -] +plugins = ["importlib-metadata ; python_version < \"3.8\""] [[package]] name = "pyjwt" @@ -1831,6 +1946,7 @@ version = "2.6.0" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, @@ -1851,6 +1967,7 @@ version = "0.13.0" description = "Macaroon library for Python" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "pymacaroons-0.13.0-py2.py3-none-any.whl", hash = "sha256:3e14dff6a262fdbf1a15e769ce635a8aea72e6f8f91e408f9a97166c53b91907"}, {file = "pymacaroons-0.13.0.tar.gz", hash = "sha256:1e6bba42a5f66c245adf38a5a4006a99dcc06a0703786ea636098667d42903b8"}, @@ -1866,6 +1983,8 @@ version = "1.0.1" description = "A development tool to measure, monitor and analyze the memory behavior of Python objects." optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -1877,6 +1996,7 @@ version = "1.5.0" description = "Python binding to the Networking and Cryptography (NaCl) library" optional = false python-versions = ">=3.6" +groups = ["main", "dev"] files = [ {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, @@ -1899,31 +2019,35 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pyopenssl" -version = "24.2.1" +version = "25.1.0" description = "Python wrapper module around the OpenSSL library" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "pyOpenSSL-24.2.1-py3-none-any.whl", hash = "sha256:967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d"}, - {file = "pyopenssl-24.2.1.tar.gz", hash = "sha256:4247f0dbe3748d560dcbb2ff3ea01af0f9a1a001ef5f7c4c647956ed8cbf0e95"}, + {file = "pyopenssl-25.1.0-py3-none-any.whl", hash = "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab"}, + {file = "pyopenssl-25.1.0.tar.gz", hash = "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b"}, ] [package.dependencies] -cryptography = ">=41.0.5,<44" +cryptography = ">=41.0.5,<46" +typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and python_version >= \"3.8\""} [package.extras] -docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] [[package]] name = "pysaml2" -version = "7.3.1" +version = "7.5.0" description = "Python implementation of SAML Version 2 Standard" optional = true -python-versions = ">=3.6.2,<4.0.0" +python-versions = ">=3.9,<4.0" +groups = ["main"] +markers = "extra == \"all\" or extra == \"saml2\"" files = [ - {file = "pysaml2-7.3.1-py3-none-any.whl", hash = "sha256:2cc66e7a371d3f5ff9601f0ed93b5276cca816fce82bb38447d5a0651f2f5193"}, - {file = "pysaml2-7.3.1.tar.gz", hash = "sha256:eab22d187c6dd7707c58b5bb1688f9b8e816427667fc99d77f54399e15cd0a0a"}, + {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, + {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, ] [package.dependencies] @@ -1933,7 +2057,7 @@ pyopenssl = "*" python-dateutil = "*" pytz = "*" requests = ">=2,<3" -xmlschema = ">=1.2.1" +xmlschema = ">=2,<3" [package.extras] s2repoze = ["paste", "repoze.who", "zope.interface"] @@ -1944,6 +2068,8 @@ version = "2.8.2" description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -1954,13 +2080,14 @@ six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.16" +version = "0.0.20" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "python_multipart-0.0.16-py3-none-any.whl", hash = "sha256:c2759b7b976ef3937214dfb592446b59dfaa5f04682a076f78b117c94776d87a"}, - {file = "python_multipart-0.0.16.tar.gz", hash = "sha256:8dee37b88dab9b59922ca173c35acb627cc12ec74019f5cd4578369c6df36554"}, + {file = "python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104"}, + {file = "python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13"}, ] [[package]] @@ -1969,6 +2096,8 @@ version = "2022.7.1" description = "World timezone definitions, modern and historical" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -1980,6 +2109,8 @@ version = "0.2.0" description = "" optional = false python-versions = "*" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and sys_platform == \"win32\"" files = [ {file = "pywin32-ctypes-0.2.0.tar.gz", hash = "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942"}, {file = "pywin32_ctypes-0.2.0-py2.py3-none-any.whl", hash = "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"}, @@ -1991,6 +2122,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -2053,6 +2185,7 @@ version = "37.3" description = "readme_renderer is a library for rendering \"readme\" descriptions for Warehouse" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "readme_renderer-37.3-py3-none-any.whl", hash = "sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343"}, {file = "readme_renderer-37.3.tar.gz", hash = "sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273"}, @@ -2072,6 +2205,7 @@ version = "0.29.1" description = "JSON Referencing + Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "referencing-0.29.1-py3-none-any.whl", hash = "sha256:d3c8f323ee1480095da44d55917cfb8278d73d6b4d5f677e3e40eb21314ac67f"}, {file = "referencing-0.29.1.tar.gz", hash = "sha256:90cb53782d550ba28d2166ef3f55731f38397def8832baac5d45235f1995e35e"}, @@ -2083,18 +2217,19 @@ rpds-py = ">=0.7.0" [[package]] name = "requests" -version = "2.32.2" +version = "2.32.4" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ - {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, - {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, + {file = "requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c"}, + {file = "requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422"}, ] [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" +charset_normalizer = ">=2,<4" idna = ">=2.5,<4" urllib3 = ">=1.21.1,<3" @@ -2108,6 +2243,7 @@ version = "1.0.0" description = "A utility belt for advanced users of python-requests" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["dev"] files = [ {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, @@ -2122,6 +2258,7 @@ version = "2.0.0" description = "Validating URI References per RFC 3986" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"}, {file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"}, @@ -2132,18 +2269,20 @@ idna2008 = ["idna"] [[package]] name = "rich" -version = "13.3.2" +version = "14.0.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" +groups = ["dev"] files = [ - {file = "rich-13.3.2-py3-none-any.whl", hash = "sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f"}, - {file = "rich-13.3.2.tar.gz", hash = "sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001"}, + {file = "rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0"}, + {file = "rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725"}, ] [package.dependencies] -markdown-it-py = ">=2.2.0,<3.0.0" +markdown-it-py = ">=2.2.0" pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.11\""} [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] @@ -2154,6 +2293,7 @@ version = "0.8.10" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "rpds_py-0.8.10-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:93d06cccae15b3836247319eee7b6f1fdcd6c10dabb4e6d350d27bd0bdca2711"}, {file = "rpds_py-0.8.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3816a890a6a9e9f1de250afa12ca71c9a7a62f2b715a29af6aaee3aea112c181"}, @@ -2256,29 +2396,31 @@ files = [ [[package]] name = "ruff" -version = "0.7.3" +version = "0.12.10" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ - {file = "ruff-0.7.3-py3-none-linux_armv6l.whl", hash = "sha256:34f2339dc22687ec7e7002792d1f50712bf84a13d5152e75712ac08be565d344"}, - {file = "ruff-0.7.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fb397332a1879b9764a3455a0bb1087bda876c2db8aca3a3cbb67b3dbce8cda0"}, - {file = "ruff-0.7.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:37d0b619546103274e7f62643d14e1adcbccb242efda4e4bdb9544d7764782e9"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d59f0c3ee4d1a6787614e7135b72e21024875266101142a09a61439cb6e38a5"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:44eb93c2499a169d49fafd07bc62ac89b1bc800b197e50ff4633aed212569299"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d0242ce53f3a576c35ee32d907475a8d569944c0407f91d207c8af5be5dae4e"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:6b6224af8b5e09772c2ecb8dc9f3f344c1aa48201c7f07e7315367f6dd90ac29"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c50f95a82b94421c964fae4c27c0242890a20fe67d203d127e84fbb8013855f5"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f3eff9961b5d2644bcf1616c606e93baa2d6b349e8aa8b035f654df252c8c67"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8963cab06d130c4df2fd52c84e9f10d297826d2e8169ae0c798b6221be1d1d2"}, - {file = "ruff-0.7.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:61b46049d6edc0e4317fb14b33bd693245281a3007288b68a3f5b74a22a0746d"}, - {file = "ruff-0.7.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:10ebce7696afe4644e8c1a23b3cf8c0f2193a310c18387c06e583ae9ef284de2"}, - {file = "ruff-0.7.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3f36d56326b3aef8eeee150b700e519880d1aab92f471eefdef656fd57492aa2"}, - {file = "ruff-0.7.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5d024301109a0007b78d57ab0ba190087b43dce852e552734ebf0b0b85e4fb16"}, - {file = "ruff-0.7.3-py3-none-win32.whl", hash = "sha256:4ba81a5f0c5478aa61674c5a2194de8b02652f17addf8dfc40c8937e6e7d79fc"}, - {file = "ruff-0.7.3-py3-none-win_amd64.whl", hash = "sha256:588a9ff2fecf01025ed065fe28809cd5a53b43505f48b69a1ac7707b1b7e4088"}, - {file = "ruff-0.7.3-py3-none-win_arm64.whl", hash = "sha256:1713e2c5545863cdbfe2cbce21f69ffaf37b813bfd1fb3b90dc9a6f1963f5a8c"}, - {file = "ruff-0.7.3.tar.gz", hash = "sha256:e1d1ba2e40b6e71a61b063354d04be669ab0d39c352461f3d789cac68b54a313"}, + {file = "ruff-0.12.10-py3-none-linux_armv6l.whl", hash = "sha256:8b593cb0fb55cc8692dac7b06deb29afda78c721c7ccfed22db941201b7b8f7b"}, + {file = "ruff-0.12.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ebb7333a45d56efc7c110a46a69a1b32365d5c5161e7244aaf3aa20ce62399c1"}, + {file = "ruff-0.12.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d59e58586829f8e4a9920788f6efba97a13d1fa320b047814e8afede381c6839"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:822d9677b560f1fdeab69b89d1f444bf5459da4aa04e06e766cf0121771ab844"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:37b4a64f4062a50c75019c61c7017ff598cb444984b638511f48539d3a1c98db"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c6f4064c69d2542029b2a61d39920c85240c39837599d7f2e32e80d36401d6e"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:059e863ea3a9ade41407ad71c1de2badfbe01539117f38f763ba42a1206f7559"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bef6161e297c68908b7218fa6e0e93e99a286e5ed9653d4be71e687dff101cf"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f1345fbf8fb0531cd722285b5f15af49b2932742fc96b633e883da8d841896b"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f68433c4fbc63efbfa3ba5db31727db229fa4e61000f452c540474b03de52a9"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:141ce3d88803c625257b8a6debf4a0473eb6eed9643a6189b68838b43e78165a"}, + {file = "ruff-0.12.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f3fc21178cd44c98142ae7590f42ddcb587b8e09a3b849cbc84edb62ee95de60"}, + {file = "ruff-0.12.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7d1a4e0bdfafcd2e3e235ecf50bf0176f74dd37902f241588ae1f6c827a36c56"}, + {file = "ruff-0.12.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:e67d96827854f50b9e3e8327b031647e7bcc090dbe7bb11101a81a3a2cbf1cc9"}, + {file = "ruff-0.12.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ae479e1a18b439c59138f066ae79cc0f3ee250712a873d00dbafadaad9481e5b"}, + {file = "ruff-0.12.10-py3-none-win32.whl", hash = "sha256:9de785e95dc2f09846c5e6e1d3a3d32ecd0b283a979898ad427a9be7be22b266"}, + {file = "ruff-0.12.10-py3-none-win_amd64.whl", hash = "sha256:7837eca8787f076f67aba2ca559cefd9c5cbc3a9852fd66186f4201b87c1563e"}, + {file = "ruff-0.12.10-py3-none-win_arm64.whl", hash = "sha256:cc138cc06ed9d4bfa9d667a65af7172b47840e1a98b02ce7011c391e54635ffc"}, + {file = "ruff-0.12.10.tar.gz", hash = "sha256:189ab65149d11ea69a2d775343adf5f49bb2426fc4780f65ee33b423ad2e47f9"}, ] [[package]] @@ -2287,6 +2429,8 @@ version = "3.3.3" description = "Python bindings to FreeDesktop.org Secret Service API" optional = false python-versions = ">=3.6" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and sys_platform == \"linux\"" files = [ {file = "SecretStorage-3.3.3-py3-none-any.whl", hash = "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99"}, {file = "SecretStorage-3.3.3.tar.gz", hash = "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77"}, @@ -2302,24 +2446,27 @@ version = "2.10.0" description = "A library implementing the 'SemVer' scheme." optional = false python-versions = ">=2.7" +groups = ["main"] files = [ {file = "semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177"}, {file = "semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c"}, ] [package.extras] -dev = ["Django (>=1.11)", "check-manifest", "colorama (<=0.4.1)", "coverage", "flake8", "nose2", "readme-renderer (<25.0)", "tox", "wheel", "zest.releaser[recommended]"] +dev = ["Django (>=1.11)", "check-manifest", "colorama (<=0.4.1) ; python_version == \"3.4\"", "coverage", "flake8", "nose2", "readme-renderer (<25.0) ; python_version == \"3.4\"", "tox", "wheel", "zest.releaser[recommended]"] doc = ["Sphinx", "sphinx-rtd-theme"] [[package]] name = "sentry-sdk" -version = "2.17.0" +version = "2.34.1" description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"all\" or extra == \"sentry\"" files = [ - {file = "sentry_sdk-2.17.0-py2.py3-none-any.whl", hash = "sha256:625955884b862cc58748920f9e21efdfb8e0d4f98cca4ab0d3918576d5b606ad"}, - {file = "sentry_sdk-2.17.0.tar.gz", hash = "sha256:dd0a05352b78ffeacced73a94e86f38b32e2eae15fff5f30ca5abb568a72eacf"}, + {file = "sentry_sdk-2.34.1-py2.py3-none-any.whl", hash = "sha256:b7a072e1cdc5abc48101d5146e1ae680fa81fe886d8d95aaa25a0b450c818d32"}, + {file = "sentry_sdk-2.34.1.tar.gz", hash = "sha256:69274eb8c5c38562a544c3e9f68b5be0a43be4b697f5fd385bf98e4fbe672687"}, ] [package.dependencies] @@ -2345,14 +2492,16 @@ grpcio = ["grpcio (>=1.21.1)", "protobuf (>=3.8.0)"] http2 = ["httpcore[http2] (==1.*)"] httpx = ["httpx (>=0.16.0)"] huey = ["huey (>=2)"] -huggingface-hub = ["huggingface-hub (>=0.22)"] +huggingface-hub = ["huggingface_hub (>=0.22)"] langchain = ["langchain (>=0.0.210)"] +launchdarkly = ["launchdarkly-server-sdk (>=9.8.0)"] litestar = ["litestar (>=2.0.0)"] loguru = ["loguru (>=0.5)"] openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"] +openfeature = ["openfeature-sdk (>=0.7.1)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry-experimental = ["opentelemetry-distro"] -pure-eval = ["asttokens", "executing", "pure-eval"] +pure-eval = ["asttokens", "executing", "pure_eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] @@ -2361,17 +2510,20 @@ sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] starlette = ["starlette (>=0.19.1)"] starlite = ["starlite (>=1.48)"] +statsig = ["statsig (>=0.55.3)"] tornado = ["tornado (>=6)"] +unleash = ["UnleashClient (>=6.0.1)"] [[package]] name = "service-identity" -version = "24.1.0" +version = "24.2.0" description = "Service identity verification for pyOpenSSL & cryptography." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "service_identity-24.1.0-py3-none-any.whl", hash = "sha256:a28caf8130c8a5c1c7a6f5293faaf239bbfb7751e4862436920ee6f2616f568a"}, - {file = "service_identity-24.1.0.tar.gz", hash = "sha256:6829c9d62fb832c2e1c435629b0a8c476e1929881f28bee4d20bc24161009221"}, + {file = "service_identity-24.2.0-py3-none-any.whl", hash = "sha256:6b047fbd8a84fd0bb0d55ebce4031e400562b9196e1e0d3e0fe2b8a59f6d4a85"}, + {file = "service_identity-24.2.0.tar.gz", hash = "sha256:b8683ba13f0d39c6cd5d625d2c5f65421d6d707b013b375c355751557cbe8e09"}, ] [package.dependencies] @@ -2381,7 +2533,7 @@ pyasn1 = "*" pyasn1-modules = "*" [package.extras] -dev = ["pyopenssl", "service-identity[idna,mypy,tests]"] +dev = ["coverage[toml] (>=5.0.2)", "idna", "mypy", "pyopenssl", "pytest", "types-pyopenssl"] docs = ["furo", "myst-parser", "pyopenssl", "sphinx", "sphinx-notfound-page"] idna = ["idna"] mypy = ["idna", "mypy", "types-pyopenssl"] @@ -2389,35 +2541,40 @@ tests = ["coverage[toml] (>=5.0.2)", "pytest"] [[package]] name = "setuptools" -version = "72.1.0" +version = "78.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main", "dev"] files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-78.1.1-py3-none-any.whl", hash = "sha256:c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561"}, + {file = "setuptools-78.1.1.tar.gz", hash = "sha256:fcc17fd9cd898242f6b4adfaca46137a9edef687f43e6f78469692a5e70d851d"}, ] [package.extras] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "setuptools-rust" -version = "1.8.1" +version = "1.11.1" description = "Setuptools Rust extension plugin" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "setuptools-rust-1.8.1.tar.gz", hash = "sha256:94b1dd5d5308b3138d5b933c3a2b55e6d6927d1a22632e509fcea9ddd0f7e486"}, - {file = "setuptools_rust-1.8.1-py3-none-any.whl", hash = "sha256:b5324493949ccd6aa0c03890c5f6b5f02de4512e3ac1697d02e9a6c02b18aa8e"}, + {file = "setuptools_rust-1.11.1-py3-none-any.whl", hash = "sha256:5eaaddaed268dc24a527ffa659ce56b22d3cf17b781247b779efd611031fe8ea"}, + {file = "setuptools_rust-1.11.1.tar.gz", hash = "sha256:7dabc4392252ced314b8050d63276e05fdc5d32398fc7d3cce1f6a6ac35b76c0"}, ] [package.dependencies] -semantic-version = ">=2.8.2,<3" +semantic_version = ">=2.8.2,<3" setuptools = ">=62.4" -tomli = {version = ">=1.2.1", markers = "python_version < \"3.11\""} [[package]] name = "signedjson" @@ -2425,6 +2582,7 @@ version = "1.1.4" description = "Sign JSON with Ed25519 signatures" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "signedjson-1.1.4-py3-none-any.whl", hash = "sha256:45569ec54241c65d2403fe3faf7169be5322547706a231e884ca2b427f23d228"}, {file = "signedjson-1.1.4.tar.gz", hash = "sha256:cd91c56af53f169ef032c62e9c4a3292dc158866933318d0592e3462db3d6492"}, @@ -2444,6 +2602,7 @@ version = "1.16.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["main"] files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -2455,6 +2614,7 @@ version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, @@ -2466,6 +2626,7 @@ version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, @@ -2477,6 +2638,8 @@ version = "235" description = "Python interface for libsystemd" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"systemd\"" files = [ {file = "systemd-python-235.tar.gz", hash = "sha256:4e57f39797fd5d9e2d22b8806a252d7c0106c936039d1e71c8c6b8008e695c0a"}, ] @@ -2487,6 +2650,8 @@ version = "1.0.2" description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2501,6 +2666,8 @@ version = "0.16.0" description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2515,44 +2682,79 @@ twisted = ["twisted"] [[package]] name = "tomli" -version = "2.0.2" +version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ - {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, - {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] [[package]] name = "tornado" -version = "6.4.1" +version = "6.5" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = true -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ - {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8"}, - {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698"}, - {file = "tornado-6.4.1-cp38-abi3-win32.whl", hash = "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d"}, - {file = "tornado-6.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7"}, - {file = "tornado-6.4.1.tar.gz", hash = "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9"}, + {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, + {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, + {file = "tornado-6.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c625b9d03f1fb4d64149c47d0135227f0434ebb803e2008040eb92906b0105a"}, + {file = "tornado-6.5-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a0d8d2309faf015903080fb5bdd969ecf9aa5ff893290845cf3fd5b2dd101bc"}, + {file = "tornado-6.5-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03576ab51e9b1677e4cdaae620d6700d9823568b7939277e4690fe4085886c55"}, + {file = "tornado-6.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ab75fe43d0e1b3a5e3ceddb2a611cb40090dd116a84fc216a07a298d9e000471"}, + {file = "tornado-6.5-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:119c03f440a832128820e87add8a175d211b7f36e7ee161c631780877c28f4fb"}, + {file = "tornado-6.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:231f2193bb4c28db2bdee9e57bc6ca0cd491f345cd307c57d79613b058e807e0"}, + {file = "tornado-6.5-cp39-abi3-win32.whl", hash = "sha256:fd20c816e31be1bbff1f7681f970bbbd0bb241c364220140228ba24242bcdc59"}, + {file = "tornado-6.5-cp39-abi3-win_amd64.whl", hash = "sha256:007f036f7b661e899bd9ef3fa5f87eb2cb4d1b2e7d67368e778e140a2f101a7a"}, + {file = "tornado-6.5-cp39-abi3-win_arm64.whl", hash = "sha256:542e380658dcec911215c4820654662810c06ad872eefe10def6a5e9b20e9633"}, + {file = "tornado-6.5.tar.gz", hash = "sha256:c70c0a26d5b2d85440e4debd14a8d0b463a0cf35d92d3af05f5f1ffa8675c826"}, ] [[package]] name = "towncrier" -version = "24.8.0" +version = "25.8.0" description = "Building newsfiles for your project." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["dev"] files = [ - {file = "towncrier-24.8.0-py3-none-any.whl", hash = "sha256:9343209592b839209cdf28c339ba45792fbfe9775b5f9c177462fd693e127d8d"}, - {file = "towncrier-24.8.0.tar.gz", hash = "sha256:013423ee7eed102b2f393c287d22d95f66f1a3ea10a4baa82d298001a7f18af3"}, + {file = "towncrier-25.8.0-py3-none-any.whl", hash = "sha256:b953d133d98f9aeae9084b56a3563fd2519dfc6ec33f61c9cd2c61ff243fb513"}, + {file = "towncrier-25.8.0.tar.gz", hash = "sha256:eef16d29f831ad57abb3ae32a0565739866219f1ebfbdd297d32894eb9940eb1"}, ] [package.dependencies] @@ -2567,42 +2769,46 @@ dev = ["furo (>=2024.05.06)", "nox", "packaging", "sphinx (>=5)", "twisted"] [[package]] name = "treq" -version = "24.9.1" +version = "25.5.0" description = "High-level Twisted HTTP Client API" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8.0" +groups = ["main"] files = [ - {file = "treq-24.9.1-py3-none-any.whl", hash = "sha256:eee4756fd9a857c77f180fd5202b52c518f2d3e2826dce28b89066c03bfc45d0"}, - {file = "treq-24.9.1.tar.gz", hash = "sha256:15da7fc404f3e4ed59d0abe5f8eef4966fabbe618039a2a23bc7c15305cefea8"}, + {file = "treq-25.5.0-py3-none-any.whl", hash = "sha256:e99d4e66cacaa1f0da82bb60b317d104c29dbd8ac0a75d7f657b348178d830f4"}, + {file = "treq-25.5.0.tar.gz", hash = "sha256:25dde3a55ae85ec2f2c56332c99aef255ab14f997d0d00552ebff13538a9804a"}, ] [package.dependencies] attrs = "*" hyperlink = ">=21.0.0" -incremental = "*" +incremental = ">=24.7.2" +multipart = "*" requests = ">=2.1.0" -Twisted = {version = ">=22.10.0", extras = ["tls"]} +twisted = {version = ">=22.10.0", extras = ["tls"]} typing-extensions = ">=3.10.0" [package.extras] dev = ["httpbin (==0.7.0)", "pep8", "pyflakes", "werkzeug (==2.0.3)"] -docs = ["sphinx (<7.0.0)"] +docs = ["sphinx", "sphinx-rtd-theme"] [[package]] name = "twine" -version = "5.1.1" +version = "6.1.0" description = "Collection of utilities for publishing packages on PyPI" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ - {file = "twine-5.1.1-py3-none-any.whl", hash = "sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997"}, - {file = "twine-5.1.1.tar.gz", hash = "sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db"}, + {file = "twine-6.1.0-py3-none-any.whl", hash = "sha256:a47f973caf122930bf0fbbf17f80b83bc1602c9ce393c7845f289a3001dc5384"}, + {file = "twine-6.1.0.tar.gz", hash = "sha256:be324f6272eff91d07ee93f251edf232fc647935dd585ac003539b42404a8dbd"}, ] [package.dependencies] -importlib-metadata = ">=3.6" -keyring = ">=15.1" -pkginfo = ">=1.8.1,<1.11" +id = "*" +importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""} +keyring = {version = ">=15.1", markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} +packaging = ">=24.0" readme-renderer = ">=35.0" requests = ">=2.20" requests-toolbelt = ">=0.8.0,<0.9.0 || >0.9.0" @@ -2610,20 +2816,24 @@ rfc3986 = ">=1.4.0" rich = ">=12.0.0" urllib3 = ">=1.26.0" +[package.extras] +keyring = ["keyring (>=15.1)"] + [[package]] name = "twisted" -version = "24.7.0" +version = "25.5.0" description = "An asynchronous networking framework written in Python" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ - {file = "twisted-24.7.0-py3-none-any.whl", hash = "sha256:734832ef98108136e222b5230075b1079dad8a3fc5637319615619a7725b0c81"}, - {file = "twisted-24.7.0.tar.gz", hash = "sha256:5a60147f044187a127ec7da96d170d49bcce50c6fd36f594e60f4587eff4d394"}, + {file = "twisted-25.5.0-py3-none-any.whl", hash = "sha256:8559f654d01a54a8c3efe66d533d43f383531ebf8d81d9f9ab4769d91ca15df7"}, + {file = "twisted-25.5.0.tar.gz", hash = "sha256:1deb272358cb6be1e3e8fc6f9c8b36f78eb0fa7c2233d2dbe11ec6fee04ea316"}, ] [package.dependencies] -attrs = ">=21.3.0" -automat = ">=0.8.0" +attrs = ">=22.2.0" +automat = ">=24.8.0" constantly = ">=15.1" hyperlink = ">=17.1.1" idna = {version = ">=2.4", optional = true, markers = "extra == \"tls\""} @@ -2634,29 +2844,32 @@ typing-extensions = ">=4.2.0" zope-interface = ">=5" [package.extras] -all-non-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +all-non-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)"] -dev = ["coverage (>=7.5,<8.0)", "cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "python-subunit (>=1.4,<2.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)"] -dev-release = ["pydoctor (>=23.9.0,<23.10.0)", "pydoctor (>=23.9.0,<23.10.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"] -gtk-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pygobject", "pygobject", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] -http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] -macos-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] -mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "coverage (>=7.5,<8.0)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (>=1.8,<2.0)", "mypy-zope (>=1.0.3,<1.1.0)", "priority (>=1.1.0,<2.0)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools"] -osx-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] -serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] -test = ["cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"] +dev = ["coverage (>=7.5,<8.0)", "cython-test-exception-raiser (>=1.0.2,<2)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "pydoctor (>=24.11.1,<24.12.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "python-subunit (>=1.4,<2.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)"] +dev-release = ["pydoctor (>=24.11.1,<24.12.0)", "pydoctor (>=24.11.1,<24.12.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"] +gtk-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pygobject", "pygobject", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] +http2 = ["h2 (>=3.2,<5.0)", "priority (>=1.1.0,<2.0)"] +macos-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] +mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "coverage (>=7.5,<8.0)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (==1.10.1)", "mypy-zope (==1.0.6)", "priority (>=1.1.0,<2.0)", "pydoctor (>=24.11.1,<24.12.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools", "wsproto"] +osx-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] +serial = ["pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\""] +test = ["cython-test-exception-raiser (>=1.0.2,<2)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"] tls = ["idna (>=2.4)", "pyopenssl (>=21.0.0)", "service-identity (>=18.1.0)"] -windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "twisted-iocpsupport (>=1.0.2)"] +websocket = ["wsproto"] +windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "twisted-iocpsupport (>=1.0.2)", "wsproto", "wsproto"] [[package]] name = "txredisapi" -version = "1.4.10" +version = "1.4.11" description = "non-blocking redis client for python" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"all\" or extra == \"redis\"" files = [ - {file = "txredisapi-1.4.10-py3-none-any.whl", hash = "sha256:0a6ea77f27f8cf092f907654f08302a97b48fa35f24e0ad99dfb74115f018161"}, - {file = "txredisapi-1.4.10.tar.gz", hash = "sha256:7609a6af6ff4619a3189c0adfb86aeda789afba69eb59fc1e19ac0199e725395"}, + {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, + {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, ] [package.dependencies] @@ -2665,13 +2878,14 @@ twisted = "*" [[package]] name = "types-bleach" -version = "6.1.0.20240331" +version = "6.2.0.20250809" description = "Typing stubs for bleach" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["dev"] files = [ - {file = "types-bleach-6.1.0.20240331.tar.gz", hash = "sha256:2ee858a84fb06fc2225ff56ba2f7f6c88b65638659efae0d7bfd6b24a1b5a524"}, - {file = "types_bleach-6.1.0.20240331-py3-none-any.whl", hash = "sha256:399bc59bfd20a36a56595f13f805e56c8a08e5a5c07903e5cf6fafb5a5107dd4"}, + {file = "types_bleach-6.2.0.20250809-py3-none-any.whl", hash = "sha256:0b372a75117947d9ac8a31ae733fd0f8d92ec75c4772e7b37093ba3fa5b48fb9"}, + {file = "types_bleach-6.2.0.20250809.tar.gz", hash = "sha256:188d7a1119f6c953140b513ed57ba4213755695815472c19d0c22ac09c79b90b"}, ] [package.dependencies] @@ -2683,6 +2897,7 @@ version = "1.16.0.20240331" description = "Typing stubs for cffi" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-cffi-1.16.0.20240331.tar.gz", hash = "sha256:b8b20d23a2b89cfed5f8c5bc53b0cb8677c3aac6d970dbc771e28b9c698f5dee"}, {file = "types_cffi-1.16.0.20240331-py3-none-any.whl", hash = "sha256:a363e5ea54a4eb6a4a105d800685fde596bc318089b025b27dee09849fe41ff0"}, @@ -2691,23 +2906,13 @@ files = [ [package.dependencies] types-setuptools = "*" -[[package]] -name = "types-commonmark" -version = "0.9.2.20240106" -description = "Typing stubs for commonmark" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types-commonmark-0.9.2.20240106.tar.gz", hash = "sha256:52a062b71766d6ab258fca2d8e19fb0853796e25ca9afa9d0f67a1e42c93479f"}, - {file = "types_commonmark-0.9.2.20240106-py3-none-any.whl", hash = "sha256:606d9de1e3a96cab0b1c0b6cccf4df099116148d1d864d115fde2e27ad6877c3"}, -] - [[package]] name = "types-html5lib" version = "1.1.11.20240228" description = "Typing stubs for html5lib" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-html5lib-1.1.11.20240228.tar.gz", hash = "sha256:22736b7299e605ec4ba539d48691e905fd0c61c3ea610acc59922232dc84cede"}, {file = "types_html5lib-1.1.11.20240228-py3-none-any.whl", hash = "sha256:af5de0125cb0fe5667543b158db83849b22e25c0e36c9149836b095548bf1020"}, @@ -2715,13 +2920,14 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.23.0.20240813" +version = "4.25.1.20250822" description = "Typing stubs for jsonschema" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["dev"] files = [ - {file = "types-jsonschema-4.23.0.20240813.tar.gz", hash = "sha256:c93f48206f209a5bc4608d295ac39f172fb98b9e24159ce577dbd25ddb79a1c0"}, - {file = "types_jsonschema-4.23.0.20240813-py3-none-any.whl", hash = "sha256:be283e23f0b87547316c2ee6b0fd36d95ea30e921db06478029e10b5b6aa6ac3"}, + {file = "types_jsonschema-4.25.1.20250822-py3-none-any.whl", hash = "sha256:f82c2d7fa1ce1c0b84ba1de4ed6798469768188884db04e66421913a4e181294"}, + {file = "types_jsonschema-4.25.1.20250822.tar.gz", hash = "sha256:aac69ed4b23f49aaceb7fcb834141d61b9e4e6a7f6008cb2f0d3b831dfa8464a"}, ] [package.dependencies] @@ -2733,6 +2939,7 @@ version = "1.3.0.20240530" description = "Typing stubs for netaddr" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-netaddr-1.3.0.20240530.tar.gz", hash = "sha256:742c2ec1f202b666f544223e2616b34f1f13df80c91e5aeaaa93a72e4d0774ea"}, {file = "types_netaddr-1.3.0.20240530-py3-none-any.whl", hash = "sha256:354998d018e326da4f1d9b005fc91137b7c2c473aaf03c4ef64bf83c6861b440"}, @@ -2740,13 +2947,14 @@ files = [ [[package]] name = "types-opentracing" -version = "2.4.10.6" +version = "2.4.10.20250622" description = "Typing stubs for opentracing" optional = false -python-versions = "*" +python-versions = ">=3.9" +groups = ["dev"] files = [ - {file = "types-opentracing-2.4.10.6.tar.gz", hash = "sha256:87a1bdfce9de5e555e30497663583b9b9c3bb494d029ef9806aa1f137c19e744"}, - {file = "types_opentracing-2.4.10.6-py3-none-any.whl", hash = "sha256:25914c834db033a4a38fc322df0b5e5e14503b0ac97f78304ae180d721555e97"}, + {file = "types_opentracing-2.4.10.20250622-py3-none-any.whl", hash = "sha256:26bc21f9e385d54898b47e9bd1fa13f200c2dada50394f6eafd063ed53813062"}, + {file = "types_opentracing-2.4.10.20250622.tar.gz", hash = "sha256:00db48b7f57136c45ac3250218bd0f18b9792566dfcbd5ad1de9f7e180347e74"}, ] [[package]] @@ -2755,6 +2963,7 @@ version = "10.2.0.20240822" description = "Typing stubs for Pillow" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-Pillow-10.2.0.20240822.tar.gz", hash = "sha256:559fb52a2ef991c326e4a0d20accb3bb63a7ba8d40eb493e0ecb0310ba52f0d3"}, {file = "types_Pillow-10.2.0.20240822-py3-none-any.whl", hash = "sha256:d9dab025aba07aeb12fd50a6799d4eac52a9603488eca09d7662543983f16c5d"}, @@ -2762,13 +2971,14 @@ files = [ [[package]] name = "types-psycopg2" -version = "2.9.21.20241019" +version = "2.9.21.20250915" description = "Typing stubs for psycopg2" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["dev"] files = [ - {file = "types-psycopg2-2.9.21.20241019.tar.gz", hash = "sha256:bca89b988d2ebd19bcd08b177d22a877ea8b841decb10ed130afcf39404612fa"}, - {file = "types_psycopg2-2.9.21.20241019-py3-none-any.whl", hash = "sha256:44d091e67732d16a941baae48cd7b53bf91911bc36888652447cf1ef0c1fb3f6"}, + {file = "types_psycopg2-2.9.21.20250915-py3-none-any.whl", hash = "sha256:eefe5ccdc693fc086146e84c9ba437bb278efe1ef330b299a0cb71169dc6c55f"}, + {file = "types_psycopg2-2.9.21.20250915.tar.gz", hash = "sha256:bfeb8f54c32490e7b5edc46215ab4163693192bc90407b4a023822de9239f5c8"}, ] [[package]] @@ -2777,6 +2987,7 @@ version = "24.1.0.20240722" description = "Typing stubs for pyOpenSSL" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-pyOpenSSL-24.1.0.20240722.tar.gz", hash = "sha256:47913b4678a01d879f503a12044468221ed8576263c1540dcb0484ca21b08c39"}, {file = "types_pyOpenSSL-24.1.0.20240722-py3-none-any.whl", hash = "sha256:6a7a5d2ec042537934cfb4c9d4deb0e16c4c6250b09358df1f083682fe6fda54"}, @@ -2788,24 +2999,26 @@ types-cffi = "*" [[package]] name = "types-pyyaml" -version = "6.0.12.20240917" +version = "6.0.12.20250809" description = "Typing stubs for PyYAML" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["dev"] files = [ - {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"}, - {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"}, + {file = "types_pyyaml-6.0.12.20250809-py3-none-any.whl", hash = "sha256:032b6003b798e7de1a1ddfeefee32fac6486bdfe4845e0ae0e7fb3ee4512b52f"}, + {file = "types_pyyaml-6.0.12.20250809.tar.gz", hash = "sha256:af4a1aca028f18e75297da2ee0da465f799627370d74073e96fee876524f61b5"}, ] [[package]] name = "types-requests" -version = "2.32.0.20241016" +version = "2.32.4.20250809" description = "Typing stubs for requests" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["dev"] files = [ - {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, - {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, + {file = "types_requests-2.32.4.20250809-py3-none-any.whl", hash = "sha256:f73d1832fb519ece02c85b1f09d5f0dd3108938e7d47e7f94bbfa18a6782b163"}, + {file = "types_requests-2.32.4.20250809.tar.gz", hash = "sha256:d8060de1c8ee599311f56ff58010fb4902f462a1470802cf9f6ed27bc46c4df3"}, ] [package.dependencies] @@ -2813,32 +3026,50 @@ urllib3 = ">=2" [[package]] name = "types-setuptools" -version = "75.2.0.20241019" +version = "80.9.0.20250822" description = "Typing stubs for setuptools" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["dev"] files = [ - {file = "types-setuptools-75.2.0.20241019.tar.gz", hash = "sha256:86ea31b5f6df2c6b8f2dc8ae3f72b213607f62549b6fa2ed5866e5299f968694"}, - {file = "types_setuptools-75.2.0.20241019-py3-none-any.whl", hash = "sha256:2e48ff3acd4919471e80d5e3f049cce5c177e108d5d36d2d4cee3fa4d4104258"}, + {file = "types_setuptools-80.9.0.20250822-py3-none-any.whl", hash = "sha256:53bf881cb9d7e46ed12c76ef76c0aaf28cfe6211d3fab12e0b83620b1a8642c3"}, + {file = "types_setuptools-80.9.0.20250822.tar.gz", hash = "sha256:070ea7716968ec67a84c7f7768d9952ff24d28b65b6594797a464f1b3066f965"}, ] [[package]] name = "typing-extensions" -version = "4.12.2" -description = "Backported and Experimental Type Hints for Python 3.8+" +version = "4.14.1" +description = "Backported and Experimental Type Hints for Python 3.9+" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main", "dev"] files = [ - {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, - {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, + {file = "typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76"}, + {file = "typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36"}, ] +[[package]] +name = "typing-inspection" +version = "0.4.0" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +groups = ["main", "dev"] +files = [ + {file = "typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f"}, + {file = "typing_inspection-0.4.0.tar.gz", hash = "sha256:9765c87de36671694a67904bf2c96e395be9c6439bb6c87b5142569dcdd65122"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + [[package]] name = "unpaddedbase64" version = "2.1.0" description = "Encode and decode Base64 without \"=\" padding" optional = false python-versions = ">=3.6,<4.0" +groups = ["main"] files = [ {file = "unpaddedbase64-2.1.0-py3-none-any.whl", hash = "sha256:485eff129c30175d2cd6f0cd8d2310dff51e666f7f36175f738d75dfdbd0b1c6"}, {file = "unpaddedbase64-2.1.0.tar.gz", hash = "sha256:7273c60c089de39d90f5d6d4a7883a79e319dc9d9b1c8924a7fab96178a5f005"}, @@ -2846,17 +3077,18 @@ files = [ [[package]] name = "urllib3" -version = "2.2.2" +version = "2.5.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main", "dev"] files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, + {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, + {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -2867,101 +3099,20 @@ version = "0.5.1" description = "Character encoding aliases for legacy web content" optional = false python-versions = "*" +groups = ["main", "dev"] files = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, ] -[[package]] -name = "wrapt" -version = "1.15.0" -description = "Module for decorators, wrappers and monkey patching." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, -] - [[package]] name = "xmlschema" version = "2.4.0" description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, @@ -2981,6 +3132,8 @@ version = "3.19.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\" or python_version < \"3.10\"" files = [ {file = "zipp-3.19.1-py3-none-any.whl", hash = "sha256:2828e64edb5386ea6a52e7ba7cdb17bb30a73a858f5eb6eb93d8d36f5ea26091"}, {file = "zipp-3.19.1.tar.gz", hash = "sha256:35427f6d5594f4acf82d25541438348c26736fa9b3afa2754bcd63cdb99d8e8f"}, @@ -2996,6 +3149,7 @@ version = "4.6" description = "Very basic event publishing system" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "zope.event-4.6-py2.py3-none-any.whl", hash = "sha256:73d9e3ef750cca14816a9c322c7250b0d7c9dbc337df5d1b807ff8d3d0b9e97c"}, {file = "zope.event-4.6.tar.gz", hash = "sha256:81d98813046fc86cc4136e3698fee628a3282f9c320db18658c21749235fce80"}, @@ -3014,6 +3168,7 @@ version = "7.1.0" description = "Interfaces for Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "zope.interface-7.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2bd9e9f366a5df08ebbdc159f8224904c1c5ce63893984abb76954e6fbe4381a"}, {file = "zope.interface-7.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:661d5df403cd3c5b8699ac480fa7f58047a3253b029db690efa0c3cf209993ef"}, @@ -3068,6 +3223,7 @@ version = "7.0.1" description = "zope.interface extension for defining data schemas" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "zope.schema-7.0.1-py3-none-any.whl", hash = "sha256:cf006c678793b00e0075ad54d55281c8785ea21e5bc1f5ec0584787719c2aab2"}, {file = "zope.schema-7.0.1.tar.gz", hash = "sha256:ead4dbcb03354d4e410c9a3b904451eb44d90254751b1cbdedf4a61aede9fbb9"}, @@ -3083,7 +3239,7 @@ docs = ["Sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"] [extras] -all = ["Pympler", "authlib", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pyicu", "pysaml2", "sentry-sdk", "txredisapi"] +all = ["Pympler", "authlib", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "sentry-sdk", "txredisapi"] cache-memory = ["Pympler"] jwt = ["authlib"] matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] @@ -3096,9 +3252,8 @@ sentry = ["sentry-sdk"] systemd = ["systemd-python"] test = ["idna", "parameterized"] url-preview = ["lxml"] -user-search = ["pyicu"] [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "d71159b19349fdc0b7cd8e06e8c8778b603fc37b941c6df34ddc31746783d94d" +content-hash = "2e8ea085e1a0c6f0ac051d4bc457a96827d01f621b1827086de01a5ffa98cf79" diff --git a/pyproject.toml b/pyproject.toml index 36d3437185..f9dd0ca26b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,10 @@ select = [ "PIE", # flake8-executable "EXE", + # flake8-logging + "LOG", + # flake8-logging-format + "G", ] [tool.ruff.lint.isort] @@ -97,7 +101,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.120.0rc1" +version = "1.138.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" @@ -174,8 +178,13 @@ signedjson = "^1.1.0" service-identity = ">=18.1.0" # Twisted 18.9 introduces some logger improvements that the structured # logger utilises -Twisted = {extras = ["tls"], version = ">=18.9.0"} -treq = ">=15.1" +# Twisted 19.7.0 moves test helpers to a new module and deprecates the old location. +# Twisted 21.2.0 introduces contextvar support. +# We could likely bump this to 22.1 without making distro packagers' +# lives hard (as of 2025-07, distro support is Ubuntu LTS: 22.1, Debian stable: 22.4, +# RHEL 9: 22.10) +Twisted = {extras = ["tls"], version = ">=21.2.0"} +treq = ">=21.5.0" # Twisted has required pyopenssl 16.0 since about Twisted 16.6. pyOpenSSL = ">=16.0.0" PyYAML = ">=5.3" @@ -191,7 +200,9 @@ pymacaroons = ">=0.13.0" msgpack = ">=0.5.2" phonenumbers = ">=8.2.0" # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. -prometheus-client = ">=0.4.0" +# `prometheus_client.metrics` was added in 0.5.0, so we require that too. +# We chose 0.6.0 as that is the current version in Debian Buster (oldstable). +prometheus-client = ">=0.6.0" # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936 attrs = ">=19.2.0,!=21.1.0" @@ -220,7 +231,7 @@ pydantic = ">=1.7.4, <3" # https://github.com/python-poetry/poetry/issues/6154). Both `pip install` and # `poetry build` do the right thing without this explicit dependency. # -# This isn't really a dev-dependency, as `poetry install --no-dev` will fail, +# This isn't really a dev-dependency, as `poetry install --without dev` will fail, # but the alternative is to add it to the main list of deps where it isn't # needed. setuptools_rust = ">=1.3" @@ -250,7 +261,6 @@ hiredis = { version = "*", optional = true } Pympler = { version = "*", optional = true } parameterized = { version = ">=0.7.4", optional = true } idna = { version = ">=2.5", optional = true } -pyicu = { version = ">=2.10.2", optional = true } [tool.poetry.extras] # NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified @@ -273,10 +283,6 @@ redis = ["txredisapi", "hiredis"] # Required to use experimental `caches.track_memory_usage` config option. cache-memory = ["pympler"] test = ["parameterized", "idna"] -# Allows for better search for international characters in the user directory. This -# requires libicu's development headers installed on the system (e.g. libicu-dev on -# Debian-based distributions). -user-search = ["pyicu"] # The duplication here is awful. I hate hate hate hate hate it. However, for now I want # to ensure you can still `pip install matrix-synapse[all]` like today. Two motivations: @@ -308,8 +314,6 @@ all = [ "txredisapi", "hiredis", # cache-memory "pympler", - # improved user search - "pyicu", # omitted: # - test: it's useful to have this separate from dev deps in the olddeps job # - systemd: this is a system-based requirement @@ -320,7 +324,7 @@ all = [ # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.7.3" +ruff = "0.12.10" # Type checking only works with the pydantic.v1 compat module from pydantic v2 pydantic = "^2" @@ -329,7 +333,6 @@ lxml-stubs = ">=0.4.0" mypy = "*" mypy-zope = "*" types-bleach = ">=4.1.0" -types-commonmark = ">=0.9.2" types-jsonschema = ">=3.2.0" types-netaddr = ">=0.8.0.6" types-opentracing = ">=2.4.2" @@ -352,7 +355,7 @@ idna = ">=2.5" click = ">=8.1.3" # GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. GitPython = ">=3.1.20" -commonmark = ">=0.9.1" +markdown-it-py = ">=3.0.0" pygithub = ">=1.55" # The following are executed as commands by the release script. twine = "*" @@ -370,7 +373,7 @@ tomli = ">=1.2.3" # runtime errors caused by build system changes. # We are happy to raise these upper bounds upon request, # provided we check that it's safe to do so (i.e. that CI passes). -requires = ["poetry-core>=1.1.0,<=1.9.1", "setuptools_rust>=1.3,<=1.10.2"] +requires = ["poetry-core>=1.1.0,<=2.1.3", "setuptools_rust>=1.3,<=1.11.1"] build-backend = "poetry.core.masonry.api" @@ -378,16 +381,19 @@ build-backend = "poetry.core.masonry.api" # Skip unsupported platforms (by us or by Rust). # See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets. # We skip: -# - CPython 3.6, 3.7 and 3.8: EOLed -# - PyPy 3.7 and 3.8: we only support Python 3.9+ +# - CPython and PyPy 3.8: EOLed # - musllinux i686: excluded to reduce number of wheels we build. # c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677 -# - PyPy on Aarch64 and musllinux on aarch64: too slow to build. -# c.f. https://github.com/matrix-org/synapse/pull/14259 -skip = "cp36* cp37* cp38* pp37* pp38* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64" +skip = "cp38* pp38* *-musllinux_i686" +# Enable non-default builds. +# "pypy" used to be included by default up until cibuildwheel 3. +enable = "pypy" -# We need a rust compiler -before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal" +# We need a rust compiler. +# +# We temporarily pin Rust to 1.82.0 to work around +# https://github.com/element-hq/synapse/issues/17988 +before-all = "sh .ci/before_build_wheel.sh" environment= { PATH = "$PATH:$HOME/.cargo/bin" } # For some reason if we don't manually clean the build directory we diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 026487275c..0706357294 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -7,7 +7,7 @@ name = "synapse" version = "0.1.0" edition = "2021" -rust-version = "1.66.0" +rust-version = "1.82.0" [lib] name = "synapse" @@ -23,26 +23,36 @@ name = "synapse.synapse_rust" [dependencies] anyhow = "1.0.63" -base64 = "0.21.7" +base64 = "0.22.1" bytes = "1.6.0" headers = "0.4.0" http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.21.0", features = [ +pyo3 = { version = "0.25.1", features = [ "macros", "anyhow", "abi3", - "abi3-py38", + "abi3-py39", ] } -pyo3-log = "0.10.0" -pythonize = "0.21.0" +pyo3-log = "0.12.4" +pythonize = "0.25.0" regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive"] } serde_json = "1.0.85" ulid = "1.1.2" +icu_segmenter = "2.0.0" +reqwest = { version = "0.12.15", default-features = false, features = [ + "http2", + "stream", + "rustls-tls-native-roots", +] } +http-body-util = "0.1.3" +futures = "0.3.31" +tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } +once_cell = "1.18.0" [features] extension-module = ["pyo3/extension-module"] diff --git a/rust/benches/evaluator.rs b/rust/benches/evaluator.rs index 28537e187e..96169fd45d 100644 --- a/rust/benches/evaluator.rs +++ b/rust/benches/evaluator.rs @@ -61,6 +61,7 @@ fn bench_match_exact(b: &mut Bencher) { vec![], false, false, + false, ) .unwrap(); @@ -71,10 +72,10 @@ fn bench_match_exact(b: &mut Bencher) { }, )); - let matched = eval.match_condition(&condition, None, None).unwrap(); + let matched = eval.match_condition(&condition, None, None, None).unwrap(); assert!(matched, "Didn't match"); - b.iter(|| eval.match_condition(&condition, None, None).unwrap()); + b.iter(|| eval.match_condition(&condition, None, None, None).unwrap()); } #[bench] @@ -107,6 +108,7 @@ fn bench_match_word(b: &mut Bencher) { vec![], false, false, + false, ) .unwrap(); @@ -117,10 +119,10 @@ fn bench_match_word(b: &mut Bencher) { }, )); - let matched = eval.match_condition(&condition, None, None).unwrap(); + let matched = eval.match_condition(&condition, None, None, None).unwrap(); assert!(matched, "Didn't match"); - b.iter(|| eval.match_condition(&condition, None, None).unwrap()); + b.iter(|| eval.match_condition(&condition, None, None, None).unwrap()); } #[bench] @@ -153,6 +155,7 @@ fn bench_match_word_miss(b: &mut Bencher) { vec![], false, false, + false, ) .unwrap(); @@ -163,10 +166,10 @@ fn bench_match_word_miss(b: &mut Bencher) { }, )); - let matched = eval.match_condition(&condition, None, None).unwrap(); + let matched = eval.match_condition(&condition, None, None, None).unwrap(); assert!(!matched, "Didn't match"); - b.iter(|| eval.match_condition(&condition, None, None).unwrap()); + b.iter(|| eval.match_condition(&condition, None, None, None).unwrap()); } #[bench] @@ -199,6 +202,7 @@ fn bench_eval_message(b: &mut Bencher) { vec![], false, false, + false, ) .unwrap(); @@ -210,7 +214,8 @@ fn bench_eval_message(b: &mut Bencher) { false, false, false, + false, ); - b.iter(|| eval.run(&rules, Some("bob"), Some("person"))); + b.iter(|| eval.run(&rules, Some("bob"), Some("person"), None)); } diff --git a/rust/src/acl/mod.rs b/rust/src/acl/mod.rs index 982720ba90..57b45475fd 100644 --- a/rust/src/acl/mod.rs +++ b/rust/src/acl/mod.rs @@ -32,14 +32,14 @@ use crate::push::utils::{glob_to_regex, GlobMatchType}; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let child_module = PyModule::new_bound(py, "acl")?; + let child_module = PyModule::new(py, "acl")?; child_module.add_class::()?; m.add_submodule(&child_module)?; // We need to manually add the module to sys.modules to make `from // synapse.synapse_rust import acl` work. - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.acl", child_module)?; diff --git a/rust/src/errors.rs b/rust/src/errors.rs index 4e580e3e8c..149019ff4b 100644 --- a/rust/src/errors.rs +++ b/rust/src/errors.rs @@ -58,3 +58,15 @@ impl NotFoundError { NotFoundError::new_err(()) } } + +import_exception!(synapse.api.errors, HttpResponseException); + +impl HttpResponseException { + pub fn new(status: StatusCode, bytes: Vec) -> pyo3::PyErr { + HttpResponseException::new_err(( + status.as_u16(), + status.canonical_reason().unwrap_or_default(), + bytes, + )) + } +} diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index ad87825f16..4711fc540f 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -41,9 +41,11 @@ use pyo3::{ pybacked::PyBackedStr, pyclass, pymethods, types::{PyAnyMethods, PyDict, PyDictMethods, PyString}, - Bound, IntoPy, PyAny, PyObject, PyResult, Python, + Bound, IntoPyObject, PyAny, PyObject, PyResult, Python, }; +use crate::UnwrapInfallible; + /// Definitions of the various fields of the internal metadata. #[derive(Clone)] enum EventInternalMetadataData { @@ -52,6 +54,7 @@ enum EventInternalMetadataData { RecheckRedaction(bool), SoftFailed(bool), ProactivelySend(bool), + PolicyServerSpammy(bool), Redacted(bool), TxnId(Box), TokenId(i64), @@ -60,31 +63,66 @@ enum EventInternalMetadataData { impl EventInternalMetadataData { /// Convert the field to its name and python object. - fn to_python_pair<'a>(&self, py: Python<'a>) -> (&'a Bound<'a, PyString>, PyObject) { + fn to_python_pair<'a>(&self, py: Python<'a>) -> (&'a Bound<'a, PyString>, Bound<'a, PyAny>) { match self { - EventInternalMetadataData::OutOfBandMembership(o) => { - (pyo3::intern!(py, "out_of_band_membership"), o.into_py(py)) - } - EventInternalMetadataData::SendOnBehalfOf(o) => { - (pyo3::intern!(py, "send_on_behalf_of"), o.into_py(py)) - } - EventInternalMetadataData::RecheckRedaction(o) => { - (pyo3::intern!(py, "recheck_redaction"), o.into_py(py)) - } - EventInternalMetadataData::SoftFailed(o) => { - (pyo3::intern!(py, "soft_failed"), o.into_py(py)) - } - EventInternalMetadataData::ProactivelySend(o) => { - (pyo3::intern!(py, "proactively_send"), o.into_py(py)) - } - EventInternalMetadataData::Redacted(o) => { - (pyo3::intern!(py, "redacted"), o.into_py(py)) - } - EventInternalMetadataData::TxnId(o) => (pyo3::intern!(py, "txn_id"), o.into_py(py)), - EventInternalMetadataData::TokenId(o) => (pyo3::intern!(py, "token_id"), o.into_py(py)), - EventInternalMetadataData::DeviceId(o) => { - (pyo3::intern!(py, "device_id"), o.into_py(py)) - } + EventInternalMetadataData::OutOfBandMembership(o) => ( + pyo3::intern!(py, "out_of_band_membership"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::SendOnBehalfOf(o) => ( + pyo3::intern!(py, "send_on_behalf_of"), + o.into_pyobject(py).unwrap_infallible().into_any(), + ), + EventInternalMetadataData::RecheckRedaction(o) => ( + pyo3::intern!(py, "recheck_redaction"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::SoftFailed(o) => ( + pyo3::intern!(py, "soft_failed"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::ProactivelySend(o) => ( + pyo3::intern!(py, "proactively_send"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::PolicyServerSpammy(o) => ( + pyo3::intern!(py, "policy_server_spammy"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::Redacted(o) => ( + pyo3::intern!(py, "redacted"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::TxnId(o) => ( + pyo3::intern!(py, "txn_id"), + o.into_pyobject(py).unwrap_infallible().into_any(), + ), + EventInternalMetadataData::TokenId(o) => ( + pyo3::intern!(py, "token_id"), + o.into_pyobject(py).unwrap_infallible().into_any(), + ), + EventInternalMetadataData::DeviceId(o) => ( + pyo3::intern!(py, "device_id"), + o.into_pyobject(py).unwrap_infallible().into_any(), + ), } } @@ -125,6 +163,11 @@ impl EventInternalMetadataData { .extract() .with_context(|| format!("'{key_str}' has invalid type"))?, ), + "policy_server_spammy" => EventInternalMetadataData::PolicyServerSpammy( + value + .extract() + .with_context(|| format!("'{key_str}' has invalid type"))?, + ), "redacted" => EventInternalMetadataData::Redacted( value .extract() @@ -247,7 +290,7 @@ impl EventInternalMetadata { /// /// Note that `outlier` and `stream_ordering` are stored in separate columns so are not returned here. fn get_dict(&self, py: Python<'_>) -> PyResult { - let dict = PyDict::new_bound(py); + let dict = PyDict::new(py); for entry in &self.data { let (key, value) = entry.to_python_pair(py); @@ -397,6 +440,17 @@ impl EventInternalMetadata { set_property!(self, ProactivelySend, obj); } + #[getter] + fn get_policy_server_spammy(&self) -> PyResult { + Ok(get_property_opt!(self, PolicyServerSpammy) + .copied() + .unwrap_or(false)) + } + #[setter] + fn set_policy_server_spammy(&mut self, obj: bool) { + set_property!(self, PolicyServerSpammy, obj); + } + #[getter] fn get_redacted(&self) -> PyResult { let bool = get_property!(self, Redacted)?; diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 0bb6cdb181..209efb917b 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -30,7 +30,7 @@ mod internal_metadata; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let child_module = PyModule::new_bound(py, "events")?; + let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; @@ -38,7 +38,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> // We need to manually add the module to sys.modules to make `from // synapse.synapse_rust import events` work. - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.events", child_module)?; diff --git a/rust/src/http.rs b/rust/src/http.rs index af052ab721..63ed05be54 100644 --- a/rust/src/http.rs +++ b/rust/src/http.rs @@ -70,7 +70,7 @@ pub fn http_request_from_twisted(request: &Bound<'_, PyAny>) -> PyResult. + */ + +use std::{collections::HashMap, future::Future}; + +use anyhow::Context; +use futures::TryStreamExt; +use once_cell::sync::OnceCell; +use pyo3::{create_exception, exceptions::PyException, prelude::*}; +use reqwest::RequestBuilder; +use tokio::runtime::Runtime; + +use crate::errors::HttpResponseException; + +create_exception!( + synapse.synapse_rust.http_client, + RustPanicError, + PyException, + "A panic which happened in a Rust future" +); + +impl RustPanicError { + fn from_panic(panic_err: &(dyn std::any::Any + Send + 'static)) -> PyErr { + // Apparently this is how you extract the panic message from a panic + let panic_message = if let Some(str_slice) = panic_err.downcast_ref::<&str>() { + str_slice + } else if let Some(string) = panic_err.downcast_ref::() { + string + } else { + "unknown error" + }; + Self::new_err(panic_message.to_owned()) + } +} + +/// This is the name of the attribute where we store the runtime on the reactor +static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; + +/// A Python wrapper around a Tokio runtime. +/// +/// This allows us to 'store' the runtime on the reactor instance, starting it +/// when the reactor starts, and stopping it when the reactor shuts down. +#[pyclass] +struct PyTokioRuntime { + runtime: Option, +} + +#[pymethods] +impl PyTokioRuntime { + fn start(&mut self) -> PyResult<()> { + // TODO: allow customization of the runtime like the number of threads + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build()?; + + self.runtime = Some(runtime); + + Ok(()) + } + + fn shutdown(&mut self) -> PyResult<()> { + let runtime = self + .runtime + .take() + .context("Runtime was already shutdown")?; + + // Dropping the runtime will shut it down + drop(runtime); + + Ok(()) + } +} + +impl PyTokioRuntime { + /// Get the handle to the Tokio runtime, if it is running. + fn handle(&self) -> PyResult<&tokio::runtime::Handle> { + let handle = self + .runtime + .as_ref() + .context("Tokio runtime is not running")? + .handle(); + + Ok(handle) + } +} + +/// Get a handle to the Tokio runtime stored on the reactor instance, or create +/// a new one. +fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { + install_runtime(reactor)?; + } + + get_runtime(reactor) +} + +/// Install a new Tokio runtime on the reactor instance. +fn install_runtime(reactor: &Bound) -> PyResult<()> { + let py = reactor.py(); + let runtime = PyTokioRuntime { runtime: None }; + let runtime = runtime.into_pyobject(py)?; + + // Attach the runtime to the reactor, starting it when the reactor is + // running, stopping it when the reactor is shutting down + reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; + reactor.call_method1( + "addSystemEventTrigger", + ("after", "shutdown", runtime.getattr("shutdown")?), + )?; + reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; + + Ok(()) +} + +/// Get a reference to a Tokio runtime handle stored on the reactor instance. +fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is + // not a `Runtime`. Careful that this could happen if the user sets it + // manually, or if multiple versions of `pyo3-twisted` are used! + let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; + Ok(runtime.borrow()) +} + +/// A reference to the `twisted.internet.defer` module. +static DEFER: OnceCell = OnceCell::new(); + +/// Access to the `twisted.internet.defer` module. +fn defer(py: Python<'_>) -> PyResult<&Bound> { + Ok(DEFER + .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))? + .bind(py)) +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module: Bound<'_, PyModule> = PyModule::new(py, "http_client")?; + child_module.add_class::()?; + + // Make sure we fail early if we can't load some modules + defer(py)?; + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import http_client` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.http_client", child_module)?; + + Ok(()) +} + +#[pyclass] +struct HttpClient { + client: reqwest::Client, + reactor: PyObject, +} + +#[pymethods] +impl HttpClient { + #[new] + pub fn py_new(reactor: Bound, user_agent: &str) -> PyResult { + // Make sure the runtime gets installed + let _ = runtime(&reactor)?; + + Ok(HttpClient { + client: reqwest::Client::builder() + .user_agent(user_agent) + .build() + .context("building reqwest client")?, + reactor: reactor.unbind(), + }) + } + + pub fn get<'a>( + &self, + py: Python<'a>, + url: String, + response_limit: usize, + ) -> PyResult> { + self.send_request(py, self.client.get(url), response_limit) + } + + pub fn post<'a>( + &self, + py: Python<'a>, + url: String, + response_limit: usize, + headers: HashMap, + request_body: String, + ) -> PyResult> { + let mut builder = self.client.post(url); + for (name, value) in headers { + builder = builder.header(name, value); + } + builder = builder.body(request_body); + + self.send_request(py, builder, response_limit) + } +} + +impl HttpClient { + fn send_request<'a>( + &self, + py: Python<'a>, + builder: RequestBuilder, + response_limit: usize, + ) -> PyResult> { + create_deferred(py, self.reactor.bind(py), async move { + let response = builder.send().await.context("sending request")?; + + let status = response.status(); + + let mut stream = response.bytes_stream(); + let mut buffer = Vec::new(); + while let Some(chunk) = stream.try_next().await.context("reading body")? { + if buffer.len() + chunk.len() > response_limit { + Err(anyhow::anyhow!("Response size too large"))?; + } + + buffer.extend_from_slice(&chunk); + } + + if !status.is_success() { + return Err(HttpResponseException::new(status, buffer)); + } + + let r = Python::with_gil(|py| buffer.into_pyobject(py).map(|o| o.unbind()))?; + + Ok(r) + }) + } +} + +/// Creates a twisted deferred from the given future, spawning the task on the +/// tokio runtime. +/// +/// Does not handle deferred cancellation or contextvars. +fn create_deferred<'py, F, O>( + py: Python<'py>, + reactor: &Bound<'py, PyAny>, + fut: F, +) -> PyResult> +where + F: Future> + Send + 'static, + for<'a> O: IntoPyObject<'a> + Send + 'static, +{ + let deferred = defer(py)?.call_method0("Deferred")?; + let deferred_callback = deferred.getattr("callback")?.unbind(); + let deferred_errback = deferred.getattr("errback")?.unbind(); + + let rt = runtime(reactor)?; + let handle = rt.handle()?; + let task = handle.spawn(fut); + + // Unbind the reactor so that we can pass it to the task + let reactor = reactor.clone().unbind(); + handle.spawn(async move { + let res = task.await; + + Python::with_gil(move |py| { + // Flatten the panic into standard python error + let res = match res { + Ok(r) => r, + Err(join_err) => match join_err.try_into_panic() { + Ok(panic_err) => Err(RustPanicError::from_panic(&panic_err)), + Err(err) => Err(PyException::new_err(format!("Task cancelled: {err}"))), + }, + }; + + // Re-bind the reactor + let reactor = reactor.bind(py); + + // Send the result to the deferred, via `.callback(..)` or `.errback(..)` + match res { + Ok(obj) => { + reactor + .call_method("callFromThread", (deferred_callback, obj), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + Err(err) => { + reactor + .call_method("callFromThread", (deferred_errback, err), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + } + }); + }); + + Ok(deferred) +} diff --git a/rust/src/identifier.rs b/rust/src/identifier.rs index b199c5838e..03d1ebdc8a 100644 --- a/rust/src/identifier.rs +++ b/rust/src/identifier.rs @@ -27,7 +27,7 @@ pub enum IdentifierError { impl fmt::Display for IdentifierError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } @@ -71,6 +71,34 @@ impl TryFrom<&str> for UserID { } } +impl TryFrom for UserID { + type Error = IdentifierError; + + /// Will try creating a `UserID` from the provided `&str`. + /// Can fail if the user_id is incorrectly formatted. + fn try_from(s: String) -> Result { + if !s.starts_with('@') { + return Err(IdentifierError::IncorrectSigil); + } + + if s.find(':').is_none() { + return Err(IdentifierError::MissingColon); + } + + Ok(UserID(s)) + } +} + +impl<'de> serde::Deserialize<'de> for UserID { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s: String = serde::Deserialize::deserialize(deserializer)?; + UserID::try_from(s).map_err(serde::de::Error::custom) + } +} + impl Deref for UserID { type Target = str; @@ -84,3 +112,141 @@ impl fmt::Display for UserID { write!(f, "{}", self.0) } } + +/// A Matrix room_id. +#[derive(Clone, Debug, PartialEq)] +pub struct RoomID(String); + +impl RoomID { + /// Returns the `localpart` of the room_id. + pub fn localpart(&self) -> &str { + &self[1..self.colon_pos()] + } + + /// Returns the `server_name` / `domain` of the room_id. + pub fn server_name(&self) -> &str { + &self[self.colon_pos() + 1..] + } + + /// Returns the position of the ':' inside of the room_id. + /// Used when splitting the room_id into it's respective parts. + fn colon_pos(&self) -> usize { + self.find(':').unwrap() + } +} + +impl TryFrom<&str> for RoomID { + type Error = IdentifierError; + + /// Will try creating a `RoomID` from the provided `&str`. + /// Can fail if the room_id is incorrectly formatted. + fn try_from(s: &str) -> Result { + if !s.starts_with('!') { + return Err(IdentifierError::IncorrectSigil); + } + + if s.find(':').is_none() { + return Err(IdentifierError::MissingColon); + } + + Ok(RoomID(s.to_string())) + } +} + +impl TryFrom for RoomID { + type Error = IdentifierError; + + /// Will try creating a `RoomID` from the provided `String`. + /// Can fail if the room_id is incorrectly formatted. + fn try_from(s: String) -> Result { + if !s.starts_with('!') { + return Err(IdentifierError::IncorrectSigil); + } + + if s.find(':').is_none() { + return Err(IdentifierError::MissingColon); + } + + Ok(RoomID(s)) + } +} + +impl<'de> serde::Deserialize<'de> for RoomID { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s: String = serde::Deserialize::deserialize(deserializer)?; + RoomID::try_from(s).map_err(serde::de::Error::custom) + } +} + +impl Deref for RoomID { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Display for RoomID { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// A Matrix event_id. +#[derive(Clone, Debug, PartialEq)] +pub struct EventID(String); + +impl TryFrom<&str> for EventID { + type Error = IdentifierError; + + /// Will try creating a `EventID` from the provided `&str`. + /// Can fail if the event_id is incorrectly formatted. + fn try_from(s: &str) -> Result { + if !s.starts_with('$') { + return Err(IdentifierError::IncorrectSigil); + } + + Ok(EventID(s.to_string())) + } +} + +impl TryFrom for EventID { + type Error = IdentifierError; + + /// Will try creating a `EventID` from the provided `String`. + /// Can fail if the event_id is incorrectly formatted. + fn try_from(s: String) -> Result { + if !s.starts_with('$') { + return Err(IdentifierError::IncorrectSigil); + } + + Ok(EventID(s)) + } +} + +impl<'de> serde::Deserialize<'de> for EventID { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s: String = serde::Deserialize::deserialize(deserializer)?; + EventID::try_from(s).map_err(serde::de::Error::custom) + } +} + +impl Deref for EventID { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Display for EventID { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 5de9238326..6522148fa1 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,3 +1,5 @@ +use std::convert::Infallible; + use lazy_static::lazy_static; use pyo3::prelude::*; use pyo3_log::ResetHandle; @@ -6,10 +8,12 @@ pub mod acl; pub mod errors; pub mod events; pub mod http; +pub mod http_client; pub mod identifier; pub mod matrix_const; pub mod push; pub mod rendezvous; +pub mod segmenter; lazy_static! { static ref LOGGING_HANDLE: ResetHandle = pyo3_log::init(); @@ -48,7 +52,22 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { acl::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; + http_client::register_module(py, m)?; rendezvous::register_module(py, m)?; + segmenter::register_module(py, m)?; Ok(()) } + +pub trait UnwrapInfallible { + fn unwrap_infallible(self) -> T; +} + +impl UnwrapInfallible for Result { + fn unwrap_infallible(self) -> T { + match self { + Ok(val) => val, + Err(never) => match never {}, + } + } +} diff --git a/rust/src/push/base_rules.rs b/rust/src/push/base_rules.rs index e0832ada1c..47d5289006 100644 --- a/rust/src/push/base_rules.rs +++ b/rust/src/push/base_rules.rs @@ -289,6 +289,29 @@ pub const BASE_APPEND_CONTENT_RULES: &[PushRule] = &[PushRule { default_enabled: true, }]; +pub const BASE_APPEND_POSTCONTENT_RULES: &[PushRule] = &[ + PushRule { + rule_id: Cow::Borrowed("global/postcontent/.io.element.msc4306.rule.unsubscribed_thread"), + priority_class: 6, + conditions: Cow::Borrowed(&[Condition::Known( + KnownCondition::Msc4306ThreadSubscription { subscribed: false }, + )]), + actions: Cow::Borrowed(&[]), + default: true, + default_enabled: true, + }, + PushRule { + rule_id: Cow::Borrowed("global/postcontent/.io.element.msc4306.rule.subscribed_thread"), + priority_class: 6, + conditions: Cow::Borrowed(&[Condition::Known( + KnownCondition::Msc4306ThreadSubscription { subscribed: true }, + )]), + actions: Cow::Borrowed(&[Action::Notify, SOUND_ACTION]), + default: true, + default_enabled: true, + }, +]; + pub const BASE_APPEND_UNDERRIDE_RULES: &[PushRule] = &[ PushRule { rule_id: Cow::Borrowed("global/underride/.m.rule.call"), @@ -706,6 +729,7 @@ lazy_static! { .iter() .chain(BASE_APPEND_OVERRIDE_RULES.iter()) .chain(BASE_APPEND_CONTENT_RULES.iter()) + .chain(BASE_APPEND_POSTCONTENT_RULES.iter()) .chain(BASE_APPEND_UNDERRIDE_RULES.iter()) .map(|rule| { (&*rule.rule_id, rule) }) .collect(); diff --git a/rust/src/push/evaluator.rs b/rust/src/push/evaluator.rs index 0d436a1d7b..1cbca4c635 100644 --- a/rust/src/push/evaluator.rs +++ b/rust/src/push/evaluator.rs @@ -106,8 +106,11 @@ pub struct PushRuleEvaluator { /// flag as MSC1767 (extensible events core). msc3931_enabled: bool, - // If MSC4210 (remove legacy mentions) is enabled. + /// If MSC4210 (remove legacy mentions) is enabled. msc4210_enabled: bool, + + /// If MSC4306 (thread subscriptions) is enabled. + msc4306_enabled: bool, } #[pymethods] @@ -126,6 +129,7 @@ impl PushRuleEvaluator { room_version_feature_flags, msc3931_enabled, msc4210_enabled, + msc4306_enabled, ))] pub fn py_new( flattened_keys: BTreeMap, @@ -138,6 +142,7 @@ impl PushRuleEvaluator { room_version_feature_flags: Vec, msc3931_enabled: bool, msc4210_enabled: bool, + msc4306_enabled: bool, ) -> Result { let body = match flattened_keys.get("content.body") { Some(JsonValue::Value(SimpleJsonValue::Str(s))) => s.clone().into_owned(), @@ -156,6 +161,7 @@ impl PushRuleEvaluator { room_version_feature_flags, msc3931_enabled, msc4210_enabled, + msc4306_enabled, }) } @@ -167,11 +173,19 @@ impl PushRuleEvaluator { /// /// Returns the set of actions, if any, that match (filtering out any /// `dont_notify` and `coalesce` actions). + /// + /// msc4306_thread_subscription_state: (Only populated if MSC4306 is enabled) + /// The thread subscription state corresponding to the thread containing this event. + /// - `None` if the event is not in a thread, or if MSC4306 is disabled. + /// - `Some(true)` if the event is in a thread and the user has a subscription for that thread + /// - `Some(false)` if the event is in a thread and the user does NOT have a subscription for that thread + #[pyo3(signature = (push_rules, user_id=None, display_name=None, msc4306_thread_subscription_state=None))] pub fn run( &self, push_rules: &FilteredPushRules, user_id: Option<&str>, display_name: Option<&str>, + msc4306_thread_subscription_state: Option, ) -> Vec { 'outer: for (push_rule, enabled) in push_rules.iter() { if !enabled { @@ -203,7 +217,12 @@ impl PushRuleEvaluator { Condition::Known(KnownCondition::RoomVersionSupports { feature: _ }), ); - match self.match_condition(condition, user_id, display_name) { + match self.match_condition( + condition, + user_id, + display_name, + msc4306_thread_subscription_state, + ) { Ok(true) => {} Ok(false) => continue 'outer, Err(err) => { @@ -236,13 +255,20 @@ impl PushRuleEvaluator { } /// Check if the given condition matches. + #[pyo3(signature = (condition, user_id=None, display_name=None, msc4306_thread_subscription_state=None))] fn matches( &self, condition: Condition, user_id: Option<&str>, display_name: Option<&str>, + msc4306_thread_subscription_state: Option, ) -> bool { - match self.match_condition(&condition, user_id, display_name) { + match self.match_condition( + &condition, + user_id, + display_name, + msc4306_thread_subscription_state, + ) { Ok(true) => true, Ok(false) => false, Err(err) => { @@ -260,6 +286,7 @@ impl PushRuleEvaluator { condition: &Condition, user_id: Option<&str>, display_name: Option<&str>, + msc4306_thread_subscription_state: Option, ) -> Result { let known_condition = match condition { Condition::Known(known) => known, @@ -391,6 +418,13 @@ impl PushRuleEvaluator { && self.room_version_feature_flags.contains(&flag) } } + KnownCondition::Msc4306ThreadSubscription { subscribed } => { + if !self.msc4306_enabled { + false + } else { + msc4306_thread_subscription_state == Some(*subscribed) + } + } }; Ok(result) @@ -534,10 +568,11 @@ fn push_rule_evaluator() { vec![], true, false, + false, ) .unwrap(); - let result = evaluator.run(&FilteredPushRules::default(), None, Some("bob")); + let result = evaluator.run(&FilteredPushRules::default(), None, Some("bob"), None); assert_eq!(result.len(), 3); } @@ -564,6 +599,7 @@ fn test_requires_room_version_supports_condition() { flags, true, false, + false, ) .unwrap(); @@ -573,6 +609,7 @@ fn test_requires_room_version_supports_condition() { &FilteredPushRules::default(), Some("@bob:example.org"), None, + None, ); assert_eq!(result.len(), 3); @@ -591,7 +628,17 @@ fn test_requires_room_version_supports_condition() { }; let rules = PushRules::new(vec![custom_rule]); result = evaluator.run( - &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false, false), + &FilteredPushRules::py_new( + rules, + BTreeMap::new(), + true, + false, + true, + false, + false, + false, + ), + None, None, None, ); diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs index ef8ed150d4..b0cedd758c 100644 --- a/rust/src/push/mod.rs +++ b/rust/src/push/mod.rs @@ -65,8 +65,8 @@ use anyhow::{Context, Error}; use log::warn; use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; -use pyo3::types::{PyBool, PyList, PyLong, PyString}; -use pythonize::{depythonize_bound, pythonize}; +use pyo3::types::{PyBool, PyInt, PyList, PyString}; +use pythonize::{depythonize, pythonize, PythonizeError}; use serde::de::Error as _; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -79,7 +79,7 @@ pub mod utils; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let child_module = PyModule::new_bound(py, "push")?; + let child_module = PyModule::new(py, "push")?; child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; @@ -90,7 +90,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> // We need to manually add the module to sys.modules to make `from // synapse.synapse_rust import push` work. - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.push", child_module)?; @@ -182,12 +182,16 @@ pub enum Action { Unknown(Value), } -impl IntoPy for Action { - fn into_py(self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for Action { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PythonizeError; + + fn into_pyobject(self, py: Python<'py>) -> Result { // When we pass the `Action` struct to Python we want it to be converted // to a dict. We use `pythonize`, which converts the struct using the // `serde` serialization. - pythonize(py, &self).expect("valid action") + pythonize(py, &self) } } @@ -270,13 +274,13 @@ pub enum SimpleJsonValue { } impl<'source> FromPyObject<'source> for SimpleJsonValue { - fn extract(ob: &'source PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { if let Ok(s) = ob.downcast::() { Ok(SimpleJsonValue::Str(Cow::Owned(s.to_string()))) // A bool *is* an int, ensure we try bool first. } else if let Ok(b) = ob.downcast::() { Ok(SimpleJsonValue::Bool(b.extract()?)) - } else if let Ok(i) = ob.downcast::() { + } else if let Ok(i) = ob.downcast::() { Ok(SimpleJsonValue::Int(i.extract()?)) } else if ob.is_none() { Ok(SimpleJsonValue::Null) @@ -298,15 +302,19 @@ pub enum JsonValue { } impl<'source> FromPyObject<'source> for JsonValue { - fn extract(ob: &'source PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { if let Ok(l) = ob.downcast::() { - match l.iter().map(SimpleJsonValue::extract).collect() { + match l + .iter() + .map(|it| SimpleJsonValue::extract_bound(&it)) + .collect() + { Ok(a) => Ok(JsonValue::Array(a)), Err(e) => Err(PyTypeError::new_err(format!( "Can't convert to JsonValue::Array: {e}" ))), } - } else if let Ok(v) = SimpleJsonValue::extract(ob) { + } else if let Ok(v) = SimpleJsonValue::extract_bound(ob) { Ok(JsonValue::Value(v)) } else { Err(PyTypeError::new_err(format!( @@ -361,17 +369,25 @@ pub enum KnownCondition { RoomVersionSupports { feature: Cow<'static, str>, }, + #[serde(rename = "io.element.msc4306.thread_subscription")] + Msc4306ThreadSubscription { + subscribed: bool, + }, } -impl IntoPy for Condition { - fn into_py(self, py: Python<'_>) -> PyObject { - pythonize(py, &self).expect("valid condition") +impl<'source> IntoPyObject<'source> for Condition { + type Target = PyAny; + type Output = Bound<'source, Self::Target>; + type Error = PythonizeError; + + fn into_pyobject(self, py: Python<'source>) -> Result { + pythonize(py, &self) } } impl<'source> FromPyObject<'source> for Condition { fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { - Ok(depythonize_bound(ob.clone())?) + Ok(depythonize(ob)?) } } @@ -511,6 +527,7 @@ impl PushRules { .chain(base_rules::BASE_APPEND_OVERRIDE_RULES.iter()) .chain(self.content.iter()) .chain(base_rules::BASE_APPEND_CONTENT_RULES.iter()) + .chain(base_rules::BASE_APPEND_POSTCONTENT_RULES.iter()) .chain(self.room.iter()) .chain(self.sender.iter()) .chain(self.underride.iter()) @@ -535,11 +552,13 @@ pub struct FilteredPushRules { msc3664_enabled: bool, msc4028_push_encrypted_events: bool, msc4210_enabled: bool, + msc4306_enabled: bool, } #[pymethods] impl FilteredPushRules { #[new] + #[allow(clippy::too_many_arguments)] pub fn py_new( push_rules: PushRules, enabled_map: BTreeMap, @@ -548,6 +567,7 @@ impl FilteredPushRules { msc3664_enabled: bool, msc4028_push_encrypted_events: bool, msc4210_enabled: bool, + msc4306_enabled: bool, ) -> Self { Self { push_rules, @@ -557,6 +577,7 @@ impl FilteredPushRules { msc3664_enabled, msc4028_push_encrypted_events, msc4210_enabled, + msc4306_enabled, } } @@ -607,6 +628,10 @@ impl FilteredPushRules { return false; } + if !self.msc4306_enabled && rule.rule_id.contains("/.io.element.msc4306.rule.") { + return false; + } + true }) .map(|r| { diff --git a/rust/src/rendezvous/mod.rs b/rust/src/rendezvous/mod.rs index f69f45490f..3148e0f67a 100644 --- a/rust/src/rendezvous/mod.rs +++ b/rust/src/rendezvous/mod.rs @@ -29,7 +29,7 @@ use pyo3::{ exceptions::PyValueError, pyclass, pymethods, types::{PyAnyMethods, PyModule, PyModuleMethods}, - Bound, Py, PyAny, PyObject, PyResult, Python, ToPyObject, + Bound, IntoPyObject, Py, PyAny, PyObject, PyResult, Python, }; use ulid::Ulid; @@ -37,6 +37,7 @@ use self::session::Session; use crate::{ errors::{NotFoundError, SynapseError}, http::{http_request_from_twisted, http_response_to_twisted, HeaderMapPyExt}, + UnwrapInfallible, }; mod session; @@ -46,7 +47,7 @@ fn prepare_headers(headers: &mut HeaderMap, session: &Session) { headers.typed_insert(AccessControlAllowOrigin::ANY); headers.typed_insert(AccessControlExposeHeaders::from_iter([ETAG])); headers.typed_insert(Pragma::no_cache()); - headers.typed_insert(CacheControl::new().with_no_store()); + headers.typed_insert(CacheControl::new().with_no_store().with_no_transform()); headers.typed_insert(session.etag()); headers.typed_insert(session.expires()); headers.typed_insert(session.last_modified()); @@ -125,7 +126,11 @@ impl RendezvousHandler { let base = Uri::try_from(format!("{base}_synapse/client/rendezvous")) .map_err(|_| PyValueError::new_err("Invalid base URI"))?; - let clock = homeserver.call_method0("get_clock")?.to_object(py); + let clock = homeserver + .call_method0("get_clock")? + .into_pyobject(py) + .unwrap_infallible() + .unbind(); // Construct a Python object so that we can get a reference to the // evict method and schedule it to run. @@ -187,10 +192,12 @@ impl RendezvousHandler { "url": uri, }) .to_string(); + let length = response.len() as _; let mut response = Response::new(response.as_bytes()); *response.status_mut() = StatusCode::CREATED; response.headers_mut().typed_insert(ContentType::json()); + response.headers_mut().typed_insert(ContentLength(length)); prepare_headers(response.headers_mut(), &session); http_response_to_twisted(twisted_request, response)?; @@ -288,6 +295,14 @@ impl RendezvousHandler { let mut response = Response::new(Bytes::new()); *response.status_mut() = StatusCode::ACCEPTED; prepare_headers(response.headers_mut(), session); + + // Even though this isn't mandated by the MSC, we set a Content-Type on the response. It + // doesn't do any harm as the body is empty, but this helps escape a bug in some reverse + // proxy/cache setup which strips the ETag header if there is no Content-Type set. + // Specifically, we noticed this behaviour when placing Synapse behind Cloudflare. + response.headers_mut().typed_insert(ContentType::text()); + response.headers_mut().typed_insert(ContentLength(0)); + http_response_to_twisted(twisted_request, response)?; Ok(()) @@ -304,6 +319,7 @@ impl RendezvousHandler { response .headers_mut() .typed_insert(AccessControlAllowOrigin::ANY); + response.headers_mut().typed_insert(ContentLength(0)); http_response_to_twisted(twisted_request, response)?; Ok(()) @@ -311,7 +327,7 @@ impl RendezvousHandler { } pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let child_module = PyModule::new_bound(py, "rendezvous")?; + let child_module = PyModule::new(py, "rendezvous")?; child_module.add_class::()?; @@ -319,7 +335,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> // We need to manually add the module to sys.modules to make `from // synapse.synapse_rust import rendezvous` work. - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.rendezvous", child_module)?; diff --git a/rust/src/segmenter.rs b/rust/src/segmenter.rs new file mode 100644 index 0000000000..135b3c1779 --- /dev/null +++ b/rust/src/segmenter.rs @@ -0,0 +1,33 @@ +use icu_segmenter::options::WordBreakInvariantOptions; +use icu_segmenter::WordSegmenter; +use pyo3::prelude::*; + +#[pyfunction] +pub fn parse_words(text: &str) -> PyResult> { + let segmenter = WordSegmenter::new_auto(WordBreakInvariantOptions::default()); + let mut parts = Vec::new(); + let mut last = 0usize; + + // `segment_str` gives us word boundaries as a vector of indexes. Use that + // to build a vector of words, and return. + for boundary in segmenter.segment_str(text) { + if boundary > last { + parts.push(text[last..boundary].to_string()); + } + last = boundary; + } + Ok(parts) +} + +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "segmenter")?; + child_module.add_function(wrap_pyfunction!(parse_words, m)?)?; + + m.add_submodule(&child_module)?; + + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.segmenter", child_module)?; + + Ok(()) +} diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml new file mode 100644 index 0000000000..fdce4219ae --- /dev/null +++ b/schema/synapse-config.schema.yaml @@ -0,0 +1,5891 @@ +$schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.138/synapse-config.schema.json +type: object +properties: + modules: + type: array + description: >- + Use the `module` sub-option to add modules under this option to extend + functionality. The `module` setting then has a sub-option, `config`, which + can be used to define some configuration for the `module`. + items: + type: object + properties: + module: + type: string + description: Path to the Python class of the module. + config: + type: object + description: Configuration options for the module. + default: [] + examples: + - - module: my_super_module.MySuperClass + config: + do_thing: true + - module: my_other_super_module.SomeClass + config: {} + server_name: + type: string + description: >- + This sets the public-facing domain of the server. + + + The `server_name` name will appear at the end of usernames and room + addresses created on your server. For example if the `server_name` was + example.com, usernames on your server would be in the format + `@user:example.com`. + + + In most cases you should avoid using a matrix specific subdomain such as + matrix.example.com or synapse.example.com as the `server_name` for the + same reasons you wouldn't use user@email.example.com as your email + address. See [here](../../delegate.md) for information on how to host + Synapse on a subdomain while preserving a clean `server_name`. + + + The `server_name` cannot be changed later so it is important to configure + this correctly before you start Synapse. It should be all lowercase and + may contain an explicit port. + examples: + - matrix.org + - localhost:8080 + pid_file: + type: ["string", "null"] + description: When running Synapse as a daemon, the file to store the pid in. + default: null + examples: + - DATADIR/homeserver.pid + daemonize: + type: boolean + description: >- + Specifies whether Synapse should be started as a daemon process. If + Synapse is being managed by [systemd](../../systemd-with-workers/), this + option must be omitted or set to `false`. + + + This can also be set by the `--daemonize` (`-D`) argument when starting + Synapse. + + + See `worker_daemonize` for more information on daemonizing workers. + default: false + examples: + - true + print_pidfile: + type: boolean + description: >- + Print the path to the pidfile just before daemonizing. + + + This can also be set by the `--print-pidfile` argument when starting Synapse. + default: false + examples: + - true + user_agent_suffix: + type: ["string", "null"] + description: >- + A suffix that is appended to the Synapse user-agent (ex. `Synapse/v1.123.0`). + default: null + examples: + - " (I'm a teapot; Linux x86_64)" + use_frozen_dicts: + type: boolean + description: >- + Determines whether we should freeze the internal dict object in + `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. + signature dicts. However, freezing a dict is expensive. + default: false + examples: + - true + web_client_location: + type: ["string", "null"] + description: The absolute URL to the web client which `/` will redirect to. + default: null + examples: + - "https://riot.example.com/" + public_baseurl: + type: ["string", "null"] + description: >- + The public-facing base URL that clients use to access this Homeserver (not + including _matrix/...). This is the same URL a user might enter into the + "Custom Homeserver URL" field on their client. If you use Synapse with a + reverse proxy, this should be the URL to reach Synapse via the proxy. + Otherwise, it should be the URL to reach Synapse's client HTTP listener + (see [`listeners`](#listeners) below). + + + If unset or null, `https:///` is used. + default: null + examples: + - "https://example.com/" + serve_server_wellknown: + type: boolean + description: >- + By default, other servers will try to reach our server on port 8448, which + can be inconvenient in some environments. + + + Provided `https:///` on port 443 is routed to Synapse, this + option configures Synapse to serve a file at + `https:///.well-known/matrix/server`. This will tell other + servers to send traffic to port 443 instead. + + + This option currently defaults to false. + + + See [Delegation of incoming federation traffic](../../delegate.md) for + more information. + default: false + examples: + - true + extra_well_known_client_content: + type: object + description: >- + This option allows server runners to add arbitrary key-value pairs to the + [client-facing `.well-known` + response](https://spec.matrix.org/latest/client-server-api/#well-known-uri). + Note that the `public_baseurl` config option must be provided for Synapse + to serve a response to `/.well-known/matrix/client` at all. + + + If this option is provided, it parses the given yaml to json and serves it + on `/.well-known/matrix/client` endpoint alongside the standard + properties. + + + *Added in Synapse 1.62.0.* + examples: + - option1: value1 + option2: value2 + soft_file_limit: + type: integer + description: >- + Set the soft limit on the number of file descriptors synapse can use. Zero + is used to indicate synapse should set the soft limit to the hard limit. + default: 0 + examples: + - 3 + presence: + type: object + description: >- + Presence tracking allows users to see the state (e.g online/offline) of + other local and remote users. This option replaces the previous top-level + `use_presence` option. + properties: + enabled: + type: ["boolean", "string"] + description: >- + Set to false to disable presence tracking on this homeserver. + + + Can also be set to a special value of "untracked" which ignores + updates received via clients and federation, while still accepting + updates from the [module API](../../modules/index.md). + + + *The "untracked" option was added in Synapse 1.96.0.* + oneOf: + - type: boolean + - type: string + const: untracked + default: true + include_offline_users_on_sync: + type: boolean + description: >- + When clients perform an initial or `full_state` sync, presence results + for offline users are not included by default. Setting + `include_offline_users_on_sync` to `true` will always include offline + users in the results. + default: false + examples: + - enabled: false + include_offline_users_on_sync: false + require_auth_for_profile_requests: + type: boolean + description: >- + Whether to require authentication to retrieve profile data (avatars, + display names) of other users through the client API. Note that profile + data is also available via the federation API, unless + `allow_profile_lookup_over_federation` is set to false. + default: false + examples: + - true + limit_profile_requests_to_users_who_share_rooms: + type: boolean + description: >- + Use this option to require a user to share a room with another user in + order to retrieve their profile information. Only checked on Client-Server + requests. Profile requests from other servers should be checked by the + requesting server. + default: false + examples: + - true + include_profile_data_on_invite: + type: boolean + description: >- + Use this option to prevent a user's profile data from being retrieved and + displayed in a room until they have joined it. By default, a user's + profile data is included in an invite event, regardless of the values of + the above two settings, and whether or not the users share a server. + default: true + examples: + - false + allow_public_rooms_without_auth: + type: boolean + description: + If set to true, removes the need for authentication to access the server's + public rooms directory through the client API, meaning that anyone can + query the room directory. + default: false + examples: + - true + allow_public_rooms_over_federation: + type: boolean + description: >- + If set to true, allows any other homeserver to fetch the server's public + rooms directory via federation. + default: false + examples: + - true + default_room_version: + type: string + description: >- + The default room version for newly created rooms on this server. + + + Known room versions are listed + [here](https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions) + + + For example, for room version 1, `default_room_version` should be set to + "1". + + + _Changed in Synapse 1.76:_ the default version room version was increased + from [9](https://spec.matrix.org/v1.5/rooms/v9/) to + [10](https://spec.matrix.org/v1.5/rooms/v10/). + default: "10" + examples: + - "8" + gc_thresholds: + type: ["array", "null"] + description: >- + The garbage collection threshold parameters to pass to `gc.set_threshold`, + if defined. + default: null + examples: + - - 700 + - 10 + - 10 + gc_min_interval: + type: array + description: >- + The minimum time in seconds between each GC for a generation, regardless + of the GC thresholds. This ensures that we don't do GC too frequently. A + value of `[1s, 10s, 30s]` indicates that a second must pass between + consecutive generation 0 GCs, etc. + default: + - 1s + - 10s + - 30s + examples: + - - 0.5s + - 30s + - 1m + filter_timeline_limit: + type: integer + description: >- + Set the limit on the returned events in the timeline in the get and sync + operations. A value of -1 means no upper limit. + default: 100 + examples: + - 5000 + block_non_admin_invites: + type: boolean + description: >- + Whether room invites to users on this server should be blocked (except + those sent by local server admins). + default: false + examples: + - true + enable_search: + type: boolean + description: >- + If set to false, new messages will not be indexed for searching and users + will receive errors when searching for messages. + default: true + examples: + - false + ip_range_blacklist: + type: array + description: >- + This option prevents outgoing requests from being sent to the specified + blacklisted IP address CIDR ranges. If this option is not specified then + it defaults to private IP address ranges (see the example below). + + + The blacklist applies to the outbound requests for federation, identity + servers, push servers, and for checking key validity for third-party + invite events. + + + (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly + listed here, since they correspond to unroutable addresses.) + + + This option replaces `federation_ip_range_blacklist` in Synapse v1.25.0. + + + Note: The value is ignored when an HTTP proxy is in use. + items: + type: string + default: + - 127.0.0.0/8 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 100.64.0.0/10 + - 192.0.0.0/24 + - 169.254.0.0/16 + - 192.88.99.0/24 + - 198.18.0.0/15 + - 192.0.2.0/24 + - 198.51.100.0/24 + - 203.0.113.0/24 + - 224.0.0.0/4 + - "::1/128" + - "fe80::/10" + - "fc00::/7" + - "2001:db8::/32" + - "ff00::/8" + - "fec0::/10" + ip_range_whitelist: + type: array + description: >- + List of IP address CIDR ranges that should be allowed for federation, + identity servers, push servers, and for checking key validity for + third-party invite events. This is useful for specifying exceptions to + wide-ranging blacklisted target IP ranges – e.g. for communication with a + push server only visible in your network. + + + This whitelist overrides `ip_range_blacklist`. + items: + type: string + default: [] + examples: + - - 192.168.1.1 + listeners: + type: array + description: >- + List of ports that Synapse should listen on, their purpose and their + configuration. + + + Valid resource names are: + + + * `client`: the client-server API (/_matrix/client). Also implies `media` + and `static`. If configuring the main process, the Synapse Admin API + (/_synapse/admin) is also implied. + + + * `consent`: user consent forms (/_matrix/consent). See + [here](../../consent_tracking.md) for more. + + + * `federation`: the server-server API (/_matrix/federation). Also implies + `media`, `keys`, `openid` + + + * `keys`: the key discovery API (/_matrix/key). + + + * `media`: the media API (/_matrix/media). + + + * `metrics`: the metrics interface. See [here](../../metrics-howto.md). + (Not compatible with Unix sockets) + + + * `openid`: OpenID authentication. See [here](../../openid.md). + + + * `replication`: the HTTP replication API (/_synapse/replication). See + [here](../../workers.md). + + + * `static`: static resources under synapse/static (/_matrix/static). + (Mostly useful for "fallback authentication".) + + + * `health`: the [health check + endpoint](../../reverse_proxy.md#health-check-endpoint). This endpoint is + by default active for all other resources and does not have to be + activated separately. This is only useful if you want to use the health + endpoint explicitly on a dedicated port or for [workers](../../workers.md) + and containers without listener e.g. [application + services](../../workers.md#notifying-application-services). + items: + type: object + properties: + port: + type: integer + description: The TCP port to bind to. + tag: + type: ["string", "null"] + description: >- + An alias for the port in the logger name. If set the tag is logged + instead of the port. Default to `None`, is optional and only valid + for listener with `type: http`. See the docs [request log + format](../administration/request_log.md). + bind_addresses: + type: ["array", "null"] + description: >- + A list of local addresses to listen on. The default is "all local + interfaces". + items: + type: string + type: + type: string + description: >- + The type of listener. Normally `http`, but other valid options are + [`manhole`](../../manhole.md) and + [`metrics`](../../metrics-howto.md). + enum: + - http + - manhole + - metrics + tls: + type: boolean + description: >- + Set to true to enable TLS for this listener. Will use the TLS + key/cert specified in tls_private_key_path/tls_certificate_path. + x_forwarded: + type: boolean + description: >- + Only valid for an `http` listener. Set to true to use the + X-Forwarded-For header as the client IP. Useful when Synapse is + behind a [reverse-proxy](../../reverse_proxy.md). + request_id_header: + type: ["string", "null"] + description: >- + The header extracted from each incoming request that is used as the + basis for the request ID. The request ID is used in + [logs](../administration/request_log.md#request-log-format) and + tracing to correlate and match up requests. When unset, Synapse will + automatically generate sequential request IDs. This option is useful + when Synapse is behind a [reverse-proxy](../../reverse_proxy.md). + + + _Added in Synapse 1.68.0._ + resources: + type: array + description: >- + Only valid for an `http` listener. A list of resources to host on this port. + items: + type: object + properties: + names: + type: array + description: >- + A list of names of HTTP resources. See below for a list of + valid resource names. + items: + type: string + enum: + - client + - consent + - federation + - keys + - media + - metrics + - openid + - replication + - static + - health + compress: + type: boolean + description: >- + Set to true to enable gzip compression on HTTP bodies for this + resource. This is currently only supported with the `client`, + `consent`, `metrics` and `federation` resources. + additional_resources: + type: object + description: >- + Only valid for an `http` listener. A map of additional endpoints + which should be loaded via dynamic modules. + additionalProperties: + type: object + properties: + module: + type: string + config: + type: object + path: + type: string + description: >- + A path and filename for a Unix socket. Make sure it is located in a + directory with read and write permissions, and that it already + exists (the directory will not be created). Defaults to `None`. + + * **Note**: The use of both `path` and `port` options for the same + `listener` is not compatible. + + * The `x_forwarded` option defaults to true when using Unix sockets + and can be omitted. + + * Other options that would not make sense to use with a UNIX socket, + such as `bind_addresses` and `tls` will be ignored and can be + removed. + + + _Added in Synapse 1.89.0_: Unix socket support + mode: + type: ["integer", "null"] + description: >- + The file permissions to set on the UNIX socket. Defaults to `666` if + unset or null. + + + **Note:** Must be set as `type: http` (does not support `metrics` + and `manhole`). Also make sure that `metrics` is not included in + `resources` -> `names` + + + _Added in Synapse 1.89.0_: Unix socket support + default: [] + examples: + - - port: 8448 + type: http + tls: true + resources: + - names: + - client + - federation + - - port: 8008 + tls: false + type: http + x_forwarded: true + bind_addresses: + - "::1" + - 127.0.0.1 + resources: + - names: + - client + - federation + compress: false + additional_resources: + /_matrix/my/custom/endpoint: + module: my_module.CustomRequestHandler + config: {} + - port: 9000 + bind_addresses: + - "::1" + - 127.0.0.1 + type: manhole + - - path: /run/synapse/main_public.sock + type: http + resources: + - names: + - client + - federation + manhole: + type: ["integer", "null"] + description: >- + Turn on the Twisted telnet manhole service on the given port. + + + This can also be set by the `--manhole` argument when starting Synapse. + default: null + examples: + - 1234 + manhole_settings: + type: object + description: >- + Connection settings for the manhole. You can find more information on the + manhole [here](../../manhole.md). + properties: + username: + type: ["string", "null"] + description: The username for the manhole. This defaults to "matrix". + password: + type: ["string", "null"] + description: The password for the manhole. This defaults to "rabbithole". + ssh_priv_key_path: + type: ["string", "null"] + description: >- + The private SSH key used to encrypt the manhole traffic. If left + unset, then hardcoded and non-secret keys are used, which could allow + traffic to be intercepted if sent over a public network. + ssh_pub_key_path: + type: ["string", "null"] + description: >- + The public SSH key corresponsing to `ssh_priv_key_path`. If left + unset, a hardcoded key is used. + examples: + - username: manhole + password: mypassword + ssh_priv_key_path: CONFDIR/id_rsa + ssh_pub_key_path: CONFDIR/id_rsa.pub + http_proxy: + type: ["string", "null"] + description: >- + Proxy server to use for HTTP requests. + + For more details, see the [forward proxy documentation](../../setup/forward_proxy.md). + examples: + - "http://USERNAME:PASSWORD@10.0.1.1:8080/" + https_proxy: + type: ["string", "null"] + description: >- + Proxy server to use for HTTPS requests. + + For more details, see the [forward proxy documentation](../../setup/forward_proxy.md). + examples: + - "http://USERNAME:PASSWORD@proxy.example.com:8080/" + no_proxy_hosts: + type: array + description: >- + List of hosts, IP addresses, or IP ranges in CIDR format which should not use the + proxy. Synapse will directly connect to these hosts. + + For more details, see the [forward proxy documentation](../../setup/forward_proxy.md). + examples: + - - master.hostname.example.com + - 10.1.0.0/16 + - 172.30.0.0/16 + matrix_authentication_service: + type: object + description: >- + The `matrix_authentication_service` setting configures integration with + [Matrix Authentication Service (MAS)](https://github.com/element-hq/matrix-authentication-service). + properties: + enabled: + type: boolean + description: >- + Whether or not to enable the MAS integration. If this is set to + `false`, Synapse will use its legacy internal authentication API. + default: false + + endpoint: + type: string + format: uri + description: >- + The URL where Synapse can reach MAS. This *must* have the `discovery` + and `oauth` resources mounted. + default: http://localhost:8080 + + secret: + type: ["string", "null"] + description: >- + A shared secret that will be used to authenticate requests from and to MAS. + + secret_path: + type: ["string", "null"] + description: >- + Alternative to `secret`, reading the shared secret from a file. + The file should be a plain text file, containing only the secret. + Synapse reads the secret from the given file once at startup. + + examples: + - enabled: true + secret: someverysecuresecret + endpoint: http://localhost:8080 + dummy_events_threshold: + type: integer + description: >- + Forward extremities can build up in a room due to networking delays + between homeservers. Once this happens in a large room, calculation of the + state of that room can become quite expensive. To mitigate this, once the + number of forward extremities reaches a given threshold, Synapse will send + an `org.matrix.dummy_event` event, which will reduce the forward + extremities in the room. + + + This setting defines the threshold (i.e. number of forward extremities in + the room) at which dummy events are sent. + default: 10 + examples: + - 5 + delete_stale_devices_after: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + An optional duration. If set, Synapse will run a daily background task to + log out and delete any device that hasn't been accessed for more than the + specified amount of time. + + + A value of null means devices are never pruned. + + + **Note:** This task will always run on the main process, regardless of the + value of `run_background_tasks_on`. This is due to workers currently not + having the ability to delete devices. + default: null + examples: + - 1y + email: + type: object + description: >- + Configuration for sending emails from Synapse. + + + Server admins can configure custom templates for email content. See + [here](../../templates.md) for more information. + properties: + smtp_host: + type: string + description: The hostname of the outgoing SMTP server to use. + default: localhost + smtp_port: + type: ["string", "null"] + description: >- + The port on the mail server for outgoing SMTP. If null or unset, 465 + is used if `force_tls` is true, else 25. + + + _Changed in Synapse 1.64.0:_ the default port is now aware of + `force_tls`. + default: null + smtp_user: + type: ["string", "null"] + description: Username for authentication to the SMTP server. + default: null + smtp_pass: + type: ["string", "null"] + description: Password for authentication to the SMTP server. + default: null + force_tls: + type: boolean + description: >- + By default, Synapse connects over plain text and then optionally + upgrades to TLS via STARTTLS. If this option is set to true, TLS is + used from the start (Implicit TLS), and the option + `require_transport_security` is ignored. It is recommended to enable + this if supported by your mail server. + + + _New in Synapse 1.64.0._ + default: false + require_transport_security: + type: boolean + description: >- + Set to true to require TLS transport security for SMTP. By default, + Synapse will connect over plain text, and will then switch to TLS via + STARTTLS *if the SMTP server supports it*. If this option is set, + Synapse will refuse to connect unless the server supports STARTTLS. + default: false + enable_tls: + type: boolean + description: >- + By default, if the server supports TLS, it will be used, and the + server must present a certificate that is valid for `tlsname`. If this + option is set to false, TLS will not be used. + default: true + tlsname: + type: string + description: >- + The domain name the SMTP server's TLS certificate must be valid for, + defaulting to `smtp_host`. + notif_from: + type: ["string", "null"] + description: >- + Defines the "From" address to use when sending emails. It must be set + if email sending is enabled. The placeholder `%(app)s` will be + replaced by the application name, which is normally set in `app_name`, + but may be overridden by the Matrix client application. Note that the + placeholder must be written `%(app)s`, including the trailing 's'. + default: null + app_name: + type: string + description: >- + Defines the default value for `%(app)s` in `notif_from` and email subjects. + default: Matrix + enable_notifs: + type: boolean + description: >- + Set to true to allow users to receive e-mail notifications. If this is + not set, users can configure e-mail notifications but will not receive + them. + default: false + notif_for_new_users: + type: boolean + description: >- + Set to false to disable automatic subscription to email notifications + for new users. + default: true + notif_delay_before_mail: + $ref: "#/$defs/duration" + description: >- + The time to wait before emailing about a notification. This gives the + user a chance to view the message via push or an open client. + + + _New in Synapse 1.99.0._ + default: 10m + client_base_url: + type: string + description: >- + Custom URL for client links within the email notifications. (This + setting used to be called `riot_base_url`; the old name is still + supported for backwards-compatibility but is now deprecated.) + default: "https://matrix.to" + validation_token_lifetime: + $ref: "#/$defs/duration" + description: >- + Configures the time that a validation email will expire after sending. + default: 1h + invite_client_location: + type: ["string", "null"] + description: >- + The web client location to direct users to during an invite. This is + passed to the identity server as the `org.matrix.web_client_location` + key. If null or unset no guidance is given to the identity server. + default: null + subjects: + type: object + description: >- + Subjects to use when sending emails from Synapse. The placeholder + `%(app)s` will be replaced with the value of the `app_name` setting, + or by a value dictated by the Matrix client application. In addition, + each subject can use the following placeholders: `%(person)s`, which + will be replaced by the displayname of the user(s) that sent the + message(s), e.g. "Alice and Bob", and `%(room)s`, which will be + replaced by the name of the room the message(s) have been sent to, + e.g. "My super room". In addition, emails related to account + administration will can use the `%(server_name)s` placeholder, which + will be replaced by the value of the `server_name` setting in your + Synapse configuration. + properties: + message_from_person_in_room: + type: string + description: >- + Subject to use to notify about one message from one or more + user(s) in a room which has a name. + default: >- + [%(app)s] You have a message on %(app)s from %(person)s in the + %(room)s room... + message_from_person: + type: string + description: >- + Subject to use to notify about one message from one or more + user(s) in a room which doesn't have a name. + default: "[%(app)s] You have a message on %(app)s from %(person)s..." + messages_from_person: + type: string + description: >- + Subject to use to notify about multiple messages from one or more + users in a room which doesn't have a name. + default: "[%(app)s] You have messages on %(app)s from %(person)s..." + messages_in_room: + type: string + description: >- + Subject to use to notify about multiple messages in a room which + has a name. + default: "[%(app)s] You have messages on %(app)s in the %(room)s room..." + messages_in_room_and_others: + type: string + description: >- + Subject to use to notify about multiple messages in multiple rooms. + default: >- + [%(app)s] You have messages on %(app)s in the %(room)s room and others... + messages_from_person_and_others: + type: string + description: >- + Subject to use to notify about multiple messages from multiple + persons in multiple rooms. This is similar to the setting above + except it's used when the room in which the notification was + triggered has no name. + default: >- + [%(app)s] You have messages on %(app)s from %(person)s and others... + invite_from_person_to_room: + type: string + description: >- + Subject to use to notify about an invite to a room which has a name. + default: >- + [%(app)s] %(person)s has invited you to join the %(room)s room on + %(app)s... + invite_from_person: + type: string + description: >- + Subject to use to notify about an invite to a room which doesn't + have a name. + default: "[%(app)s] %(person)s has invited you to chat on %(app)s..." + password_reset: + type: string + description: Subject to use when sending a password reset email. + default: "[%(server_name)s] Password reset" + email_validation: + type: string + description: >- + Subject to use when sending a verification email to assert an + address's ownership. + default: "[%(server_name)s] Validate your email" + examples: + - smtp_host: mail.server + smtp_port: 587 + smtp_user: exampleusername + smtp_pass: examplepassword + force_tls: true + require_transport_security: true + enable_tls: false + tlsname: mail.server.example.com + notif_from: "Your Friendly %(app)s homeserver " + app_name: my_branded_matrix_server + enable_notifs: true + notif_for_new_users: false + client_base_url: "http://localhost/riot" + validation_token_lifetime: 15m + invite_client_location: "https://app.element.io" + subjects: + message_from_person_in_room: >- + [%(app)s] You have a message on %(app)s from %(person)s in the + %(room)s room... + message_from_person: >- + [%(app)s] You have a message on %(app)s from %(person)s... + messages_from_person: >- + [%(app)s] You have messages on %(app)s from %(person)s... + messages_in_room: >- + [%(app)s] You have messages on %(app)s in the %(room)s room... + messages_in_room_and_others: >- + [%(app)s] You have messages on %(app)s in the %(room)s room and others... + messages_from_person_and_others: >- + [%(app)s] You have messages on %(app)s from %(person)s and others... + invite_from_person_to_room: >- + [%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s... + invite_from_person: >- + [%(app)s] %(person)s has invited you to chat on %(app)s... + password_reset: "[%(server_name)s] Password reset" + email_validation: "[%(server_name)s] Validate your email" + max_event_delay_duration: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + The maximum allowed duration by which sent events can be delayed, as + per + [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). + Must be a positive value if set. + + + If null or unset, sending of delayed events is disallowed. + default: null + examples: + - 24h + user_types: + type: object + description: >- + Configuration settings related to the user types feature. + properties: + default_user_type: + type: ["string", "null"] + description: "The default user type to use for registering new users when no value has been specified. Defaults to none." + default: null + extra_user_types: + type: array + description: "Array of additional user types to allow. These are treated as real users." + items: + type: string + default: [] + examples: + - default_user_type: "custom" + extra_user_types: ["custom", "custom2"] + admin_contact: + type: ["string", "null"] + description: How to reach the server admin, used in `ResourceLimitError`. + default: null + examples: + - "mailto:admin@server.com" + hs_disabled: + type: boolean + description: >- + Blocks users from connecting to the homeserver and provides the + human-readable reason given in `hs_disabled_message`. + default: false + examples: + - true + hs_disabled_message: + type: string + description: Human-readable reason why the connection was blocked. + default: Homeserver is currently blocked + examples: + - Reason for why the HS is blocked + limit_usage_by_mau: + type: boolean + description: >- + This option disables/enables monthly active user blocking. Used in cases + where the admin or server owner wants to limit to the number of monthly + active users. When enabled and a limit is reached the server returns a + `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. If + this is enabled, a value for `max_mau_value` must also be set. + + + See [Monthly Active Users](../administration/monthly_active_users.md) for + details on how to configure MAU. + default: false + examples: + - true + max_mau_value: + type: integer + description: >- + This option sets the hard limit of monthly active users above which the + server will start blocking user actions if `limit_usage_by_mau` is + enabled. + default: 0 + examples: + - 50 + mau_trial_days: + type: integer + description: >- + The option `mau_trial_days` is a means to add a grace period for active + users. It means that users must be active for the specified number of days + before they can be considered active and guards against the case where + lots of users sign up in a short space of time never to return after their + initial session. + default: 0 + examples: + - 5 + mau_appservice_trial_days: + type: object + description: >- + The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but + applies a different trial number if the user was registered by an + appservice. A value of 0 means no trial days are applied. Appservices not + listed in this dictionary use the value of `mau_trial_days` instead. + additionalProperties: + type: integer + default: {} + examples: + - my_appservice_id: 3 + another_appservice_id: 6 + mau_limit_alerting: + type: boolean + description: >- + Limit client-side alerting should the mau limit be reached. This is useful + for small instances where the admin has 5 mau seats (say) for 5 specific + people and no interest increasing the mau limit further. + default: true + examples: + - false + mau_stats_only: + type: boolean + description: >- + If enabled, the metrics for the number of monthly active users will be + populated, however no one will be limited based on these numbers. If + `limit_usage_by_mau` is true, this is implied to be true. + default: false + examples: + - true + mau_limit_reserved_threepids: + type: array + description: >- + Sometimes the server admin will want to ensure certain accounts are never + blocked by mau checking. These accounts are specified by this option. Add + accounts by specifying the `medium` and `address` of the reserved threepid + (3rd party identifier). + items: + type: object + properties: + medium: + type: string + description: Medium of the account threepid. + address: + type: string + description: Address of the account threepid. + default: [] + examples: + - - medium: email + address: reserved_user@example.com + server_context: + type: ["string", "null"] + description: >- + This option is used by phonehome stats to group together related servers. + default: null + examples: + - context + limit_remote_rooms: + type: object + description: >- + When this option is enabled, the room "complexity" will be checked before + a user joins a new remote room. If it is above the complexity limit, the + server will disallow joining, or will instantly leave. This is useful for + homeservers that are resource-constrained. Room complexity is an arbitrary + measure based on factors such as the number of users in the room. + properties: + enabled: + type: boolean + description: Whether this check is enabled. + default: false + complexity: + type: number + description: The limit above which rooms cannot be joined. + default: 1.0 + complexity_error: + type: string + description: >- + Override the error which is returned when the room is too complex with + a custom message. + default: >- + Your homeserver is unable to join rooms this large or complex. Please + speak to your server administrator, or upgrade your instance to join + this room. + admins_can_join: + type: boolean + description: Allow server admins to join complex rooms. + default: false + examples: + - enabled: true + complexity: 0.5 + complexity_error: I can't let you do that, Dave. + admins_can_join: true + require_membership_for_aliases: + type: boolean + description: Whether to require a user to be in the room to add an alias to it. + default: true + examples: + - false + allow_per_room_profiles: + type: boolean + description: >- + Whether to allow per-room membership profiles through the sending of + membership events with profile information that differs from the target's + global profile. + default: true + examples: + - false + max_avatar_size: + oneOf: + - $ref: "#/$defs/bytes" + - type: "null" + description: >- + The largest permissible file size in bytes for a user avatar. Defaults to + no restriction. Use M for MB and K for KB. + + + Note that user avatar changes will not work if this is set without using + Synapse's media repository. + default: null + examples: + - 10M + allowed_avatar_mimetypes: + type: ["array", "null"] + description: >- + The MIME types allowed for user avatars. Defaults to no restriction. + + + Note that user avatar changes will not work if this is set without using + Synapse's media repository. + items: + type: string + default: null + examples: + - - image/png + - image/jpeg + - image/gif + redaction_retention_period: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + How long to keep redacted events in unredacted form in the database. After + this period redacted events get replaced with their redacted form in the + DB. + + + Synapse will check whether the rentention period has concluded for + redacted events every 5 minutes. Thus, even if this option is set to `0`, + Synapse may still take up to 5 minutes to purge redacted events from the + database. Set to `null` to disable. + default: 7d + examples: + - 28d + forgotten_room_retention_period: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + How long to keep locally forgotten rooms before purging them from the DB. + A value of `null` means it's disabled. + default: null + examples: + - 28d + user_ips_max_age: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + How long to track users' last seen time and IPs in the database. Set to + `null` to disable clearing out of old rows. + default: 28d + examples: + - 14d + request_token_inhibit_3pid_errors: + type: boolean + description: >- + Inhibits the `/requestToken` endpoints from returning an error that might + leak information about whether an e-mail address is in use or not on this + homeserver. Note that for some endpoints the error situation is the e-mail + already being used, and for others the error is entering the e-mail being + unused. If this option is enabled, instead of returning an error, these + endpoints will act as if no error happened and return a fake session ID + (`sid`) to clients. + default: false + examples: + - true + next_link_domain_whitelist: + type: ["array", "null"] + description: >- + A list of domains that the domain portion of `next_link` parameters must + match. + + + This parameter is optionally provided by clients while requesting + validation of an email or phone number, and maps to a link that users will + be automatically redirected to after validation succeeds. Clients can make + use this parameter to aid the validation process. + + + The whitelist is applied whether the homeserver or an identity server is + handling validation. + + + The default value is no whitelist functionality; all domains are allowed. + Setting this value to an empty list will instead disallow all domains. + default: null + examples: + - matrix.org + templates: + type: object + description: >- + These options define templates to use when generating email or HTML page + contents. + + + See [here](../../templates.md) for more information about using custom + templates. + properties: + custom_template_directory: + type: ["string", "null"] + description: >- + Determines which directory Synapse will try to find template files in + to use to generate email or HTML page contents. If not set, or a file + is not found within the template directory, a default template from + within the Synapse package will be used. + default: null + examples: + - custom_template_directory: /path/to/custom/templates/ + retention: + type: object + description: >- + This option and the associated options determine message retention policy + at the server level. + + + Room admins and mods can define a retention period for their rooms using + the `m.room.retention` state event, and server admins can cap this period + by setting the `allowed_lifetime_min` and `allowed_lifetime_max` config + options. + + + If this feature is enabled, Synapse will regularly look for and purge + events which are older than the room's maximum retention period. Synapse + will also filter events received over federation so that events that + should have been purged are ignored and not stored again. + + + The message retention policies feature is disabled by default. You can + read more about this feature [here](../../message_retention_policies.md). + properties: + enabled: + type: boolean + description: Enforce message retention policies + default: false + default_policy: + type: object + description: >- + Default message retention policy. If set, Synapse will apply it to + rooms that lack the `m.room.retention` state event. + properties: + min_lifetime: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Minimum message retention time of the default message retention + policy. Synapse doesn't take this option into account yet. + default: null + max_lifetime: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Maximum message retention time of the default message retention policy. + default: null + allowed_lifetime_min: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Retention policy limit. If set, and the state of a room contains a + `m.room.retention` event in its state which contains a `min_lifetime` + that's beyond this bound, Synapse will cap the room's policy to these + limits when running purge jobs. + default: null + allowed_lifetime_max: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Retention policy limit. If set, and the state of a room contains a + `m.room.retention` event in its state which contains a `max_lifetime` + that's beyond this bound, Synapse will cap the room's policy to these + limits when running purge jobs. + default: null + purge_jobs: + type: ["array", "null"] + description: >- + Server admins can define the settings of the background jobs purging + the events whose lifetime has expired under the `purge_jobs` section. + + + If no configuration is provided for this option, a single job will be + set up to delete expired events in every room daily. + + + Each job's configuration defines which range of message lifetimes the + job takes care of. For example, if `shortest_max_lifetime` is "2d" and + `longest_max_lifetime` is "3d", the job will handle purging expired + events in rooms whose state defines a `max_lifetime` that's both + higher than 2 days, and lower than or equal to 3 days. Both the + minimum and the maximum value of a range are optional, e.g. a job with + no `shortest_max_lifetime` and a `longest_max_lifetime` of "3d" will + handle every room with a retention policy whose `max_lifetime` is + lower than or equal to three days. + + + The rationale for this per-job configuration is that some rooms might + have a retention policy with a low `max_lifetime`, where history needs + to be purged of outdated messages on a more frequent basis than for + the rest of the rooms (e.g. every 12h), but not want that purge to be + performed by a job that's iterating over every room it knows, which + could be heavy on the server. + + + If any purge job is configured, it is strongly recommended to have at + least a single job with neither `shortest_max_lifetime` nor + `longest_max_lifetime` set, or one job without `shortest_max_lifetime` + and one job without `longest_max_lifetime` set. Otherwise some rooms + might be ignored, even if `allowed_lifetime_min` and + `allowed_lifetime_max` are set, because capping a room's policy to + these values is done after the policies are retrieved from Synapse's + database (which is done using the range specified in a purge job's + configuration). + items: + type: object + properties: + shortest_max_lifetime: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Apply job to rooms that have a `max_lifetime` higher than + `shortest_max_lifetime`. A value of `null` never excludes any + room. + longest_max_lifetime: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Apply job to rooms that have a `max_lifetime` lower than or + equal to `shortest_max_lifetime`. A value of `null` never + excludes any room. + interval: + $ref: "#/$defs/duration" + description: How often to run the job. + default: null + examples: + - enabled: true + default_policy: + min_lifetime: 1d + max_lifetime: 1y + allowed_lifetime_min: 1d + allowed_lifetime_max: 1y + purge_jobs: + - longest_max_lifetime: 3d + interval: 12h + - shortest_max_lifetime: 3d + interval: 1d + tls_certificate_path: + type: ["string", "null"] + description: >- + This option specifies a PEM-encoded X509 certificate for TLS. This + certificate, as of Synapse 1.0, will need to be a valid and verifiable + certificate, signed by a recognised Certificate Authority. + + + Be sure to use a `.pem` file that includes the full certificate chain + including any intermediate certificates (for instance, if using certbot, + use `fullchain.pem` as your certificate, not `cert.pem`). + default: null + examples: + - CONFDIR/SERVERNAME.tls.crt + tls_private_key_path: + type: ["string", "null"] + description: PEM-encoded private key for TLS. + default: null + examples: + - CONFDIR/SERVERNAME.tls.key + federation_verify_certificates: + type: boolean + description: >- + Whether to verify TLS server certificates for outbound federation + requests. To disable certificate verification, set the option to false. + default: true + examples: + - false + federation_client_minimum_tls_version: + type: string + description: >- + The minimum TLS version that will be used for outbound federation + requests. + + + Configurable to `"1"`, `"1.1"`, `"1.2"`, or `"1.3"`. Note that setting + this value higher than `"1.2"` will prevent federation to most of the + public Matrix network: only configure it to `"1.3"` if you have an + entirely private federation setup and you can ensure TLS 1.3 support. + default: "1" + examples: + - "1.2" + federation_certificate_verification_whitelist: + type: array + description: >- + Skip federation certificate verification on a given whitelist of domains. + + + This setting should only be used in very specific cases, such as + federation over Tor hidden services and similar. For private networks of + homeservers, you likely want to use a private CA instead. + + + Only effective if `federation_verify_certificates` is `true`. + items: + type: string + default: [] + examples: + - - lon.example.com + - "*.domain.com" + - "*.onion" + federation_custom_ca_list: + type: array + description: >- + List of custom certificate authorities for federation traffic. + + + This setting should only normally be used within a private network of + homeservers. + + + Note that this list will replace those that are provided by your operating + environment. Certificates must be in PEM format. + items: + type: string + default: [] + examples: + - - myCA1.pem + - myCA2.pem + - myCA3.pem + federation_domain_whitelist: + type: array + description: >- + Restrict federation to the given whitelist of domains. N.B. we recommend + also firewalling your federation listener to limit inbound federation + traffic as early as possible, rather than relying purely on this + application-layer restriction. If not specified, the default is to + whitelist everything. + + + Note: this does not stop a server from joining rooms that servers not on + the whitelist are in. As such, this option is really only useful to + establish a "private federation", where a group of servers all whitelist + each other and have the same whitelist. + items: + type: string + default: [] + examples: + - - lon.example.com + - nyc.example.com + - syd.example.com + federation_whitelist_endpoint_enabled: + type: boolean + description: >- + Enables an endpoint for fetching the federation whitelist config. + + + The request method and path is `GET + /_synapse/client/v1/config/federation_whitelist`, and the response format + is: + + + ```json + + { + "whitelist_enabled": true, // Whether the federation whitelist is being enforced + "whitelist": [ // Which server names are allowed by the whitelist + "example.com" + ] + } + + ``` + + + If `whitelist_enabled` is `false` then the server is permitted to federate + with all others. + + + The endpoint requires authentication. + default: false + examples: + - true + federation_metrics_domains: + type: array + description: >- + Report prometheus metrics on the age of PDUs being sent to and received + from the given domains. This can be used to give an idea of "delay" on + inbound and outbound federation, though be aware that any delay can be due + to problems at either end or with the intermediate network. + items: + type: string + default: [] + examples: + - - matrix.org + - example.com + allow_profile_lookup_over_federation: + type: boolean + description: >- + Set to false to disable profile lookup over federation. By default, the + Federation API allows other homeservers to obtain profile data of any user + on this homeserver. + default: true + examples: + - false + allow_device_name_lookup_over_federation: + type: boolean + description: >- + Set this option to true to allow device display name lookup over + federation. By default, the Federation API prevents other homeservers from + obtaining the display names of any user devices on this homeserver. + default: false + examples: + - true + federation: + type: object + description: >- + The federation section defines some sub-options related to federation. + + + The following options are related to configuring timeout and retry logic + for one request, independently of the others. Short retry algorithm is + used when something or someone will wait for the request to have an + answer, while long retry is used for requests that happen in the + background, like sending a federation transaction. + + + `destination_*` options control the retry logic when communicating with a + specific homeserver destination. Unlike the previous configuration + options, these values apply across all requests for a given destination + and the state of the backoff is stored in the database. + properties: + client_timeout: + $ref: "#/$defs/duration" + description: Timeout for the federation requests. + default: 60s + max_short_retry_delay: + $ref: "#/$defs/duration" + description: Maximum delay to be used for the short retry algo. + default: 2s + max_long_retry_delay: + $ref: "#/$defs/duration" + description: Maximum delay to be used for the long retry algo. + default: 60s + max_short_retries: + type: integer + description: Maximum number of retries for the short retry algo. + default: 3 + max_long_retries: + type: integer + description: Maximum number of retries for the long retry algo. + default: 10 + destination_min_retry_interval: + $ref: "#/$defs/duration" + description: "The initial backoff, after the first request fails." + default: 10m + destination_retry_multiplier: + type: integer + description: >- + How much we multiply the backoff by after each subsequent fail. + default: 2 + destination_max_retry_interval: + $ref: "#/$defs/duration" + description: A cap on the backoff. + default: 1w + examples: + - client_timeout: 180s + max_short_retry_delay: 7s + max_long_retry_delay: 100s + max_short_retries: 5 + max_long_retries: 20 + destination_min_retry_interval: 30s + destination_retry_multiplier: 5 + destination_max_retry_interval: 12h + event_cache_size: + $ref: "#/$defs/size" + description: >- + The number of events to cache in memory. Defaults to 10K. Like other + caches, this is affected by `caches.global_factor` (see below). + + + For example, the default is 10K and the global_factor default is 0.5. + + + Since 10K * 0.5 is 5K then the event cache size will be 5K. + + + The cache affected by this configuration is named as "\*getEvent\*". + + + Note that this option is not part of the `caches` section. + default: 10K + examples: + - 15K + caches: + type: object + description: >- + A cache "factor" is a multiplier that can be applied to each of Synapse's + caches in order to increase or decrease the maximum number of entries that + can be stored. + io.element.post_description: >- + ### Reloading cache factors + + + The cache factors (i.e. `caches.global_factor` and + `caches.per_cache_factors`) may be reloaded at any time by sending a + [`SIGHUP`](https://en.wikipedia.org/wiki/SIGHUP) signal to Synapse + using e.g. + + + ```commandline + + kill -HUP [PID_OF_SYNAPSE_PROCESS] + + ``` + + + If you are running multiple workers, you must individually update the + worker config file and send this signal to each worker process. + + + If you're using the [example systemd + service](https://github.com/element-hq/synapse/blob/develop/contrib/systemd/matrix-synapse.service) + file in Synapse's `contrib` directory, you can send a `SIGHUP` signal by + using `systemctl reload matrix-synapse`. + properties: + global_factor: + type: number + description: >- + Controls the global cache factor, which is the default cache factor + for all caches if a specific factor for that cache is not otherwise + set. + + + This can also be set by the `SYNAPSE_CACHE_FACTOR` environment + variable. Setting by environment variable takes priority over setting + through the config file. + + + Defaults to 0.5, which will halve the size of all caches. + + + Note that changing this value also affects the HTTP connection pool. + default: 0.5 + per_cache_factors: + type: object + description: >- + A dictionary of cache name to cache factor for that individual cache. + Overrides the global cache factor for a given cache. + + + These can also be set through environment variables comprised of + `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital letters and + underscores. Setting by environment variable takes priority over + setting through the config file. Ex. + `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` + + + Some caches have '*' and other characters that are not alphanumeric or + underscores. These caches can be named with or without the special + characters stripped. For example, to specify the cache factor for + `*stateGroupCache*` via an environment variable would be + `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. + additionalProperties: + type: number + default: {} + expire_caches: + type: boolean + description: >- + Controls whether cache entries are evicted after a specified time + period. Set to false to disable this feature. Note that never expiring + caches may result in excessive memory usage. + default: true + cache_entry_ttl: + $ref: "#/$defs/duration" + description: >- + If `expire_caches` is enabled, this flag controls how long an entry + can be in a cache without having been accessed before being evicted. + default: 30m + sync_response_cache_duration: + $ref: "#/$defs/duration" + description: >- + Controls how long the results of a /sync request are cached for after + a successful response is returned. A higher duration can help clients + with intermittent connections, at the cost of higher memory usage. A + value of zero means that sync responses are not cached. + + + *Changed in Synapse 1.62.0*: The default was changed from 0 to 2m. + default: 2m + cache_autotuning: + type: object + description: >- + `cache_autotuning` and its sub-options `max_cache_memory_usage`, + `target_cache_memory_usage`, and `min_cache_ttl` work in conjunction + with each other to maintain a balance between cache memory usage and + cache entry availability. You must be using + [jemalloc](../administration/admin_faq.md#help-synapse-is-slow-and-eats-all-my-ramcpu) + to utilize this option, and all three of the options must be specified + for this feature to work. This option defaults to off, enable it by + providing values for the sub-options listed below. Please note that + the feature will not work and may cause unstable behavior (such as + excessive emptying of caches or exceptions) if all of the values are + not provided. Please see the [Config Conventions](#config-conventions) + for information on how to specify memory size and cache expiry + durations. + properties: + max_cache_memory_usage: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Sets a ceiling on how much memory the cache can use before caches + begin to be continuously evicted. They will continue to be evicted + until the memory usage drops below the + `target_cache_memory_usage`, set in the setting below, or until + the `min_cache_ttl` is hit. + default: null + target_cache_memory_usage: + oneOf: + - $ref: "#/$defs/bytes" + - type: "null" + description: Sets a rough target for the desired memory usage of the caches. + default: null + min_cache_ttl: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Sets a limit under which newer cache entries are not evicted and + is only applied when caches are actively being + evicted/`max_cache_memory_usage` has been exceeded. This is to + protect hot caches from being emptied while Synapse is evicting + due to memory. + default: null + examples: + - global_factor: 1.0 + per_cache_factors: + get_users_who_share_room_with_user: 2.0 + sync_response_cache_duration: 2m + cache_autotuning: + max_cache_memory_usage: 1024M + target_cache_memory_usage: 758M + min_cache_ttl: 5m + database: + $ref: "#/$defs/database" + examples: + - name: sqlite3 + args: + database: /path/to/homeserver.db + - name: psycopg2 + txn_limit: 10000 + args: + user: synapse_user + password: secretpassword + dbname: synapse + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 + databases: + type: object + description: >- + The `databases` option allows specifying a mapping between certain + database tables and database host details, spreading the load of a single + Synapse instance across multiple database backends. This is often referred + to as "database sharding". This option is only supported for PostgreSQL + database backends. + + + **Important note:** This is a supported option, but is not currently used + in production by the Matrix.org Foundation. Proceed with caution and + always make backups. + + + `databases` is a dictionary of arbitrarily-named database entries. Each + entry is equivalent to the value of the `database` homeserver config + option (see above), with the addition of a `data_stores` key. + `data_stores` is an array of strings that specifies the data store(s) (a + defined label for a set of tables) that should be stored on the associated + database backend entry. + + + The currently defined values for `data_stores` are: + + + * `"state"`: Database that relates to state groups will be stored in this + database. + + Specifically, that means the following tables: + * `state_groups` + * `state_group_edges` + * `state_groups_state` + + And the following sequences: + * `state_groups_seq_id` + + * `"main"`: All other database tables and sequences. + + + All databases will end up with additional tables used for tracking + database schema migrations and any pending background updates. Synapse + will create these automatically on startup when checking for and/or + performing database schema migrations. + + + To migrate an existing database configuration (e.g. all tables on a single + database) to a different configuration (e.g. the "main" data store on one + database, and "state" on another), do the following: + + + 1. Take a backup of your existing database. Things can and do go wrong and + database corruption is no joke! + + 2. Ensure all pending database migrations have been applied and background + updates have run. The simplest way to do this is to use the + `update_synapse_database` script supplied with your Synapse installation. + + ```sh + update_synapse_database --database-config homeserver.yaml --run-background-updates + ``` + + 3. Copy over the necessary tables and sequences from one database to the + other. Tables relating to database migrations, schemas, schema versions + and background updates should **not** be copied. + + As an example, say that you'd like to split out the "state" data store from an existing database which currently contains all data stores. + + Simply copy the tables and sequences defined above for the "state" datastore from the existing database to the secondary database. As noted above, additional tables will be created in the secondary database when Synapse is started. + + 4. Modify/create the `databases` option in your `homeserver.yaml` to match + the desired database configuration. + + 5. Start Synapse. Check that it starts up successfully and that things + generally seem to be working. + + 6. Drop the old tables that were copied in step 3. + + + Only one of the options `database` or `databases` may be specified in your + config, but not both. + additionalProperties: + $ref: "#/$defs/database" + properties: + data_stores: + type: array + items: + type: string + enum: + - state + - main + default: {} + examples: + - basement_box: + name: psycopg2 + txn_limit: 10000 + data_stores: + - main + args: + user: synapse_user + password: secretpassword + dbname: synapse_main + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 + my_other_database: + name: psycopg2 + txn_limit: 10000 + data_stores: + - state + args: + user: synapse_user + password: secretpassword + dbname: synapse_state + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 + log_config: + type: ["string", "null"] + description: >- + This option specifies a yaml python logging config file as described + [here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). + default: null + examples: + - CONFDIR/SERVERNAME.log.config + rc_message: + $ref: "#/$defs/rc" + description: >- + Ratelimiting settings for client messaging. + + + This is a ratelimiting option for messages that ratelimits sending based + on the account the client is using. + default: + per_second: 0.2 + burst_count: 10.0 + examples: + - per_second: 0.5 + burst_count: 15.0 + rc_registration: + $ref: "#/$defs/rc" + description: >- + This option ratelimits registration requests based on the client's IP address. + default: + per_second: 0.17 + burst_count: 3.0 + examples: + - per_second: 0.15 + burst_count: 2.0 + rc_registration_token_validity: + $ref: "#/$defs/rc" + description: >- + This option checks the validity of registration tokens that ratelimits + requests based on the client's IP address. + default: + per_second: 0.1 + burst_count: 5.0 + examples: + - per_second: 0.3 + burst_count: 6.0 + rc_login: + type: object + description: This option specifies several limits for login. + properties: + address: + $ref: "#/$defs/rc" + description: Ratelimits login requests based on the client's IP address. + default: + per_second: 0.003 + burst_count: 5.0 + account: + $ref: "#/$defs/rc" + description: >- + Ratelimits login requests based on the account the client is + attempting to log into. + default: + per_second: 0.003 + burst_count: 5.0 + failed_attempts: + $ref: "#/$defs/rc" + description: >- + Ratelimits login requests based on the account the client is + attempting to log into, based on the amount of failed login attempts + for this account. + default: + per_second: 0.17 + burst_count: 3.0 + examples: + - address: + per_second: 0.15 + burst_count: 5.0 + account: + per_second: 0.18 + burst_count: 4.0 + failed_attempts: + per_second: 0.19 + burst_count: 7.0 + rc_admin_redaction: + $ref: "#/$defs/rc" + description: >- + This option sets ratelimiting redactions by room admins. If this is not + explicitly set then it uses the same ratelimiting as per `rc_message`. + This is useful to allow room admins to deal with abuse quickly. + examples: + - per_second: 1.0 + burst_count: 50.0 + rc_joins: + type: object + description: This option allows for ratelimiting number of rooms a user can join. + properties: + local: + $ref: "#/$defs/rc" + description: Ratelimits when users are joining rooms the server is already in. + default: + per_second: 0.1 + burst_count: 10.0 + remote: + $ref: "#/$defs/rc" + description: >- + Ratelimits when users are trying to join rooms not on the server + (which can be more computationally expensive than restricting + locally). + default: + per_second: 0.01 + burst_count: 10.0 + examples: + - local: + per_second: 0.2 + burst_count: 15.0 + remote: + per_second: 0.03 + burst_count: 12.0 + rc_joins_per_room: + $ref: "#/$defs/rc" + description: >- + This option allows admins to ratelimit joins to a room based on the number + of recent joins (local or remote) to that room. It is intended to mitigate + mass-join spam waves which target multiple homeservers. + + + _Added in Synapse 1.64.0._ + default: + per_second: 1.0 + burst_count: 10.0 + examples: + - per_second: 1.0 + burst_count: 10.0 + rc_3pid_validation: + $ref: "#/$defs/rc" + description: >- + This option ratelimits how often a user or IP can attempt to validate a 3PID. + default: + per_second: 0.003 + burst_count: 5.0 + examples: + - per_second: 0.003 + burst_count: 5.0 + rc_invites: + type: object + description: >- + This option sets ratelimiting how often invites can be sent in a room or + to a specific user. + + + Client requests that invite user(s) when [creating a + room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom) + will count against the `rc_invites.per_room` limit, whereas client + requests to [invite a single user to a + room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidinvite) + will count against both the `rc_invites.per_user` and + `rc_invites.per_room` limits. + + + Federation requests to invite a user will count against the + `rc_invites.per_user` limit only, as Synapse presumes ratelimiting by room + will be done by the sending server. + + + _Changed in version 1.63:_ added the `per_issuer` limit. + properties: + per_room: + $ref: "#/$defs/rc" + description: Applies to the room of the invitation. + default: + per_second: 0.3 + burst_count: 10.0 + per_user: + $ref: "#/$defs/rc" + description: >- + Applies to the *receiver* of the invite, rather than the sender, + meaning that a `rc_invite.per_user.burst_count` of 5 mandates that a + single user cannot *receive* more than a burst of 5 invites at a + time. + default: + per_second: 0.003 + burst_count: 5.0 + per_issuer: + $ref: "#/$defs/rc" + description: >- + Applies to the *issuer* of the invite, meaning that a + `rc_invite.per_issuer.burst_count` of 5 mandates that single user + cannot *send* more than a burst of 5 invites at a time. + default: + per_second: 0.3 + burst_count: 10.0 + examples: + - per_room: + per_second: 0.5 + burst_count: 5.0 + per_user: + per_second: 0.004 + burst_count: 3.0 + per_issuer: + per_second: 0.5 + burst_count: 5.0 + rc_third_party_invite: + $ref: "#/$defs/rc" + description: >- + This option ratelimits 3PID invites (i.e. invites sent to a third-party ID + such as an email address or a phone number) based on the account that's + sending the invite. + default: + per_second: 0.2 + burst_count: 10.0 + rc_media_create: + $ref: "#/$defs/rc" + description: >- + This option ratelimits creation of MXC URIs via the + `/_matrix/media/v1/create` endpoint based on the account that's creating + the media. + default: + per_second: 10.0 + burst_count: 50.0 + rc_federation: + type: object + description: Defines limits on federation requests. + properties: + window_size: + type: integer + description: Window size in milliseconds. + default: 1000 + sleep_limit: + type: integer + description: >- + Number of federation requests from a single server in a window before + the server will delay processing the request. + default: 10 + sleep_delay: + type: integer + description: >- + Duration in milliseconds to delay processing events from remote + servers by if they go over the sleep limit. + default: 500 + reject_limit: + type: integer + description: >- + Maximum number of concurrent federation requests allowed from a single server. + default: 50 + concurrent: + type: integer + description: >- + Number of federation requests to concurrently process from a single server. + default: 3 + examples: + - window_size: 750 + sleep_limit: 15 + sleep_delay: 400 + reject_limit: 40 + concurrent: 5 + rc_presence: + type: object + description: This option sets ratelimiting for presence. + properties: + per_user: + $ref: "#/$defs/rc" + description: >- + Sets rate limits on how often a specific users' presence updates are + evaluated. Ratelimited presence updates sent via sync are ignored, and + no error is returned to the client. This option also sets the rate + limit for the [`PUT /_matrix/client/v3/presence/{userId}/status`] + endpoint. + + + [`PUT /_matrix/client/v3/presence/{userId}/status`]: + + default: + per_user: + per_second: 0.1 + burst_count: 1.0 + examples: + - per_user: + per_second: 0.05 + burst_count: 1.0 + rc_delayed_event_mgmt: + $ref: "#/$defs/rc" + description: >- + Ratelimiting settings for delayed event management. + + + This is a ratelimiting option that ratelimits attempts to restart, cancel, + or view delayed events based on the sending client's account and device + ID. + + + Attempts to create or send delayed events are ratelimited not by this + setting, but by `rc_message`. + + + Setting this to a high value allows clients to make delayed event + management requests often (such as repeatedly restarting a delayed event + with a short timeout, or restarting several different delayed events all + at once) without the risk of being ratelimited. + default: + per_second: 1.0 + burst_count: 5.0 + examples: + - per_second: 2.0 + burst_count: 20.0 + rc_reports: + $ref: "#/$defs/rc" + description: >- + Ratelimiting settings for reporting content. + + This is a ratelimiting option that ratelimits reports made by users + about content they see. + + Setting this to a high value allows users to report content quickly, possibly in + duplicate. This can result in higher database usage. + default: + per_user: + per_second: 1.0 + burst_count: 5.0 + examples: + - per_second: 2.0 + burst_count: 20.0 + rc_room_creation: + $ref: "#/$defs/rc" + description: >- + Sets rate limits for how often users are able to create rooms. + default: + per_user: + per_second: 0.016 + burst_count: 10.0 + examples: + - per_second: 1.0 + burst_count: 5.0 + federation_rr_transactions_per_room_per_second: + type: integer + description: >- + Sets outgoing federation transaction frequency for sending read-receipts, + per-room. + + + If we end up trying to send out more read-receipts, they will get buffered + up into fewer transactions. + default: 50 + examples: + - 40 + enable_authenticated_media: + type: boolean + description: >- + When set to true, all subsequent media uploads will be marked as + authenticated, and will not be available over legacy unauthenticated media + endpoints (`/_matrix/media/(r0|v3|v1)/download` and + `/_matrix/media/(r0|v3|v1)/thumbnail`) – requests for authenticated media + over these endpoints will result in a 404. All media, including + authenticated media, will be available over the authenticated media + endpoints `_matrix/client/v1/media/download` and + `_matrix/client/v1/media/thumbnail`. Media uploaded prior to setting this + option to true will still be available over the legacy endpoints. Note if + the setting is switched to false after enabling, media marked as + authenticated will be available over legacy endpoints. Defaults to true + (previously false). In a future release of Synapse, this option will be + removed and become always-on. + + + In all cases, authenticated requests to download media will succeed, but + for unauthenticated requests, this case-by-case breakdown describes + whether media downloads are permitted: + + + * `enable_authenticated_media = False`: + * unauthenticated client or homeserver requesting local media: allowed + * unauthenticated client or homeserver requesting remote media: allowed as long as the media is in the cache, or as long as the remote homeserver does not require authentication to retrieve the media + * `enable_authenticated_media = True`: + * unauthenticated client or homeserver requesting local media: allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); otherwise denied. + * unauthenticated client or homeserver requesting remote media: the same as for local media; allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); otherwise denied. + + It is especially notable that media downloaded before this option existed + (in older Synapse versions), or whilst this option was set to `False`, + will perpetually be available over the legacy, unauthenticated endpoint, + even after this option is set to `True`. This is for backwards + compatibility with older clients and homeservers that do not yet support + requesting authenticated media; those older clients or homeservers will + not be cut off from media they can already see. + + + _Changed in Synapse 1.120:_ This option now defaults to `True` when not + set, whereas before this version it defaulted to `False`. + default: true + examples: + - false + enable_media_repo: + type: boolean + description: >- + Enable the media store service in the Synapse master. Set to false if you + are using a separate media store worker. + default: true + examples: + - false + media_store_path: + type: string + description: Directory where uploaded images and attachments are stored. + default: media_store + examples: + - DATADIR/media_store + max_pending_media_uploads: + type: integer + description: >- + How many *pending media uploads* can a given user have? A pending media + upload is a created MXC URI that (a) is not expired (the + `unused_expires_at` timestamp has not passed) and (b) the media has not + yet been uploaded for. + default: 5 + examples: + - 5 + unused_expiration_time: + $ref: "#/$defs/duration" + description: How long to wait in milliseconds before expiring created media IDs. + default: 24h + examples: + - 1h + media_storage_providers: + type: array + description: >- + Media storage providers allow media to be stored in different locations. + items: + type: object + properties: + module: + type: string + description: "Type of resource, e.g. `file_system`." + store_local: + type: boolean + description: Whether to store newly uploaded local files. + store_remote: + type: boolean + description: Whether to store newly downloaded local files. + store_synchronous: + type: boolean + description: Whether to wait for successful storage for local uploads. + config: + type: object + description: Sets a path to the resource through the `directory` option. + properties: + directory: + type: string + description: Path to the resource. + default: [] + examples: + - - module: file_system + store_local: false + store_remote: false + store_synchronous: false + config: + directory: /mnt/some/other/directory + max_upload_size: + $ref: "#/$defs/bytes" + description: >- + The largest allowed upload size in bytes. + + + If you are using a reverse proxy you may also need to set this value in + your reverse proxy's config. Notably Nginx has a small max body size by + default. See [here](../../reverse_proxy.md) for more on using a reverse + proxy with Synapse. + default: 50M + examples: + - 60M + media_upload_limits: + type: array + description: >- + A list of media upload limits defining how much data a given user can + upload in a given time period. + + These limits are applied in addition to the `max_upload_size` limit above + (which applies to individual uploads). + + + An empty list means no limits are applied. + + + These settings can be overridden using the `get_media_upload_limits_for_user` + module API [callback](../../modules/media_repository_callbacks.md#get_media_upload_limits_for_user). + default: [] + items: + time_period: + type: "#/$defs/duration" + description: >- + The time period over which the limit applies. Required. + max_size: + type: "#/$defs/bytes" + description: >- + Amount of data that can be uploaded in the time period by the user. + Required. + examples: + - - time_period: 1h + max_size: 100M + - time_period: 1w + max_size: 500M + max_image_pixels: + $ref: "#/$defs/bytes" + description: Maximum number of pixels that will be thumbnailed. + default: 32M + examples: + - 35M + remote_media_download_burst_count: + $ref: "#/$defs/bytes" + description: >- + Remote media downloads are ratelimited using a [leaky bucket + algorithm](https://en.wikipedia.org/wiki/Leaky_bucket), where a given + "bucket" is keyed to the IP address of the requester when requesting + remote media downloads. This configuration option sets the size of the + bucket against which the size in bytes of downloads are penalized – if the + bucket is full, i.e. a given number of bytes have already been downloaded, + further downloads will be denied until the bucket drains. See also + `remote_media_download_per_second` which determines the rate at which the + "bucket" is emptied and thus has available space to authorize new + requests. + default: 500MiB + examples: + - 200M + remote_media_download_per_second: + $ref: "#/$defs/bytes" + description: >- + Works in conjunction with `remote_media_download_burst_count` to ratelimit + remote media downloads – this configuration option determines the rate at + which the "bucket" (see above) leaks in bytes per second. As requests are + made to download remote media, the size of those requests in bytes is + added to the bucket, and once the bucket has reached it's capacity, no + more requests will be allowed until a number of bytes has "drained" from + the bucket. This setting determines the rate at which bytes drain from the + bucket, with the practical effect that the larger the number, the faster + the bucket leaks, allowing for more bytes downloaded over a shorter period + of time. Defaults to 87KiB per second. See also + `remote_media_download_burst_count`. + default: 87KiB + examples: + - 40K + prevent_media_downloads_from: + type: array + description: >- + A list of domains to never download media from. Media from these domains + that is already downloaded will not be deleted, but will be inaccessible + to users. This option does not affect admin APIs trying to + download/operate on media. + + + This will not prevent the listed domains from accessing media themselves. + It simply prevents users on this server from downloading media originating + from the listed servers. + + + This will have no effect on media originating from the local server. This + only affects media downloaded from other Matrix servers, to control URL + previews see + [`url_preview_ip_range_blacklist`](#url_preview_ip_range_blacklist) or + [`url_preview_url_blacklist`](#url_preview_url_blacklist). + items: + type: string + default: [] + examples: + - - evil.example.org + - evil2.example.org + dynamic_thumbnails: + type: boolean + description: >- + Whether to generate new thumbnails on the fly to precisely match the + resolution requested by the client. If true then whenever a new resolution + is requested by the client the server will generate a new thumbnail. If + false the server will pick a thumbnail from a precalculated list. + default: false + examples: + - true + thumbnail_sizes: + type: array + description: List of thumbnails to precalculate when an image is uploaded. + items: + type: object + properties: + width: + type: integer + description: Width of the generated thumbnail. + height: + type: integer + description: Height of the generated thumbnail. + method: + type: string + enum: + - crop + - scale + description: >- + Method to fit the thumbnail dimensions. Current options are `crop` + and `scale`. + default: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale + - width: 800 + height: 600 + method: scale + media_retention: + type: object + description: >- + Controls whether local media and entries in the remote media cache (media + that is downloaded from other homeservers) should be removed under certain + conditions, typically for the purpose of saving space. + + + Purging media files will be the carried out by the media worker (that is, + the worker that has the `enable_media_repo` homeserver config option set + to `true`). This may be the main process. + + + The `media_retention.local_media_lifetime` and + `media_retention.remote_media_lifetime` config options control whether + media will be purged if it has not been accessed in a given amount of + time. Note that media is "accessed" when loaded in a room in a client, or + otherwise downloaded by a local or remote user. If the media has never + been accessed, the media's creation time is used instead. Both thumbnails + and the original media will be removed. If either of these options are + unset, then media of that type will not be purged. + + + Local or cached remote media that has been + [quarantined](../../admin_api/media_admin_api.md#quarantining-media-in-a-room) + will not be deleted. Similarly, local media that has been marked as + [protected from + quarantine](../../admin_api/media_admin_api.md#protecting-media-from-being-quarantined) + will not be deleted. + properties: + local_media_lifetime: + description: >- + Duration without access to a local media resource after which it will + be purged. If the media has never been accessed, the media's creation + time is used instead. Both thumbnails and the original media will be + removed. If unset or null, local media will not be purged. + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + default: null + remote_media_lifetime: + description: >- + Duration without access to a remote media resource after which it will + be purged. If the media has never been accessed, the media's creation + time is used instead. Both thumbnails and the original media will be + removed. If unset or null, remote media will not be purged. + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + default: null + examples: + - local_media_lifetime: 90d + remote_media_lifetime: 14d + url_preview_enabled: + type: boolean + description: >- + This setting determines whether the preview URL API is enabled. Set to + true to enable. If enabled you must specify a + `url_preview_ip_range_blacklist` blacklist. + default: false + examples: + - true + url_preview_ip_range_blacklist: + type: ["array", "null"] + description: >- + List of IP address CIDR ranges that the URL preview spider is denied from + accessing. There are no defaults: you must explicitly specify a list for + URL previewing to work. You should specify any internal services in your + network that you do not want synapse to try to connect to, otherwise + anyone in any Matrix room could cause your synapse to issue arbitrary GET + requests to your internal services, causing serious security issues. + + + (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly + listed here, since they correspond to unroutable addresses.) + + + This must be specified if `url_preview_enabled` is set. It is recommended + that you use the following example list as a starting point. + + + Note: The value is ignored when an HTTP proxy is in use. + items: + type: string + default: null + examples: + - - 127.0.0.0/8 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 100.64.0.0/10 + - 192.0.0.0/24 + - 169.254.0.0/16 + - 192.88.99.0/24 + - 198.18.0.0/15 + - 192.0.2.0/24 + - 198.51.100.0/24 + - 203.0.113.0/24 + - 224.0.0.0/4 + - "::1/128" + - "fe80::/10" + - "fc00::/7" + - "2001:db8::/32" + - "ff00::/8" + - "fec0::/10" + url_preview_ip_range_whitelist: + type: array + description: >- + This option sets a list of IP address CIDR ranges that the URL preview + spider is allowed to access even if they are specified in + `url_preview_ip_range_blacklist`. This is useful for specifying exceptions + to wide-ranging blacklisted target IP ranges – e.g. for enabling URL + previews for a specific private website only visible in your network. + items: + type: string + default: [] + examples: + - - 192.168.1.1 + url_preview_url_blacklist: + type: array + description: >- + Optional list of URL matches that the URL preview spider is denied from + accessing. This is a usability feature, not a security one. You should use + `url_preview_ip_range_blacklist` in preference to this, otherwise someone + could define a public DNS entry that points to a private IP address and + circumvent the blacklist. Applications that perform redirects or serve + different content when detecting that Synapse is accessing them can also + bypass the blacklist. This is more useful if you know there is an entire + shape of URL that you know that you do not want Synapse to preview. + + + Each list entry is a dictionary of url component attributes as returned by + urlparse.urlsplit as applied to the absolute form of the URL. See + [here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) + for more information. Some examples are: + + + * `username` + + * `netloc` + + * `scheme` + + * `path` + + + The values of the dictionary are treated as a filename match pattern + applied to that component of URLs, unless they start with a ^ in which + case they are treated as a regular expression match. If all the specified + component matches for a given list item succeed, the URL is blacklisted. + items: + type: object + default: [] + examples: + - - username: "*" + - netloc: google.com + - netloc: "*.google.com" + - scheme: http + - netloc: www.acme.com + path: /foo + - netloc: "^[0-9]+.[0-9]+.[0-9]+.[0-9]+$" + max_spider_size: + $ref: "#/$defs/bytes" + description: The largest allowed URL preview spidering size in bytes. + default: 10M + examples: + - 8M + url_preview_accept_language: + type: array + description: >- + A list of values for the Accept-Language HTTP header used when downloading + webpages during URL preview generation. This allows Synapse to specify the + preferred languages that URL previews should be in when communicating with + remote servers. + + + Each value is a IETF language tag; a 2-3 letter identifier for a language, + optionally followed by subtags separated by `-`, specifying a country or + region variant. + + + Multiple values can be provided, and a weight can be added to each by + using quality value syntax (;q=). `*` translates to any language. + items: + type: string + default: + - en + examples: + - - en-UK + - en-US;q=0.9 + - fr;q=0.8 + - "*;q=0.7" + oembed: + type: object + description: >- + oEmbed allows for easier embedding content from a website. It can be used + for generating URLs previews of services which support it. A default list + of oEmbed providers is included with Synapse. + properties: + disable_default_providers: + type: boolean + description: Do not use Synapse's default list of oEmbed providers. + default: false + additional_providers: + type: array + description: >- + Additional files with oEmbed configuration (each should be in the form + of providers.json). + items: + type: string + default: [] + examples: + - disable_default_providers: true + additional_providers: + - oembed/my_providers.json + recaptcha_public_key: + type: ["string", "null"] + description: >- + This homeserver's ReCAPTCHA public key. Must be specified if + [`enable_registration_captcha`](#enable_registration_captcha) is enabled. + default: null + examples: + - YOUR_PUBLIC_KEY + recaptcha_public_key_path: + type: ["string", "null"] + description: >- + An alternative to [`recaptcha_public_key`](#recaptcha_public_key): allows + the public key to be specified in an external file. + + + The file should be a plain text file, containing only the public key. + Synapse reads the public key from the given file once at startup. + + + _Added in Synapse 1.135.0._ + default: null + examples: + - /path/to/key/file + recaptcha_private_key: + type: ["string", "null"] + description: >- + This homeserver's ReCAPTCHA private key. Must be specified if + [`enable_registration_captcha`](#enable_registration_captcha) is enabled. + default: null + examples: + - YOUR_PRIVATE_KEY + recaptcha_private_key_path: + type: ["string", "null"] + description: >- + An alternative to [`recaptcha_private_key`](#recaptcha_private_key): + allows the private key to be specified in an external file. + + + The file should be a plain text file, containing only the private key. + Synapse reads the private key from the given file once at startup. + + + _Added in Synapse 1.135.0._ + default: null + examples: + - /path/to/key/file + enable_registration_captcha: + type: boolean + description: >- + Set to `true` to require users to complete a CAPTCHA test when registering + an account. Requires a valid ReCaptcha public/private key. + + + Note that [`enable_registration`](#enable_registration) must also be set + to allow account registration. + default: false + examples: + - true + recaptcha_siteverify_api: + type: string + description: The API endpoint to use for verifying `m.login.recaptcha` responses. + default: "https://www.recaptcha.net/recaptcha/api/siteverify" + examples: + - "https://my.recaptcha.site" + turn_uris: + type: array + description: The public URIs of the TURN server to give to clients. + items: + type: string + default: [] + examples: + - - "turn:example.org" + turn_shared_secret: + type: ["string", "null"] + description: The shared secret used to compute passwords for the TURN server. + default: null + examples: + - YOUR_SHARED_SECRET + turn_shared_secret_path: + type: ["string", "null"] + description: >- + An alternative to [`turn_shared_secret`](#turn_shared_secret): allows the + shared secret to be specified in an external file. + + + The file should be a plain text file, containing only the shared secret. + Synapse reads the shared secret from the given file once at startup. + + + _Added in Synapse 1.116.0._ + default: null + examples: + - /path/to/secrets/file + turn_username: + type: ["string", "null"] + description: TURN server username if not using a token. + default: null + examples: + - TURNSERVER_USERNAME + turn_password: + type: ["string", "null"] + description: TURN server password if not using a token. + default: null + examples: + - TURNSERVER_PASSWORD + turn_user_lifetime: + $ref: "#/$defs/duration" + description: How long generated TURN credentials last. + default: 1h + examples: + - 2h + turn_allow_guests: + type: boolean + description: >- + Whether guests should be allowed to use the TURN server. If false, VoIP + will be unreliable for guests. However, it does introduce a slight + security risk as it allows users to connect to arbitrary endpoints without + having first signed up for a valid account (e.g. by passing a CAPTCHA). + default: true + examples: + - false + enable_registration: + type: boolean + description: >- + Enable registration for new users. + + + It is highly recommended that if you enable registration, you set one or + more or the following options, to avoid abuse of your server by "bots": + + + * [`enable_registration_captcha`](#enable_registration_captcha) + + * [`registrations_require_3pid`](#registrations_require_3pid) + + * [`registration_requires_token`](#registration_requires_token) + + + (In order to enable registration without any verification, you must also + set + [`enable_registration_without_verification`](#enable_registration_without_verification).) + + + Note that even if this setting is disabled, new accounts can still be + created via the admin API if + [`registration_shared_secret`](#registration_shared_secret) is set. + default: false + examples: + - true + enable_registration_without_verification: + type: boolean + description: >- + Enable registration without email or captcha verification. Note: this + option is *not* recommended, as registration without verification is a + known vector for spam and abuse. Has no effect unless + [`enable_registration`](#enable_registration) is also enabled. + default: false + examples: + - true + registrations_require_3pid: + type: array + description: >- + If this is set, users must provide all of the specified types of + [3PID](https://spec.matrix.org/latest/appendices/#3pid-types) when + registering an account. + + + Note that [`enable_registration`](#enable_registration) must also be set + to allow account registration. + items: + type: string + default: [] + examples: + - - email + - msisdn + disable_msisdn_registration: + type: boolean + description: >- + Explicitly disable asking for MSISDNs from the registration flow + (overrides `registrations_require_3pid` if MSISDNs are set as required). + default: false + examples: + - true + allowed_local_3pids: + type: ["array", "null"] + description: >- + Mandate that users are only allowed to associate certain formats of 3PIDs + with accounts on this server, as specified by the `medium` and `pattern` + sub-options. `pattern` is a [Perl-like regular + expression](https://docs.python.org/3/library/re.html#module-re). + + + More information about 3PIDs, allowed `medium` types and their `address` + syntax can be found [in the Matrix + spec](https://spec.matrix.org/latest/appendices/#3pid-types). + items: + type: object + description: Item allowing a given pattern for the specified 3PID medium. + properties: + medium: + $ref: "#/$defs/3pidmedium" + description: The medium for which to allow 3PID association. + pattern: + type: string + description: >- + A [Perl-like regular + expression](https://docs.python.org/3/library/re.html#module-re) + allowing association of a 3PID to a local account if it matches the + given format. + default: null + examples: + - - medium: email + pattern: "^[^@]+@matrix\\.org$" + - medium: email + pattern: "^[^@]+@vector\\.im$" + - medium: msisdn + pattern: "^44\\d{10}$" + enable_3pid_lookup: + type: boolean + description: Enable 3PIDs lookup requests to identity servers from this server. + default: true + examples: + - false + registration_requires_token: + type: boolean + description: >- + Require users to submit a token during registration. Tokens can be managed + using the admin [API](../administration/admin_api/registration_tokens.md). + Disabling this option will not delete any tokens previously generated. + + + Note that [`enable_registration`](#enable_registration) must also be set + to allow account registration. + default: false + examples: + - true + registration_shared_secret: + type: ["string", "null"] + description: >- + If set, allows registration of standard or admin accounts by anyone who + has the shared secret, even if + [`enable_registration`](#enable_registration) is not set. + + + This is primarily intended for use with the `register_new_matrix_user` + script (see [Registering a + user](../../setup/installation.md#registering-a-user)); however, the + interface is [documented](../../admin_api/register_api.html). + + + Replacing an existing `registration_shared_secret` with a new one requires + users of the [Shared-Secret Registration + API](../../admin_api/register_api.html) to start using the new secret for + requesting any further one-time nonces. + + + > ⚠️ **Warning** – The additional consequences of replacing + [`macaroon_secret_key`](#macaroon_secret_key) will apply in case it + delegates to `registration_shared_secret`. + + + See also + [`registration_shared_secret_path`](#registration_shared_secret_path). + default: null + examples: + - "" + registration_shared_secret_path: + type: ["string", "null"] + description: >- + An alternative to + [`registration_shared_secret`](#registration_shared_secret): allows the + shared secret to be specified in an external file. + + + The file should be a plain text file, containing only the shared secret. + + + If this file does not exist, Synapse will create a new shared secret on + startup and store it in this file. + + + _Added in Synapse 1.67.0._ + default: null + examples: + - /path/to/secrets/file + bcrypt_rounds: + type: integer + description: >- + Set the number of bcrypt rounds used to generate password hash. Larger + numbers increase the work factor needed to generate the hash. The default + number is 12 (which equates to 2^12 rounds). N.B. that increasing this + will exponentially increase the time required to register or login - e.g. + 24 => 2^24 rounds which will take >20 mins. + default: 12 + examples: + - 14 + allow_guest_access: + type: boolean + description: >- + Allows users to register as guests without a password/email/etc, and + participate in rooms hosted on this server which have been made accessible + to anonymous users. + default: false + examples: + - true + default_identity_server: + type: ["string", "null"] + description: >- + The identity server which we suggest that clients should use when users + log in on this server. + + + (By default, no suggestion is made, so it is left up to the client. This + setting is ignored unless `public_baseurl` is also explicitly set.) + default: null + examples: + - "https://matrix.org" + account_threepid_delegates: + type: object + properties: + msisdn: + type: ["string", "null"] + description: Identity server base URI for MSISDN (phone numbers). See above. + description: >- + Delegate verification of phone numbers to an identity server. + + + When a user wishes to add a phone number to their account, we need to + verify that they actually own that phone number, which requires sending + them a text message (SMS). Currently Synapse does not support sending + those texts itself and instead delegates the task to an identity server. + The base URI for the identity server to be used is specified by the + `account_threepid_delegates.msisdn` option. + + + If this is left unspecified, Synapse will not allow users to add phone + numbers to their account. + + + (Servers handling the these requests must answer the `/requestToken` + endpoints defined by the Matrix Identity Service API + [specification](https://matrix.org/docs/spec/identity_service/latest).) + + + *Deprecated in Synapse 1.64.0*: The `email` option is deprecated. + + + *Removed in Synapse 1.66.0*: The `email` option has been removed. If + present, Synapse will report a configuration error on startup. + default: {} + examples: + - msisdn: "http://localhost:8090" + enable_set_displayname: + type: boolean + description: >- + Whether users are allowed to change their displayname after it has been + initially set. Useful when provisioning users based on the contents of a + third-party directory. + + + Does not apply to server administrators. + default: true + examples: + - false + enable_set_avatar_url: + type: boolean + description: >- + Whether users are allowed to change their avatar after it has been + initially set. Useful when provisioning users based on the contents of a + third-party directory. + + + Does not apply to server administrators. + default: true + examples: + - false + enable_3pid_changes: + type: boolean + description: >- + Whether users can change the third-party IDs associated with their + accounts (email address and msisdn). + default: true + examples: + - false + auto_join_rooms: + type: array + description: >- + Users who register on this homeserver will automatically be joined to the + rooms listed under this option. + + + By default, any room aliases included in this list will be created as a + publicly joinable room when the first user registers for the homeserver. + If the room already exists, make certain it is a publicly joinable room, + i.e. the join rule of the room must be set to `public`. You can find more + options relating to auto-joining rooms below. + + + As Spaces are just rooms under the hood, Space aliases may also be used. + items: + type: string + default: [] + examples: + - - "#exampleroom:example.com" + - "#anotherexampleroom:example.com" + autocreate_auto_join_rooms: + type: boolean + description: >- + Where `auto_join_rooms` are specified, setting this flag ensures that the + rooms exist by creating them when the first user on the homeserver + registers. This option will not create Spaces. + + + By default the auto-created rooms are publicly joinable from any federated + server. Use the `autocreate_auto_join_rooms_federated` and + `autocreate_auto_join_room_preset` settings to customise this behaviour. + + + Setting to false means that if the rooms are not manually created, users + cannot be auto-joined since they do not exist. + default: true + examples: + - false + autocreate_auto_join_rooms_federated: + type: boolean + description: >- + Whether the rooms listed in `auto_join_rooms` that are auto-created are + available via federation. Only has an effect if + `autocreate_auto_join_rooms` is true. + + + Note that whether a room is federated cannot be modified after creation. + + + If true, the room will be joinable from other servers. If false, users + from other homeservers are prevented from joining these rooms. + default: true + examples: + - false + autocreate_auto_join_room_preset: + type: string + description: >- + The room preset to use when auto-creating one of `auto_join_rooms`. Only + has an effect if `autocreate_auto_join_rooms` is true. + + + Possible values for this option are: + + * "public_chat": the room is joinable by anyone, including federated + servers if `autocreate_auto_join_rooms_federated` is true (the default). + + * "private_chat": an invitation is required to join these rooms. + + * "trusted_private_chat": an invitation is required to join this room and + the invitee is assigned a power level of 100 upon joining the room. + + + Each preset will set up a room in the same manner as if it were provided + as the `preset` parameter when calling the [`POST + /_matrix/client/v3/createRoom`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom) + Client-Server API endpoint. + + + If a value of "private_chat" or "trusted_private_chat" is used then + `auto_join_mxid_localpart` must also be configured. + enum: + - public_chat + - private_chat + - trusted_private_chat + default: public_chat + examples: + - private_chat + auto_join_mxid_localpart: + type: ["string", "null"] + description: >- + The local part of the user id which is used to create `auto_join_rooms` if + `autocreate_auto_join_rooms` is true. If this is not provided then the + initial user account that registers will be used to create the rooms. + + + The user id is also used to invite new users to any auto-join rooms which + are set to invite-only. + + + It *must* be configured if `autocreate_auto_join_room_preset` is set to + "private_chat" or "trusted_private_chat". + + + Note that this must be specified in order for new users to be correctly + invited to any auto-join rooms which have been set to invite-only (either + at the time of creation or subsequently). + + + Note that, if the room already exists, this user must be joined and have + the appropriate permissions to invite new members. + default: null + examples: + - system + auto_join_rooms_for_guests: + type: boolean + description: >- + When `auto_join_rooms` is specified, setting this flag to false prevents + guest accounts from being automatically joined to the rooms. + default: true + examples: + - false + inhibit_user_in_use_error: + type: boolean + description: >- + Whether to inhibit errors raised when registering a new account if the + user ID already exists. If turned on, requests to `/register/available` + will always show a user ID as available, and Synapse won't raise an error + when starting a registration with a user ID that already exists. However, + Synapse will still raise an error if the registration completes and the + username conflicts. + default: false + examples: + - true + allow_underscore_prefixed_registration: + type: boolean + description: >- + Whether users are allowed to register with a underscore-prefixed + localpart. By default, AppServices use prefixes like `_example` to + namespace their associated ghost users. If turned on, this may result in + clashes or confusion. Useful when provisioning users from an external + identity provider. + default: false + examples: + - true + session_lifetime: + $ref: "#/$defs/duration" + description: >- + Time that a user's session remains valid for, after they log in. + + + Note that this is not currently compatible with guest logins. + + + Note also that this is calculated at login time: changes are not applied + retrospectively to users who have already logged in. + default: infinity + examples: + - 24h + refreshable_access_token_lifetime: + $ref: "#/$defs/duration" + description: >- + Time that an access token remains valid for, if the session is using + refresh tokens. + + + For more information about refresh tokens, please see the + [manual](user_authentication/refresh_tokens.md). + + + Note that this only applies to clients which advertise support for refresh + tokens. + + + Note also that this is calculated at login time and refresh time: changes + are not applied to existing sessions until they are refreshed. + default: 5m + examples: + - 10m + refresh_token_lifetime: + $ref: "#/$defs/duration" + description: >- + Time that a refresh token remains valid for (provided that it is not + exchanged for another one first). This option can be used to automatically + log-out inactive sessions. Please see the manual for more information. + + + Note also that this is calculated at login time and refresh time: changes + are not applied to existing sessions until they are refreshed. + default: infinity + examples: + - 24h + nonrefreshable_access_token_lifetime: + $ref: "#/$defs/duration" + description: >- + Time that an access token remains valid for, if the session is NOT using + refresh tokens. + + + Please note that not all clients support refresh tokens, so setting this + to a short value may be inconvenient for some users who will then be + logged out frequently. + + + Note also that this is calculated at login time: changes are not applied + retrospectively to existing sessions for users that have already logged + in. + default: infinity + examples: + - 24h + ui_auth: + oneOf: + - $ref: "#/$defs/duration" + - type: object + properties: + session_timeout: + $ref: "#/$defs/duration" + description: >- + The amount of time to allow a user-interactive authentication session to + be active. + + + This defaults to 0, meaning the user is queried for their credentials + before every action, but this can be overridden to allow a single + validation to be re-used. This weakens the protections afforded by the + user-interactive authentication process, by allowing for multiple (and + potentially different) operations to use the same validation session. + + + This is ignored for potentially "dangerous" operations (including + deactivating an account, modifying an account password, adding a 3PID, and + minting additional login tokens). + + + Use the `session_timeout` sub-option here to change the time allowed for + credential validation. + default: 0 + examples: + - session_timeout: 15s + login_via_existing_session: + type: object + description: >- + Matrix supports the ability of an existing session to mint a login token + for another client. + + + Synapse disables this by default as it has security ramifications – a + malicious client could use the mechanism to spawn more than one session. + properties: + enabled: + type: boolean + description: "Enable login via existing session." + default: false + require_ui_auth: + type: boolean + description: Require user-interactive authentication. + default: true + token_timeout: + $ref: "#/$defs/duration" + description: Duration of time the generated token is valid. + default: 5m + examples: + - enabled: true + require_ui_auth: false + token_timeout: 5m + enable_metrics: + type: boolean + description: Set to true to enable collection and rendering of performance metrics. + default: false + examples: + - true + sentry: + type: object + description: >- + Use this option to enable sentry integration. Provide the DSN assigned to + you by sentry with the `dsn` setting. + + + An optional `environment` field can be used to specify an environment. + This allows for log maintenance based on different environments, ensuring + better organization and analysis. + + + NOTE: While attempts are made to ensure that the logs don't contain any + sensitive information, this cannot be guaranteed. By enabling this option + the sentry server may therefore receive sensitive information, and it in + turn may then disseminate sensitive information through insecure + notification channels if so configured. + properties: + dsn: + type: ["string", "null"] + description: >- + The DSN assigned by sentry. If unset or null, sentry integration is + disabled. + default: null + environment: + type: ["string", "null"] + description: Sentry environment. + default: null + examples: + - environment: production + dsn: ... + metrics_flags: + type: object + description: >- + Flags to enable Prometheus metrics which are not suitable to be enabled by + default, either for performance reasons or limited use. Currently the only + option is `known_servers`. + properties: + known_servers: + type: boolean + description: >- + Publishes `synapse_federation_known_servers`, a gauge of the number of + servers this homeserver knows about, including itself. May cause + performance problems on large homeservers. + default: false + examples: + - known_servers: true + report_stats: + type: boolean + description: >- + Whether or not to report homeserver usage statistics. This is originally + set when generating the config. Set this option to true or false to change + the current behavior. See [Reporting Homeserver Usage + Statistics](../administration/monitoring/reporting_homeserver_usage_statistics.md) + for information on what data is reported. + + + Statistics will be reported 5 minutes after Synapse starts, and then every + 3 hours after that. + default: false + examples: + - true + report_stats_endpoint: + type: string + description: The endpoint to report homeserver usage statistics to. + default: https://matrix.org/report-usage-stats/push + examples: + - https://example.com/report-usage-stats/push + room_prejoin_state: + type: object + description: >- + This setting controls the state that is shared with users upon receiving + an invite to a room, or in reply to a knock on a room. By default, the + following state events are shared with users: + + + - `m.room.join_rules` + + - `m.room.canonical_alias` + + - `m.room.avatar` + + - `m.room.encryption` + + - `m.room.name` + + - `m.room.create` + + - `m.room.topic` + + + *Changed in Synapse 1.74:* admins can filter the events in prejoin state + based on their state key. + properties: + disable_default_event_types: + type: boolean + description: >- + Set to `true` to disable the above defaults. If this is enabled, only + the event types listed in `additional_event_types` are shared. + default: false + additional_event_types: + type: array + description: >- + A list of additional state events to include in the events to be + shared. By default, this list is empty (so only the default event + types are shared). + + + Each entry in this list should be either a single string or a list of + two strings. + + * A standalone string `t` represents all events with type `t` (i.e. + with no restrictions on state keys). + + * A pair of strings `[t, s]` represents a single event with type `t` + and state key `s`. The same type can appear in two entries with + different state keys: in this situation, both state keys are included + in prejoin state. + items: + type: ["string", "array"] + items: + type: string + default: [] + examples: + - disable_default_event_types: false + additional_event_types: + - org.example.custom.event.typeA + - - org.example.custom.event.typeB + - foo + - - org.example.custom.event.typeC + - bar + - - org.example.custom.event.typeC + - baz + track_puppeted_user_ips: + type: boolean + description: >- + We record the IP address of clients used to access the API for various + reasons, including displaying it to the user in the "Where you're signed + in" dialog. + + + By default, when puppeting another user via the admin API, the client IP + address is recorded against the user who created the access token (ie, the + admin user), and *not* the puppeted user. + + + Set this option to true to also record the IP address against the puppeted + user. (This also means that the puppeted user will count as an "active" + user for the purpose of monthly active user tracking – see + `limit_usage_by_mau` etc above.) + default: false + examples: + - true + app_service_config_files: + type: array + description: A list of application service config files to use. + items: + type: string + default: [] + examples: + - - app_service_1.yaml + - app_service_2.yaml + track_appservice_user_ips: + type: boolean + description: >- + Set to true to enable tracking of application service IP addresses. + Implicitly enables MAU tracking for application service users. + default: false + examples: + - true + use_appservice_legacy_authorization: + type: boolean + description: >- + Whether to send the application service access tokens via the + `access_token` query parameter per older versions of the Matrix + specification. Defaults to false. Set to true to enable sending access + tokens via a query parameter. + + + **Enabling this option is considered insecure and is not recommended.** + default: false + examples: + - true + macaroon_secret_key: + type: ["string", "null"] + description: >- + A secret which is used to sign + + - access token for guest users, + + - short-term login token used during SSO logins (OIDC or SAML2) and + + - token used for unsubscribing from email notifications. + + + If none is specified, the `registration_shared_secret` is used, if one is + given; otherwise, a secret key is derived from the signing key. + + + > ⚠️ **Warning** – Replacing an existing `macaroon_secret_key` with a new + one will lead to invalidation of access tokens for all guest users. It + will also break unsubscribe links in emails sent before the change. An + unlucky user might encounter a broken SSO login flow and would have to + start again. + default: null + examples: + - "" + macaroon_secret_key_path: + type: ["string", "null"] + description: >- + An alternative to [`macaroon_secret_key`](#macaroon_secret_key): allows + the secret key to be specified in an external file. + + + The file should be a plain text file, containing only the secret key. + Synapse reads the secret key from the given file once at startup. + + + _Added in Synapse 1.121.0._ + default: null + examples: + - /path/to/secrets/file + form_secret: + type: ["string", "null"] + description: >- + A secret which is used to calculate HMACs for form values, to stop + falsification of values. Must be specified for the User Consent forms to + work. + + + Replacing an existing `form_secret` with a new one might break the user + consent page for an unlucky user and require them to reopen the page from + a new link. + default: null + examples: + - "" + form_secret_path: + type: ["string", "null"] + description: >- + An alternative to [`form_secret`](#form_secret): allows the secret to be + specified in an external file. + + + The file should be a plain text file, containing only the secret. Synapse + reads the secret from the given file once at startup. + + + _Added in Synapse 1.126.0._ + default: null + examples: + - /path/to/secrets/file + signing_key_path: + type: ["string", "null"] + description: >- + Path to the signing key to sign events and federation requests with. + + + *New in Synapse 1.67*: If this file does not exist, Synapse will create a + new signing key on startup and store it in this file. + default: null + examples: + - CONFDIR/SERVERNAME.signing.key + old_signing_keys: + type: object + description: >- + The keys that the server used to sign messages with but won't use to sign + new messages. + + + It is possible to build an entry from an old `signing.key` file using the + `export_signing_key` script which is provided with synapse. + + + If you have lost the private key file, you can ask another server you + trust to tell you the public keys it has seen from your server. To fetch + the keys from `matrix.org`, try something like: + + + ``` + + curl https://matrix-federation.matrix.org/_matrix/key/v2/query/myserver.example.com | + jq '.server_keys | map(.verify_keys) | add' + ``` + additionalProperties: + type: object + properties: + key: + type: string + description: The base64-encoded public key. + expired_ts: + type: integer + description: >- + Time, in milliseconds since the unix epoch, that the key was last used. + default: {} + examples: + - "ed25519:id": + key: base64string + expired_ts: 123456789123 + key_refresh_interval: + $ref: "#/$defs/duration" + description: >- + How long key response published by this server is valid for. Used to set the + `valid_until_ts` in `/key/v2` APIs. Determines how quickly servers will query to + check which keys are still valid. + default: 1d + examples: + - 2d + trusted_key_servers: + type: array + description: >- + The trusted servers to download signing keys from. + + + When we need to fetch a signing key, each server is tried in parallel. + + + Normally, the connection to the key server is validated via TLS + certificates. Additional security can be provided by configuring a `verify + key`, which will make synapse check that the response is signed by that + key. + + + This setting supersedes an older setting named `perspectives`. The old + format is still supported for backwards-compatibility, but it is + deprecated. + + + `trusted_key_servers` defaults to matrix.org, but using it will generate a + warning on start-up. To suppress this warning, set + `suppress_key_server_warning` to true. + + + If the use of a trusted key server has to be deactivated, e.g. in a + private federation or for privacy reasons, this can be realised by setting + an empty array (`trusted_key_servers: []`). Then Synapse will request the + keys directly from the server that owns the keys. If Synapse does not get + keys directly from the server, the events of this server will be rejected. + items: + server_name: + type: string + description: The name of the server. Required. + verify_keys: + type: ["object", "null"] + description: >- + An optional map from key id to base64-encoded public key. If + specified, we will check that the response is signed by at least one + of the given keys. + additionalProperties: + type: string + accept_keys_insecurely: + type: boolean + description: >- + Normally, if `verify_keys` is unset, and + `federation_verify_certificates` is not `true`, synapse will refuse to + start, because this would allow anyone who can spoof DNS responses to + masquerade as the trusted key server. If you know what you are doing + and are sure that your network environment provides a secure + connection to the key server, you can set this to `true` to override + this behaviour. + default: + - server_name: matrix.org + examples: + - - server_name: my_trusted_server.example.com + verify_keys: + "ed25519:auto": abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr + - server_name: my_other_trusted_server.example.com + - - server_name: matrix.org + suppress_key_server_warning: + type: boolean + description: >- + Set the following to true to disable the warning that is emitted when the + `trusted_key_servers` include "matrix.org". See above. + default: false + examples: + - true + key_server_signing_keys_path: + type: ["string", "null"] + description: >- + The signing keys to use when acting as a trusted key server. If not + specified defaults to the server signing key. + + + Can contain multiple keys, one per line. + default: null + examples: + - key_server_signing_keys.key + saml2_config: + type: object + description: >- + Enable SAML2 for registration and login. Uses pysaml2. To learn more about + pysaml and to find a full list options for configuring pysaml, read the + docs [here](https://pysaml2.readthedocs.io/en/latest/). + + + At least one of `sp_config` or `config_path` must be set in this section + to enable SAML login. You can either put your entire pysaml config inline + using the `sp_config` option, or you can specify a path to a psyaml config + file with the sub-option `config_path`. + + + Once SAML support is enabled, a metadata file will be exposed at + `https://:/_synapse/client/saml2/metadata.xml`, which you + may be able to use to configure your SAML IdP with. Alternatively, you can + manually configure the IdP to use an ACS location of + `https://:/_synapse/client/saml2/authn_response`. + properties: + idp_name: + type: string + description: >- + A user-facing name for this identity provider, which is used to offer + the user a choice of login mechanisms. + idp_icon: + type: ["string", "null"] + description: >- + An optional icon for this identity provider, which is presented by + clients and Synapse's own IdP picker page. If given, must be an MXC + URI of the format `mxc:///`. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) room + and then copy the URL from the source of the event.) + idp_brand: + description: >- + An optional brand for this identity provider, allowing clients to + style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options + here. + sp_config: + type: ["object", "null"] + description: >- + Configuration for the pysaml2 Service Provider. See pysaml2 docs for + format of config. Default values will be used for the `entityid` and + `service` settings, so it is not normally necessary to specify them + unless you need to override them. Here are a few useful sub-options + for configuring pysaml: + + * `metadata`: Point this to the IdP's metadata. You must provide + either a local file via the `local` attribute or (preferably) a URL + via the `remote` attribute. + + * `accepted_time_diff: 3`: Allowed clock difference in seconds between + the homeserver and IdP. Defaults to 0. + + * `service`: By default, the user has to go to our login page first. + If you'd like to allow IdP-initiated login, set `allow_unsolicited` to + true under `sp` in the `service` section. + default: null + config_path: + type: ["string", "null"] + description: Specify a separate pysaml2 configuration file. + default: null + saml_session_lifetime: + $ref: "#/$defs/duration" + description: >- + The lifetime of a SAML session. This defines how long a user has to + complete the authentication process, if `allow_unsolicited` is unset. + default: 15m + user_mapping_provider: + type: object + description: >- + Using this option, an external module can be provided as a custom + solution to mapping attributes returned from a saml provider onto a + matrix user. + properties: + module: + type: string + description: The custom module's class. + config: + type: object + description: >- + Custom configuration values for the module. Use the values + provided in the example if you are using the built-in + user_mapping_provider, or provide your own config values for a + custom class if you are using one. This section will be passed as + a Python dictionary to the module's `parse_config` method. The + built-in provider takes the following two options: + + * `mxid_source_attribute`: The SAML attribute (after mapping via + the attribute maps) to use to derive the Matrix ID from. It is + "uid" by default. Note: This used to be configured by the + `saml2_config.mxid_source_attribute option`. If that is still + defined, its value will be used instead. + + * `mxid_mapping`: The mapping system to use for mapping the saml + attribute onto a matrix ID. Options include: `hexencode` (which + maps unpermitted characters to `=xx`) and `dotreplace` (which + replaces unpermitted characters with `.`). The default is + `hexencode`. Note: This used to be configured by the + `saml2_config.mxid_mapping option`. If that is still defined, its + value will be used instead. + grandfathered_mxid_source_attribute: + type: string + description: >- + In previous versions of synapse, the mapping from SAML attribute to + MXID was always calculated dynamically rather than stored in a table. + For backwards-compatibility, we will look for `user_ids` matching such + a pattern before creating a new account. This setting controls the + SAML attribute which will be used for this backwards-compatibility + lookup. Typically it should be "uid", but if the attribute maps are + changed, it may be necessary to change it. + default: uid + attribute_requirements: + type: array + description: >- + It is possible to configure Synapse to only allow logins if SAML + attributes match particular values. The requirements can be listed + under `attribute_requirements` as shown in the example. All of the + listed attributes must match for the login to be permitted. Values can + be specified in a `one_of` list to allow multiple values for an + attribute. + items: + type: object + description: Item allowing a specific SAML attribute. + properties: + attribute: + type: string + description: SAML attribute for which to allow logins. + value: + type: string + description: Value the SAML attribute must match. + one_of: + type: array + description: List of values the SAML attribute must all match. + items: + type: string + required: + - attribute + idp_entityid: + type: ["string", "null"] + description: >- + If the metadata XML contains multiple IdP entities then the + `idp_entityid` option must be set to the entity to redirect users to. + Most deployments only have a single IdP entity and so should omit this + option. + default: null + examples: + - sp_config: + metadata: + local: + - saml2/idp.xml + remote: + - url: "https://our_idp/metadata.xml" + accepted_time_diff: 3 + service: + sp: + allow_unsolicited: true + description: + - My awesome SP + - en + name: + - Test SP + - en + ui_info: + display_name: + - lang: en + text: Display Name is the descriptive name of your service. + description: + - lang: en + text: >- + Description should be a short paragraph explaining the purpose + of the service. + information_url: + - lang: en + text: "https://example.com/terms-of-service" + privacy_statement_url: + - lang: en + text: "https://example.com/privacy-policy" + keywords: + - lang: en + text: + - Matrix + - Element + logo: + - lang: en + text: "https://example.com/logo.svg" + width: "200" + height: "80" + organization: + name: Example com + display_name: + - - Example co + - en + url: "http://example.com" + contact_person: + - given_name: Bob + sur_name: the Sysadmin + email_address: + - admin@example.com + contact_type: technical + saml_session_lifetime: 5m + user_mapping_provider: + config: + mxid_source_attribute: displayName + mxid_mapping: dotreplace + grandfathered_mxid_source_attribute: upn + attribute_requirements: + - attribute: userGroup + value: staff + - attribute: department + one_of: + - sales + - admins + idp_entityid: "https://our_idp/entityid" + oidc_providers: + type: array + description: >- + List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for + registration and login. See [here](../../openid.md) for information on how + to configure these options. + + + For backwards compatibility, it is also possible to configure a single + OIDC provider via an `oidc_config` setting. This is now deprecated and + admins are advised to migrate to the `oidc_providers` format. (When doing + that migration, use `oidc` for the `idp_id` to ensure that existing users + continue to be recognised.) + + + It is possible to configure Synapse to only allow logins if certain + attributes match particular values in the OIDC userinfo. The requirements + can be listed under `attribute_requirements` as shown here: + + ```yaml + + attribute_requirements: + - attribute: family_name + one_of: ["Stephensson", "Smith"] + - attribute: groups + value: "admin" + # If `value` or `one_of` are not specified, the attribute only needs + # to exist, regardless of value. + - attribute: picture + ``` + + + `attribute` is a required field, while `value` and `one_of` are optional. + + + All of the listed attributes must match for the login to be permitted. + Additional attributes can be added to userinfo by expanding the `scopes` + section of the OIDC config to retrieve additional information from the + OIDC provider. + + + If the OIDC claim is a list, then the attribute must match any value in + the list. Otherwise, it must exactly match the value of the claim. Using + the example above, the `family_name` claim MUST be either "Stephensson" or + "Smith", but the `groups` claim MUST contain "admin". + items: + type: object + properties: + idp_id: + type: string + description: >- + A unique identifier for this identity provider. Used internally by + Synapse; should be a single word such as "github". Note that, if + this is changed, users authenticating via that provider will no + longer be recognised as the same user! (Use "oidc" here if you are + migrating from an old `oidc_config` configuration.) + idp_name: + type: string + description: >- + A user-facing name for this identity provider, which is used to + offer the user a choice of login mechanisms. + idp_icon: + type: string + description: >- + An optional icon for this identity provider, which is presented by + clients and Synapse's own IdP picker page. If given, must be an MXC + URI of the format `mxc:///`. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) + room and then copy the URL from the source of the event.) + idp_brand: + type: string + description: >- + An optional brand for this identity provider, allowing clients to + style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options + here. + discover: + type: boolean + description: >- + Set to false to disable the use of the OIDC discovery mechanism to + discover endpoints. Defaults to true. + issuer: + type: string + description: >- + Required. The OIDC issuer. Used to validate tokens and (if discovery + is enabled) to discover the provider's endpoints. + client_id: + type: string + description: Required. OAuth2 client id to use. + client_secret: + type: ["string", "null"] + description: >- + OAuth2 client secret to use. May be omitted if + `client_secret_jwt_key` is given, or if `client_auth_method` is + `none`. Must be omitted if `client_secret_path` is specified. + client_secret_path: + type: ["string", "null"] + description: >- + Path to the OAuth2 client secret to use. With that it's not + necessary to leak secrets into the config file itself. Mutually + exclusive with `client_secret`. Can be omitted if + `client_secret_jwt_key` is specified. + + + *Added in Synapse 1.91.0.* + client_secret_jwt_key: + type: ["object", "null"] + description: >- + Alternative to client_secret: details of a key used to create a JSON + Web Token to be used as an OAuth2 client secret. + properties: + key: + type: ["string", "null"] + description: >- + A pem-encoded signing key. Must be a suitable key for the + algorithm specified. Required unless `key_file` is given. + key_file: + type: ["string", "null"] + description: >- + Path to the file containing a pem-encoded signing key. Required + unless `key` is given. + jwt_header: + type: object + description: >- + Dictionary giving properties to include in the JWT header. Must + include the key `alg`. + properties: + alg: + type: string + description: >- + Algorithm used to sign the JWT, such as ES256, using the JWA + identifiers in RFC7518. + jwt_payload: + type: object + description: >- + Optional dictionary giving properties to include in the JWT + payload. Normally this should include an `iss` key. + client_auth_method: + type: ["string", "null"] + enum: + - client_secret_basic + - client_secret_post + - none + - null + description: >- + Auth method to use when exchanging the token. Valid values are + `client_secret_basic` (default), `client_secret_post` and `none`. + pkce_method: + type: ["string", "null"] + enum: + - auto + - always + - never + - null + description: >- + Whether to use proof key for code exchange when requesting and + exchanging the token. Valid values are: `auto`, `always`, or + `never`. Defaults to `auto`, which uses PKCE if supported during + metadata discovery. Set to `always` to force enable PKCE or `never` + to force disable PKCE. + id_token_signing_alg_values_supported: + type: array + description: >- + List of the JWS signing algorithms (`alg` values) that are supported + for signing the `id_token`. + + + This is *not* required if `discovery` is disabled. We default to + supporting `RS256` in the downstream usage if no algorithms are + configured here or in the discovery document. + + + According to the spec, the algorithm `"RS256"` MUST be included. The + absolute rigid approach would be to reject this provider as + non-compliant if it's not included but we simply allow whatever and + see what happens (you're the one that configured the value and + cooperating with the identity provider). + + + The `alg` value `"none"` MAY be supported but can only be used if + the Authorization Endpoint does not include `id_token` in the + `response_type` (ex. `/authorize?response_type=code` where `none` + can apply, `/authorize?response_type=code%20id_token` where `none` + can't apply) (such as when using the Authorization Code Flow). + items: + type: string + scopes: + type: ["array", "null"] + description: >- + List of scopes to request. This should normally include the "openid" + scope. Defaults to `["openid"]`. + items: + type: string + authorization_endpoint: + type: string + description: >- + The OAuth2 authorization endpoint. Required if provider discovery is + disabled. + token_endpoint: + type: string + description: >- + The OAuth2 token endpoint. Required if provider discovery is disabled. + userinfo_endpoint: + type: string + description: >- + The OIDC userinfo endpoint. Required if discovery is disabled and + the "openid" scope is not requested. + jwks_uri: + type: string + description: >- + URI where to fetch the JWKS. Required if discovery is disabled and + the "openid" scope is used. + skip_verification: + type: boolean + description: >- + Set to `true` to skip metadata verification. Use this if you are + connecting to a provider that is not OpenID Connect compliant. + Defaults to false. Avoid this in production. + user_profile_method: + type: ["string", "null"] + enum: + - auto + - userinfo_endpoint + - null + description: >- + Whether to fetch the user profile from the userinfo endpoint, or to + rely on the data returned in the id_token from the `token_endpoint`. + Valid values are: `auto` or `userinfo_endpoint`. Defaults to `auto`, + which uses the userinfo endpoint if `openid` is not included in + `scopes`. Set to `userinfo_endpoint` to always use the userinfo + endpoint. + redirect_uri: + type: ["string", "null"] + description: >- + An optional string, that if set will override the `redirect_uri` + parameter sent in the requests to the authorization and token + endpoints. Useful if you want to redirect the client to another + endpoint as part of the OIDC login. Be aware that the client must + then call Synapse's OIDC callback URL + (`/_synapse/client/oidc/callback`) manually + afterwards. Must be a valid URL including scheme and path. + additional_authorization_parameters: + type: object + description: >- + String to string dictionary that will be passed as additional + parameters to the authorization grant URL. + additionalProperties: + type: string + passthrough_authorization_parameters: + type: array + description: >- + List of parameters that will be passed through from the redirect + endpoint to the authorization grant URL. + items: + type: string + allow_existing_users: + type: boolean + description: >- + Set to true to allow a user logging in via OIDC to match a + pre-existing account instead of failing. This could be used if + switching from password logins to OIDC. Defaults to false. + enable_registration: + type: boolean + description: >- + Set to `false` to disable automatic registration of new users. This + allows the OIDC SSO flow to be limited to sign in only, rather than + automatically registering users that have a valid SSO login but do + not have a pre-registered account. Defaults to true. + user_mapping_provider: + type: object + description: >- + Configuration for how attributes returned from a OIDC provider are + mapped onto a matrix user. + + + When rendering, the Jinja2 templates are given a `user` variable, + which is set to the claims returned by the UserInfo Endpoint and/or + in the ID Token. + properties: + module: + type: string + description: >- + The class name of a custom mapping module. Default is + `synapse.handlers.oidc.JinjaOidcMappingProvider`. See [OpenID + Mapping + Providers](../../sso_mapping_providers.md#openid-mapping-providers) + for information on implementing a custom mapping provider. + config: + type: object + description: >- + Configuration for the mapping provider module. This section will + be passed as a Python dictionary to the user mapping provider + module's `parse_config` method. + + + For the default provider, the following settings are available: + + + * `subject_template`: Jinja2 template for a unique identifier + for the user. Defaults to `{{ user.sub }}`, which OpenID Connect + compliant providers should provide. + + This replaces and overrides `subject_claim`. + + * `subject_claim`: name of the claim containing a unique + identifier for the user. Defaults to `sub`, which OpenID Connect + compliant providers should provide. + + *Deprecated in Synapse v1.75.0.* + + * `picture_template`: Jinja2 template for an url for the user's + profile picture. Defaults to `{{ user.picture }}`, which OpenID + Connect compliant providers should provide and has to refer to a + direct image file such as PNG, JPEG, or GIF image file. + + This replaces and overrides `picture_claim`. + + Currently only supported in monolithic (single-process) server configurations where the media repository runs within the Synapse process. + + * `picture_claim`: name of the claim containing an url for the + user's profile picture. Defaults to "picture", which OpenID + Connect compliant providers should provide and has to refer to a + direct image file such as PNG, JPEG, or GIF image file. + + Currently only supported in monolithic (single-process) server configurations where the media repository runs within the Synapse process. + + *Deprecated in Synapse v1.75.0.* + + * `localpart_template`: Jinja2 template for the localpart of the + MXID. If this is not set, the user will be prompted to choose + their own username (see the documentation for the + `sso_auth_account_details.html` template). This template can use + the `localpart_from_email` filter. + + + * `confirm_localpart`: Whether to prompt the user to validate + (or change) the generated localpart (see the documentation for + the "sso_auth_account_details.html" template), instead of + registering the account right away. + + + * `display_name_template`: Jinja2 template for the display name + to set on first login. If unset, no displayname will be set. + + + * `email_template`: Jinja2 template for the email address of the + user. If unset, no email address will be added to the account. + + + * `extra_attributes`: a map of Jinja2 templates for extra + attributes to send back to the client during login. Note that + these are non-standard and clients will ignore them without + modifications. + backchannel_logout_enabled: + type: boolean + description: >- + Set to `true` to process OIDC Back-Channel Logout notifications. + Those notifications are expected to be received on + `/_synapse/client/oidc/backchannel_logout`. Defaults to `false`. + backchannel_logout_ignore_sub: + type: boolean + description: >- + By default, the OIDC Back-Channel Logout feature checks that the + `sub` claim matches the subject claim received during login. This + check can be disabled by setting this to `true`. Defaults to + `false`. + + + You might want to disable this if the `subject_claim` returned by + the mapping provider is not `sub`. + default: [] + examples: + - - idp_id: my_idp + idp_name: My OpenID provider + idp_icon: "mxc://example.com/mediaid" + discover: false + issuer: "https://accounts.example.com/" + client_id: provided-by-your-issuer + client_secret: provided-by-your-issuer + client_auth_method: client_secret_post + scopes: + - openid + - profile + authorization_endpoint: "https://accounts.example.com/oauth2/auth" + token_endpoint: "https://accounts.example.com/oauth2/token" + userinfo_endpoint: "https://accounts.example.com/userinfo" + jwks_uri: "https://accounts.example.com/.well-known/jwks.json" + additional_authorization_parameters: + acr_values: 2fa + passthrough_authorization_parameters: + - login_hint + skip_verification: true + enable_registration: true + user_mapping_provider: + config: + subject_claim: id + localpart_template: "{{ user.login }}" + display_name_template: "{{ user.name }}" + email_template: "{{ user.email }}" + attribute_requirements: + - attribute: userGroup + value: synapseUsers + cas_config: + type: object + description: Enable Central Authentication Service (CAS) for registration and login. + properties: + enabled: + type: boolean + description: Set this to true to enable authorization against a CAS server. + default: false + idp_name: + type: string + description: >- + A user-facing name for this identity provider, which is used to offer + the user a choice of login mechanisms. + idp_icon: + type: ["string", "null"] + description: >- + An optional icon for this identity provider, which is presented by + clients and Synapse's own IdP picker page. If given, must be an MXC + URI of the format `mxc:///`. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) room + and then copy the URL from the source of the event.) + default: null + idp_brand: + type: ["string", "null"] + description: >- + An optional brand for this identity provider, allowing clients to + style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options + here. + default: null + server_url: + type: string + description: The URL of the CAS authorization endpoint. + protocol_version: + type: ["integer", "null"] + description: >- + The CAS protocol version. (Version 3 is required if you want to use + `required_attributes`). + default: null + displayname_attribute: + type: ["string", "null"] + description: >- + The attribute of the CAS response to use as the display name. If no + name is given here, no displayname will be set. + default: null + required_attributes: + type: object + description: >- + It is possible to configure Synapse to only allow logins if CAS + attributes match particular values. All of the keys given below must + exist and the values must match the given value. Alternately if the + given value is `None` then any value is allowed (the attribute just + must exist). All of the listed attributes must match for the login to + be permitted. + additionalProperties: + type: ["string", "null"] + default: {} + enable_registration: + type: boolean + description: >- + Set to `false` to disable automatic registration of new users. This + allows the CAS SSO flow to be limited to sign in only, rather than + automatically registering users that have a valid SSO login but do not + have a pre-registered account. + default: true + allow_numeric_ids: + type: boolean + description: >- + Set to `true` allow numeric user IDs. This allows CAS SSO flow to + provide user IDs composed of numbers only. These identifiers will be + prefixed by the letter "u" by default. The prefix can be configured + using the `numeric_ids_prefix` option. Be careful to choose the prefix + correctly to avoid any possible conflicts (e.g. user 1234 becomes + u1234 when a user u1234 already exists). + default: false + numeric_ids_prefix: + type: string + description: >- + The prefix you wish to add in front of a numeric user ID when the + `allow_numeric_ids` option is set to `true`. Only alphanumeric + characters are allowed. + + + *Added in Synapse 1.93.0.* + default: u + examples: + - enabled: true + server_url: "https://cas-server.com" + protocol_version: 3 + displayname_attribute: name + required_attributes: + userGroup: staff + department: None + enable_registration: true + allow_numeric_ids: true + numeric_ids_prefix: numericuser + sso: + type: object + description: >- + Additional settings to use with single-sign on systems such as OpenID + Connect, SAML2 and CAS. + + + Server admins can configure custom templates for pages related to SSO. See + [here](../../templates.md) for more information. + properties: + client_whitelist: + type: ["array", "null"] + description: >- + A list of client URLs which are whitelisted so that the user does not + have to confirm giving access to their account to the URL. Any client + whose URL starts with an entry in the following list will not be + subject to an additional confirmation step after the SSO login is + completed. + + + WARNING: An entry such as "https://my.client" is insecure, because it + will also match "https://my.client.evil.site", exposing your users to + phishing attacks from evil.site. To avoid this, include a slash after + the hostname: "https://my.client/". + + + The login fallback page (used by clients that don't natively support + the required login flows) is whitelisted in addition to any URLs in + this list. By default, this list contains only the login fallback + page. + items: + type: string + default: null + update_profile_information: + type: boolean + description: >- + Use this setting to keep a user's profile fields in sync with + information from the identity provider. Currently only syncing the + displayname is supported. Fields are checked on every SSO login, and + are updated if necessary. Note that enabling this option will override + user profile information, regardless of whether users have opted-out + of syncing that information when first signing in. + default: false + examples: + - client_whitelist: + - "https://riot.im/develop" + - "https://my.custom.client/" + update_profile_information: true + jwt_config: + type: object + description: >- + JSON web token integration. The following settings can be used to make + Synapse JSON web tokens for authentication, instead of its internal + password database. + + + Each JSON Web Token needs to contain a "sub" (subject) claim, which is + used as the localpart of the mxid. + + + Additionally, the expiration time ("exp"), not before time ("nbf"), and + issued at ("iat") claims are validated if present. + + + Note that this is a non-standard login type and client support is expected + to be non-existent. + + + See [here](../../jwt.md) for more. + properties: + enabled: + type: boolean + description: Set to true to enable authorization using JSON web tokens. + default: false + secret: + type: string + description: >- + This is either the private shared secret or the public key used to + decode the contents of the JSON web token. Required if `enabled` is + set to true. + algorithm: + type: string + description: >- + The algorithm used to sign (or HMAC) the JSON web token. Supported + algorithms are listed [here (section + JWS)](https://docs.authlib.org/en/latest/specs/rfc7518.html). Required + if `enabled` is set to true. + subject_claim: + type: ["string", "null"] + description: Name of the claim containing a unique identifier for the user. + default: sub + display_name_claim: + type: ["string", "null"] + description: >- + Name of the claim containing the display name for the user. If + provided, the display name will be set to the value of this claim upon + first login. + default: null + issuer: + type: ["string", "null"] + description: >- + The issuer to validate the "iss" claim against. If provided the "iss" + claim will be required and validated for all JSON web tokens. + default: null + audiences: + type: ["array", "null"] + description: >- + A list of audiences to validate the "aud" claim against. If provided + the "aud" claim will be required and validated for all JSON web + tokens. Note that if the "aud" claim is included in a JSON web token + then validation will fail without configuring audiences. + items: + type: string + default: null + examples: + - enabled: true + secret: provided-by-your-issuer + algorithm: provided-by-your-issuer + subject_claim: name_of_claim + display_name_claim: name_of_claim + issuer: provided-by-your-issuer + audiences: + - provided-by-your-issuer + password_config: + type: object + description: Use this setting to enable password-based logins. + properties: + enabled: + type: ["boolean", "string"] + enum: + - true + - false + - only_for_reauth + description: >- + Set to false to disable password authentication. Set to + `only_for_reauth` to allow users with existing passwords to use them + to reauthenticate (not log in), whilst preventing new users from + setting passwords. + default: true + localdb_enabled: + type: boolean + description: >- + Set to false to disable authentication against the local password + database. This is ignored if `enabled` is false, and is only useful if + you have other `password_providers`. + default: true + pepper: + type: ["string", "null"] + description: >- + Set the value here to a secret random string for extra security. DO + NOT CHANGE THIS AFTER INITIAL SETUP! + default: null + policy: + type: object + description: >- + Define and enforce a password policy, such as minimum lengths for + passwords, etc. This is an implementation of MSC2000. + properties: + enabled: + type: boolean + description: Set to true to enable. + default: false + minimum_length: + type: integer + description: Minimum accepted length for a password. + default: 0 + require_digit: + type: boolean + description: Whether a password must contain at least one digit. + default: false + require_symbol: + type: boolean + description: >- + Whether a password must contain at least one symbol. A symbol is + any character that's not a number or a letter. + default: false + require_lowercase: + type: boolean + description: Whether a password must contain at least one lowercase letter. + default: false + require_uppercase: + type: boolean + description: Whether a password must contain at least one uppercase letter. + default: false + examples: + - enabled: false + localdb_enabled: false + pepper: EVEN_MORE_SECRET + policy: + enabled: true + minimum_length: 15 + require_digit: true + require_symbol: true + require_lowercase: true + require_uppercase: true + push: + type: object + description: This setting defines options for push notifications. + properties: + enabled: + type: boolean + description: >- + Enables or disables push notification calculation. Note, disabling + this will also stop unread counts being calculated for rooms. This + mode of operation is intended for homeservers which may only have bots + or appservice users connected, or are otherwise not interested in + push/unread counters. + default: true + include_content: + type: boolean + description: >- + Clients requesting push notifications can either have the body of the + message sent in the notification poke along with other details like + the sender, or just the event ID and room ID (`event_id_only`). If + clients choose to have the body sent, this option controls whether the + notification request includes the content of the event (other details + like the sender are still included). If `event_id_only` is enabled, it + has no effect. For modern Android devices the notification content + will still appear because it is loaded by the app. iPhone, however + will send a notification saying only that a message arrived and who it + came from. Set to false to only include the event ID and room ID in + push notification payloads. + default: true + group_unread_count_by_room: + type: boolean + description: >- + When a push notification is received, an unread count is also sent. + This number can either be calculated as the number of unread messages + for the user, or the number of *rooms* the user has unread messages + in. If true, push clients will see the number of rooms with unread + messages in them. Set to false to instead send the number of unread + messages. + default: true + jitter_delay: + $ref: "#/$defs/duration" + description: >- + Delays push notifications by a random amount up to the given duration. + Useful for mitigating timing attacks. Optional. + + + _Added in Synapse 1.84.0._ + default: 0s + examples: + - enabled: true + include_content: false + group_unread_count_by_room: false + jitter_delay: 10s + encryption_enabled_by_default_for_room_type: + type: string + description: >- + Controls whether locally-created rooms should be end-to-end encrypted by + default. + + + Possible options are "all", "invite", and "off". They are defined as: + + + * "all": any locally-created room + + * "invite": any room created with the `private_chat` or + `trusted_private_chat` room creation presets + + * "off": this option will take no effect + + + Note that this option will only affect rooms created after it is set. It + will also not affect rooms created by other servers. + enum: + - all + - invite + - "off" + default: "off" + examples: + - invite + user_directory: + type: object + description: This setting defines options related to the user directory. + properties: + enabled: + type: boolean + description: >- + Defines whether users can search the user directory. If `false` then + empty responses are returned to all queries. + + + *Warning: While the homeserver may determine which subset of users are + searched, the Matrix specification requires homeservers to include (at + minimum) users visible in public rooms and users sharing a room with + the requester. Using `false` improves performance but violates this + requirement.* + default: true + search_all_users: + type: boolean + description: >- + Defines whether to search all users visible to your homeserver at the + time the search is performed. If set to true, will return all users + known to the homeserver matching the search query. If false, search + results will only contain users visible in public rooms and users + sharing a room with the requester. + + + NB. If you set this to true, and the last time the user_directory + search indexes were (re)built was before Synapse 1.44, you'll have to + rebuild the indexes in order to search through all known users. + + + These indexes are built the first time Synapse starts; admins can + manually trigger a rebuild via the API following the instructions [for + running background + updates](../administration/admin_api/background_updates.md#run), set + to true to return search results containing all known users, even if + that user does not share a room with the requester. + default: false + prefer_local_users: + type: boolean + description: >- + Defines whether to prefer local users in search query results. If set + to true, local users are more likely to appear above remote users when + searching the user directory. + default: false + exclude_remote_users: + type: boolean + description: If set to true, the search will only return local users. + default: false + show_locked_users: + type: boolean + description: Defines whether to show locked users in search query results. + default: false + examples: + - enabled: false + search_all_users: true + prefer_local_users: true + exclude_remote_users: false + show_locked_users: true + user_consent: + type: object + description: >- + For detailed instructions on user consent configuration, see + [here](../../consent_tracking.md). + + + Parts of this section are required if enabling the `consent` resource + under [`listeners`](#listeners), in particular `template_dir` and + `version`. + properties: + template_dir: + type: string + description: >- + Gives the location of the templates for the HTML forms. This directory + should contain one subdirectory per language (eg, `en`, `fr`), and + each language directory should contain the policy document (named as + .html) and a success page (success.html). + version: + type: number + description: >- + Specifies the "current" version of the policy document. It defines the + version to be served by the consent resource if there is no `v` + parameter. + server_notice_content: + type: object + description: >- + If enabled, will send a user a "Server Notice" asking them to consent + to the privacy policy. The [`server_notices` section](#server_notices) + must also be configured for this to work. Notices will *not* be sent + to guest users unless `send_server_notice_to_guests` is set to true. + properties: + msgtype: + type: string + description: Message type of the notice event. + body: + type: string + description: Message template for the server notice event body. + send_server_notice_to_guests: + type: boolean + description: Send server notices to guest users, too. + default: false + block_events_error: + type: ["string", "null"] + description: >- + If set, will block any attempts to send events until the user consents + to the privacy policy. The value of the setting is used as the text of + the error. + default: null + require_at_registration: + type: boolean + description: >- + If enabled, will add a step to the registration process, similar to + how captcha works. Users will be required to accept the policy before + their account is created. + policy_name: + type: string + description: Human-readable name of the privacy policy. + default: Privacy Policy + examples: + - template_dir: res/templates/privacy + version: 1.0 + server_notice_content: + msgtype: m.text + body: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + send_server_notice_to_guests: true + block_events_error: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + require_at_registration: false + policy_name: Privacy Policy + stats: + type: object + description: >- + Settings for local room and user statistics collection. See + [here](../../room_and_user_statistics.md) for more. + properties: + enabled: + type: boolean + description: >- + Set to false to disable room and user statistics. Note that doing so + may cause certain features (such as the room directory) not to work + correctly. + default: true + examples: + - enabled: false + server_notices: + type: object + description: >- + Use this setting to enable a room which can be used to send notices from + the server to users. It is a special room which users cannot leave; + notices in the room come from a special "notices" user id. + + + If you use this setting, you *must* define the `system_mxid_localpart` + sub-setting, which defines the id of the user which will be used to send + the notices. + + + Note that the name, topic and avatar of existing server notice rooms will + only be updated when a new notice event is sent. + properties: + system_mxid_display_name: + type: string + description: Display name of the "notices" user. + default: Notices + system_mxid_avatar_url: + type: ["string", "null"] + description: Avatar for the "notices" user. + default: null + room_name: + type: string + description: Room name of the server notices room. + default: Server Notices + room_avatar_url: + type: ["string", "null"] + description: >- + Room avatar to use for server notice rooms. If set to the empty string + `""`, notice rooms will not be given an avatar. + + + _Added in Synapse 1.99.0._ + default: null + room_topic: + type: ["string", "null"] + description: >- + Topic to use for server notice rooms. If set to the empty string `""`, + notice rooms will not be given a topic. Defaults to the empty string. + + + _Added in Synapse 1.99.0._ + default: null + auto_join: + type: boolean + description: >- + If true, the user will be automatically joined to the room instead of + being invited. + + + _Added in Synapse 1.98.0._ + default: false + examples: + - system_mxid_localpart: notices + system_mxid_display_name: Server Notices + system_mxid_avatar_url: "mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ" + room_name: Server Notices + room_avatar_url: "mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ" + room_topic: >- + Room used by your server admin to notice you of important information + auto_join: true + enable_room_list_search: + type: boolean + description: >- + Set to false to disable searching the public room list. When disabled + blocks searching local and remote room lists for local and remote users by + always returning an empty list for all queries. + default: true + examples: + - false + alias_creation_rules: + type: ["array", "null"] + description: >- + The `alias_creation_rules` option allows server admins to prevent unwanted + alias creation on this server. + + + This setting is an optional list of 0 or more rules. By default, no list + is provided, meaning that all alias creations are permitted. + + + Otherwise, requests to create aliases are matched against each rule in + order. The first rule that matches decides if the request is allowed or + denied. If no rule matches, the request is denied. In particular, this + means that configuring an empty list of rules will deny every alias + creation request. + + + Each of the glob patterns is optional, defaulting to `*` ("match + anything"). Note that the patterns match against fully qualified IDs, e.g. + against `@alice:example.com`, `#room:example.com` and + `!abcdefghijk:example.com` instead of `alice`, `room` and `abcedgghijk`. + + + Each rule is a YAML object containing four fields, each of which is an + optional string + items: + type: object + properties: + user_id: + type: ["string", "null"] + description: Glob pattern that matches against the creator of the alias. + alias: + type: ["string", "null"] + description: Glob pattern that matches against the alias being created. + room_id: + type: ["string", "null"] + description: >- + Glob pattern that matches against the room ID the alias is being pointed at. + action: + type: string + enum: + - allow + - deny + description: >- + Either `allow` or `deny`. What to do with the request if the rule + matches. Defaults to `allow`. + default: null + examples: + - null + - - action: allow + - [] + - - action: deny + - - user_id: "@bad_user:example.com" + action: deny + - action: allow + - - room_id: "!forbiddenRoom:example.com" + action: deny + - action: allow + room_list_publication_rules: + type: ["array", "null"] + description: >- + The `room_list_publication_rules` option allows server admins to prevent + unwanted entries from being published in the public room list. + + + The format of this option is the same as that for + [`alias_creation_rules`](#alias_creation_rules): an optional list of 0 or + more rules. By default, no list is provided, meaning that no one may + publish to the room list (except server admins). + + + Otherwise, requests to publish a room are matched against each rule in + order. The first rule that matches decides if the request is allowed or + denied. If no rule matches, the request is denied. In particular, this + means that configuring an empty list of rules will deny every alias + creation request. + + + Requests to create a public (public as in published to the room directory) + room which violates the configured rules will result in the room being + created but not published to the room directory. + + + Each of the glob patterns is optional, defaulting to `*` ("match + anything"). Note that the patterns match against fully qualified IDs, e.g. + against `@alice:example.com`, `#room:example.com` and + `!abcdefghijk:example.com` instead of `alice`, `room` and `abcedgghijk`. + + + Each rule is a YAML object containing four fields, each of which is an + optional string. + + + _Changed in Synapse 1.126.0: The default was changed to deny publishing to + the room list by default_ + items: + type: object + properties: + user_id: + type: ["string", "null"] + description: Glob pattern that matches against the user publishing the room. + alias: + type: ["string", "null"] + description: >- + Glob pattern that matches against one of published room's aliases. + + - If the room has no aliases, the alias match fails unless `alias` + is unspecified or `*`. + + - If the room has exactly one alias, the alias match succeeds if the + `alias` pattern matches that alias. + + - If the room has two or more aliases, the alias match succeeds if + the pattern matches at least one of the aliases. + room_id: + type: ["string", "null"] + description: >- + Glob pattern that matches against the room ID of the room being published. + action: + type: string + enum: + - allow + - deny + description: >- + Either `allow` or `deny`. What to do with the request if the rule + matches. Defaults to `allow`. + default: null + examples: + - null + - - action: deny + - [] + - - action: allow + - - user_id: "@bad_user:example.com" + action: deny + - action: allow + - - room_id: "!forbiddenRoom:example.com" + action: deny + - action: allow + - - alias: "#*potato*:example.com" + action: deny + - action: allow + default_power_level_content_override: + type: object + description: >- + The `default_power_level_content_override` option controls the default + power levels for rooms. + + + Useful if you know that your users need special permissions in rooms that + they create (e.g. to send particular types of state events without needing + an elevated power level). This takes the same shape as the + `power_level_content_override` parameter in the /createRoom API, but is + applied before that parameter. + + + Note that each key provided inside a preset (for example `events` in the + example below) will overwrite all existing defaults inside that key. So in + Example #1, newly-created private_chat rooms will have no rules for any + event types except `com.example.foo`. + + + The default power levels for each preset are: + + + ```yaml + + "m.room.name": 50 + + "m.room.power_levels": 100 + + "m.room.history_visibility": 100 + + "m.room.canonical_alias": 50 + + "m.room.avatar": 50 + + "m.room.tombstone": 100 (150 if MSC4289 is used) + + "m.room.server_acl": 100 + + "m.room.encryption": 100 + + ``` + + + In Example #2 the default power-levels for a preset are maintained, but + the power level for a new key is set. + default: {} + examples: + - private_chat: + events: + com.example.foo: 0 + trusted_private_chat: null + public_chat: null + - private_chat: + events: + com.example.foo: 0 + m.room.name: 50 + m.room.power_levels: 100 + m.room.history_visibility: 100 + m.room.canonical_alias: 50 + m.room.avatar: 50 + m.room.tombstone: 100 + m.room.server_acl: 100 + m.room.encryption: 100 + trusted_private_chat: null + public_chat: null + forget_rooms_on_leave: + type: boolean + description: >- + Set to true to automatically forget rooms for users when they leave them, + either normally or via a kick or ban. + default: false + examples: + - true + exclude_rooms_from_sync: + type: array + description: >- + A list of rooms to exclude from sync responses. This is useful for server + administrators wishing to group users into a room without these users + being able to see it from their client. + items: + type: string + default: [] + examples: + - - "!foo:example.com" + opentracing: + type: object + description: >- + These settings enable and configure opentracing, which implements + distributed tracing. This allows you to observe the causal chains of + events across servers including requests, key lookups etc., across any + server running synapse or any other services which support opentracing + (specifically those implemented with Jaeger). + properties: + enabled: + type: boolean + description: Whether tracing is enabled. Set to true to enable. + default: false + homeserver_whitelist: + type: array + description: >- + The list of homeservers we wish to send and receive span contexts and + span baggage. See [here](../../opentracing.md) for more. This is a + list of regexes which are matched against the `server_name` of the + homeserver. If the list is empty, no servers are matched. + items: + type: string + default: [] + force_tracing_for_users: + type: array + description: >- + A list of the matrix IDs of users whose requests will always be + traced, even if the tracing system would otherwise drop the traces due + to probabilistic sampling. + items: + type: string + default: [] + jaeger_config: + type: object + description: >- + Jaeger can be configured to sample traces at different rates. All + configuration options provided by Jaeger can be set here. Jaeger's + configuration is mostly related to trace sampling which is documented + [here](https://www.jaegertracing.io/docs/latest/sampling/). + default: {} + examples: + - enabled: true + homeserver_whitelist: + - ".*" + force_tracing_for_users: + - "@user1:server_name" + - "@user2:server_name" + jaeger_config: + sampler: + type: const + param: 1 + logging: false + worker_replication_secret: + type: ["string", "null"] + description: >- + A shared secret used by the replication APIs on the main process to + authenticate HTTP requests from workers. + + + If unset or null, traffic between the workers and the main process is not + authenticated. + + + Replacing an existing `worker_replication_secret` with a new one will + break communication with all workers that have not yet updated their + secret. + default: null + examples: + - secret_secret + worker_replication_secret_path: + type: ["string", "null"] + description: >- + An alternative to + [`worker_replication_secret`](#worker_replication_secret): allows the + secret to be specified in an external file. + + + The file should be a plain text file, containing only the secret. Synapse + reads the secret from the given file once at startup. + + + _Added in Synapse 1.126.0._ + default: null + examples: + - /path/to/secrets/file + start_pushers: + type: boolean + description: >- + Unnecessary to set if using [`pusher_instances`](#pusher_instances) with + [`generic_workers`](../../workers.md#synapseappgeneric_worker). + + + Controls sending of push notifications on the main process. Set to `false` + if using a [pusher worker](../../workers.md#synapseapppusher). + default: true + examples: + - false + pusher_instances: + type: array + description: >- + It is possible to scale the processes that handle sending push + notifications to [sygnal](https://github.com/matrix-org/sygnal) and email + by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) + and adding it's [`worker_name`](#worker_name) to a `pusher_instances` map. + Doing so will remove handling of this function from the main process. + Multiple workers can be added to this map, in which case the work is + balanced across them. Ensure the main process and all pusher workers are + restarted after changing this option. + items: + type: string + default: [] + examples: + - - pusher_worker1 + - - pusher_worker1 + - pusher_worker2 + send_federation: + type: boolean + description: >- + Unnecessary to set if using + [`federation_sender_instances`](#federation_sender_instances) with + [`generic_workers`](../../workers.md#synapseappgeneric_worker). + + + Controls sending of outbound federation transactions on the main process. + Set to `false` if using a [federation sender + worker](../../workers.md#synapseappfederation_sender). + default: true + examples: + - false + federation_sender_instances: + type: array + description: >- + It is possible to scale the processes that handle sending outbound + federation requests by running a + [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding + it's [`worker_name`](#worker_name) to a `federation_sender_instances` map. + Doing so will remove handling of this function from the main process. + Multiple workers can be added to this map, in which case the work is + balanced across them. + + + The way that the load balancing works is any outbound federation request + will be assigned to a federation sender worker based on the hash of the + destination server name. This means that all requests being sent to the + same destination will be processed by the same worker instance. Multiple + `federation_sender_instances` are useful if there is a federation with + multiple servers. + + + This configuration setting must be shared between all workers handling + federation sending, and if changed all federation sender workers must be + stopped at the same time and then started, to ensure that all instances + are running with the same config (otherwise events may be dropped). + items: + type: string + default: [] + examples: + - - federation_sender1 + - - federation_sender1 + - federation_sender2 + instance_map: + type: object + description: >- + When using workers this should be a map from [`worker_name`](#worker_name) + to the HTTP replication listener of the worker, if configured, and to the + main process. Each worker declared under + [`stream_writers`](../../workers.md#stream-writers) and + [`outbound_federation_restricted_to`](#outbound_federation_restricted_to) + needs a HTTP replication listener, and that listener should be included in + the `instance_map`. The main process also needs an entry on the + `instance_map`, and it should be listed under `main` **if even one other + worker exists**. Ensure the port matches with what is declared inside the + `listener` block for a `replication` listener. + additionalProperties: + type: object + default: {} + examples: + - main: + host: localhost + port: 8030 + worker1: + host: localhost + port: 8034 + other: + host: localhost + port: 8035 + tls: true + - main: + path: /run/synapse/main_replication.sock + worker1: + path: /run/synapse/worker1_replication.sock + stream_writers: + type: object + description: >- + Experimental: When using workers you can define which workers should + handle writing to streams such as event persistence and typing + notifications. Any worker specified here must also be in the + [`instance_map`](#instance_map). + + + See the list of available streams in the [worker + documentation](../../workers.md#stream-writers). + properties: + events: + type: string + description: Name of a worker assigned to the `events` stream. + typing: + type: string + description: Name of a worker assigned to the `typing` stream. + to_device: + type: string + description: Name of a worker assigned to the `to_device` stream. + account_data: + type: string + description: Name of a worker assigned to the `account_data` stream. + receipts: + type: string + description: Name of a worker assigned to the `receipts` stream. + presence: + type: string + description: Name of a worker assigned to the `presence` stream. + push_rules: + type: string + description: Name of a worker assigned to the `push_rules` stream. + device_lists: + type: string + description: Name of a worker assigned to the `device_lists` stream. + default: {} + examples: + - events: worker1 + typing: worker1 + outbound_federation_restricted_to: + type: array + description: >- + When using workers, you can restrict outbound federation traffic to only + go through a specific subset of workers. Any worker specified here must + also be in the [`instance_map`](#instance_map). + [`worker_replication_secret`](#worker_replication_secret) must also be + configured to authorize inter-worker communication. + + + Also see the [worker + documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers) + for more info. + + + _Added in Synapse 1.89.0._ + items: + type: string + default: [] + examples: + - - federation_sender1 + - federation_sender2 + run_background_tasks_on: + type: ["string", "null"] + description: >- + The [worker](../../workers.md#background-tasks) that is used to run + background tasks (e.g. cleaning up expired data). If not provided this + defaults to the main process. + default: null + examples: + - worker1 + update_user_directory_from_worker: + type: ["string", "null"] + description: >- + The [worker](../../workers.md#updating-the-user-directory) that is used to + update the user directory. If not provided this defaults to the main + process. + + + _Added in Synapse 1.59.0._ + default: null + examples: + - worker1 + notify_appservices_from_worker: + type: ["string", "null"] + description: >- + The [worker](../../workers.md#notifying-application-services) that is used + to send output traffic to Application Services. If not provided this + defaults to the main process. + + + _Added in Synapse 1.59.0._ + default: null + examples: + - worker1 + media_instance_running_background_jobs: + type: ["string", "null"] + description: >- + The [worker](../../workers.md#synapseappmedia_repository) that is used to + run background tasks for media repository. If running multiple media + repositories you must configure a single instance to run the background + tasks. If not provided this defaults to the main process or your single + `media_repository` worker. + + + _Added in Synapse 1.16.0._ + default: null + examples: + - worker1 + redis: + type: object + description: >- + Configuration for Redis when using workers. This *must* be enabled when + using workers. + + + _Added in Synapse 1.78.0._ + + + _Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, + private\_key\_file, ca\_file and ca\_path attributes_ + + + _Changed in Synapse 1.85.0: Added path option to use a local Unix socket_ + + + _Changed in Synapse 1.116.0: Added password\_path_ + properties: + enabled: + type: boolean + description: Whether to use Redis support. + default: false + host: + type: string + description: Optional host to use to connect to Redis. + default: localhost + port: + type: integer + description: Optional port to use to connect to Redis. + default: 6379 + path: + type: string + description: >- + The full path to a local Unix socket file. **If this is used, `host` + and `port` are ignored.** + default: /tmp/redis.sock + password: + type: ["string", "null"] + description: Optional password if configured on the Redis instance. + default: null + password_path: + type: ["string", "null"] + description: >- + Alternative to `password`, reading the password from an external file. + The file should be a plain text file, containing only the password. + Synapse reads the password from the given file once at startup. + default: null + dbid: + type: ["string", "null"] + description: >- + Optional redis dbid if needs to connect to specific redis logical db. + default: null + use_tls: + type: boolean + description: Whether to use a TLS connection. + default: false + certificate_file: + type: ["string", "null"] + description: Optional path to the certificate file. + default: null + private_key_file: + type: ["string", "null"] + description: Optional path to the private key file. + default: null + ca_file: + type: ["string", "null"] + description: >- + Optional path to the CA certificate file. Use this one or `ca_path` + default: null + ca_path: + type: ["string", "null"] + description: >- + Optional path to the folder containing the CA certificate file. Use + this one or `ca_file` + default: null + examples: + - enabled: true + host: localhost + port: 6379 + password_path: "" + dbid: "" + worker_app: + type: string + description: >- + The type of worker. The currently available worker applications are listed + in [worker documentation](../../workers.md#available-worker-applications). + + + The most common worker is the + [`synapse.app.generic_worker`](../../workers.md#synapseappgeneric_worker). + examples: + - synapse.app.generic_worker + worker_name: + type: string + description: >- + A unique name for the worker. The worker needs a name to be addressed in + further parameters and identification in log files. We strongly recommend + giving each worker a unique `worker_name`. + examples: + - generic_worker1 + worker_listeners: + type: array + description: >- + A worker can handle HTTP requests. To do so, a `worker_listeners` option + must be declared, in the same way as the [`listeners` option](#listeners) + in the shared config. + + + Workers declared in [`stream_writers`](#stream_writers) and + [`instance_map`](#instance_map) will need to include a `replication` + listener here, in order to accept internal HTTP requests from other + workers. + + + Example #2 is using UNIX sockets with a `replication` listener. + default: [] + examples: + - - type: http + port: 8083 + resources: + - names: + - client + - federation + - - type: http + path: /run/synapse/worker_replication.sock + resources: + - names: + - replication + - type: http + path: /run/synapse/worker_public.sock + resources: + - names: + - client + - federation + worker_manhole: + type: ["integer", "null"] + description: >- + A worker may have a listener for [`manhole`](../../manhole.md). It allows + server administrators to access a Python shell on the worker. + + + The example below is a short form for + + ```yaml + + worker_listeners: + - port: 9000 + bind_addresses: ['127.0.0.1'] + type: manhole + ``` + + + It needs also an additional [`manhole_settings`](#manhole_settings) + configuration. + default: null + examples: + - 9000 + worker_daemonize: + type: boolean + description: >- + Specifies whether the worker should be started as a daemon process. If + Synapse is being managed by [systemd](../../systemd-with-workers/), this + option must be omitted or set to `false`. + default: false + examples: + - true + worker_pid_file: + type: ["string", "null"] + description: >- + When running a worker as a daemon, we need a place to store the + [PID](https://en.wikipedia.org/wiki/Process_identifier) of the worker. + This option defines the location of that "pid file". + + + This option is required if `worker_daemonize` is `true` and ignored + otherwise. + + + See also the [`pid_file` option](#pid_file) option for the main Synapse + process. + default: null + examples: + - DATADIR/generic_worker1.pid + worker_log_config: + type: ["string", "null"] + description: >- + This option specifies a yaml python logging config file as described + [here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). + See also the [`log_config` option](#log_config) option for the main + Synapse process. + default: null + examples: + - /etc/matrix-synapse/generic-worker-log.yaml + background_updates: + type: object + description: >- + Background updates are database updates that are run in the background in + batches. The duration, minimum batch size, default batch size, whether to + sleep between batches and if so, how long to sleep can all be configured. + This is helpful to speed up or slow down the updates. + properties: + background_update_duration_ms: + type: integer + description: How long in milliseconds to run a batch of background updates for. + default: 100 + sleep_enabled: + type: boolean + description: Whether to sleep between updates. + default: true + sleep_duration_ms: + type: integer + description: If sleeping between updates, how long in milliseconds to sleep for. + default: 1000 + min_batch_size: + type: integer + description: >- + Minimum size a batch of background updates can be. Must be greater than 0. + default: 1 + default_batch_size: + type: integer + description: >- + The batch size to use for the first iteration of a new background update. + default: 100 + examples: + - background_update_duration_ms: 500 + sleep_enabled: false + sleep_duration_ms: 300 + min_batch_size: 10 + default_batch_size: 50 + auto_accept_invites: + type: object + description: >- + Automatically accepting invites controls whether users are presented with + an invite request or if they are instead automatically joined to a room + when receiving an invite. Set the `enabled` sub-option to true to enable + auto-accepting invites. + + + NOTE: Care should be taken not to enable this setting if the + `synapse_auto_accept_invite` module is enabled and installed. The two + modules will compete to perform the same task and may result in undesired + behaviour. For example, multiple join events could be generated from a + single invite. + properties: + enabled: + type: boolean + description: Whether to run the auto-accept invites logic. + default: false + only_for_direct_messages: + type: boolean + description: >- + Whether invites should be automatically accepted for all room types, + or only for direct messages. + default: false + only_from_local_users: + type: boolean + description: >- + Whether to only automatically accept invites from users on this homeserver. + default: false + worker_to_run_on: + type: ["string", "null"] + description: >- + Which worker to run this module on. This must match the "worker_name". + If not set or `null`, invites will be accepted on the main process. + default: null + examples: + - enabled: true + only_for_direct_messages: true + only_from_local_users: true + worker_to_run_on: worker_1 +$defs: + bytes: + type: ["string", "integer"] + io.element.type_name: byte size + duration: + type: ["string", "integer"] + io.element.type_name: duration + size: + type: ["string", "integer"] + io.element.type_name: size + 3pidmedium: + type: string + enum: + - email + - msisdn + rc: + type: object + properties: + per_second: + type: number + description: Maximum number of requests a client can send per second. + burst_count: + type: number + description: >- + Maximum number of requests a client can send before being throttled. + database: + type: object + description: >- + The `database` setting defines the database that synapse uses to store all + of its data. + + + For more information on using Synapse with Postgres, see + [here](../../postgres.md). + properties: + name: + type: string + enum: + - sqlite3 + - psycopg2 + description: >- + This option specifies the database engine to use: either `sqlite3` + (for SQLite) or `psycopg2` (for PostgreSQL). If no name is specified + Synapse will default to SQLite. + default: sqlite3 + txn_limit: + type: integer + description: >- + Gives the maximum number of transactions to run per connection before + reconnecting. 0 means no limit. + default: 0 + allow_unsafe_locale: + type: boolean + description: >- + This option is specific to Postgres. Under the default behavior, + Synapse will refuse to start if the postgres db is set to a non-C + locale. You can override this behavior (which is *not* recommended) by + setting `allow_unsafe_locale` to true. Note that doing so may corrupt + your database. You can find more information + [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and + [here](https://wiki.postgresql.org/wiki/Locale_data_changes). + default: false + args: + type: object + description: >- + Gives options which are passed through to the database engine, except + for options starting with `cp_`, which are used to configure the + Twisted connection pool. For a reference to valid arguments, see: + + * for + [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) + + * for + [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) + + * for [the connection + pool](https://docs.twistedmatrix.com/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) diff --git a/schema/v1/Do not edit files in this folder b/schema/v1/Do not edit files in this folder new file mode 100644 index 0000000000..c95c3bfed0 --- /dev/null +++ b/schema/v1/Do not edit files in this folder @@ -0,0 +1,2 @@ +If you want to update the meta schema, copy this folder and increase its version +number instead. diff --git a/schema/v1/meta.schema.json b/schema/v1/meta.schema.json new file mode 100644 index 0000000000..0c2c46f1a9 --- /dev/null +++ b/schema/v1/meta.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/applicator": true, + "https://json-schema.org/draft/2020-12/vocab/unevaluated": true, + "https://json-schema.org/draft/2020-12/vocab/validation": true, + "https://json-schema.org/draft/2020-12/vocab/meta-data": true, + "https://json-schema.org/draft/2020-12/vocab/format-annotation": true, + "https://json-schema.org/draft/2020-12/vocab/content": true, + "https://element-hq.github.io/synapse/latest/schema/v1/vocab/documentation": false + }, + "$ref": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "io.element.type_name": { + "type": "string", + "description": "Human-readable type of a schema that is displayed instead of the standard JSON Schema types like `object` or `integer`. In case the JSON Schema type contains `null`, this information should be presented alongside the human-readable type name.", + "examples": ["duration", "byte size"] + }, + "io.element.post_description": { + "type": "string", + "description": "Additional description of a schema, better suited to be placed less prominently in the generated documentation, e.g., at the end of a section after listings of items and properties.", + "examples": [ + "### Advanced uses\n\nThe spent coffee grounds can be added to compost for improving soil and growing plants." + ] + } + } +} diff --git a/schema/v1/vocab/documentation.html b/schema/v1/vocab/documentation.html new file mode 100644 index 0000000000..6a45f4beb3 --- /dev/null +++ b/schema/v1/vocab/documentation.html @@ -0,0 +1,11 @@ + + + + + + Redirecting to ../meta.schema.json… + + +

Redirecting to ../meta.schema.json…

+ + diff --git a/scripts-dev/check_pydantic_models.py b/scripts-dev/check_pydantic_models.py index 5eb1f0a9df..26a473a61b 100755 --- a/scripts-dev/check_pydantic_models.py +++ b/scripts-dev/check_pydantic_models.py @@ -243,7 +243,7 @@ def do_lint() -> Set[str]: importlib.import_module(module_info.name) except ModelCheckerException as e: logger.warning( - f"Bad annotation found when importing {module_info.name}" + "Bad annotation found when importing %s", module_info.name ) failures.add(format_model_checker_exception(e)) diff --git a/scripts-dev/check_schema_delta.py b/scripts-dev/check_schema_delta.py index 467be96fdf..454784c3ae 100755 --- a/scripts-dev/check_schema_delta.py +++ b/scripts-dev/check_schema_delta.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 # Check that no schema deltas have been added to the wrong version. +# +# Also checks that schema deltas do not try and create or drop indices. import re from typing import Any, Dict, List @@ -9,6 +11,13 @@ import click import git SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") +INDEX_CREATION_REGEX = re.compile(r"CREATE .*INDEX .*ON ([a-z_]+)", flags=re.IGNORECASE) +INDEX_DELETION_REGEX = re.compile(r"DROP .*INDEX ([a-z_]+)", flags=re.IGNORECASE) +TABLE_CREATION_REGEX = re.compile(r"CREATE .*TABLE ([a-z_]+)", flags=re.IGNORECASE) + +# The base branch we want to check against. We use the main development branch +# on the assumption that is what we are developing against. +DEVELOP_BRANCH = "develop" @click.command() @@ -20,6 +29,9 @@ SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") help="Always output ANSI colours", ) def main(force_colors: bool) -> None: + # Return code. Set to non-zero when we encounter an error + return_code = 0 + click.secho( "+++ Checking schema deltas are in the right folder", fg="green", @@ -30,17 +42,17 @@ def main(force_colors: bool) -> None: click.secho("Updating repo...") repo = git.Repo() - repo.remote().fetch() + repo.remote().fetch(refspec=DEVELOP_BRANCH) click.secho("Getting current schema version...") - r = repo.git.show("origin/develop:synapse/storage/schema/__init__.py") + r = repo.git.show(f"origin/{DEVELOP_BRANCH}:synapse/storage/schema/__init__.py") locals: Dict[str, Any] = {} exec(r, locals) current_schema_version = locals["SCHEMA_VERSION"] - diffs: List[git.Diff] = repo.remote().refs.develop.commit.diff(None) + diffs: List[git.Diff] = repo.remote().refs[DEVELOP_BRANCH].commit.diff(None) # Get the schema version of the local file to check against current schema on develop with open("synapse/storage/schema/__init__.py") as file: @@ -53,7 +65,7 @@ def main(force_colors: bool) -> None: # local schema version must be +/-1 the current schema version on develop if abs(local_schema_version - current_schema_version) != 1: click.secho( - "The proposed schema version has diverged more than one version from develop, please fix!", + f"The proposed schema version has diverged more than one version from {DEVELOP_BRANCH}, please fix!", fg="red", bold=True, color=force_colors, @@ -67,21 +79,28 @@ def main(force_colors: bool) -> None: click.secho(f"Current schema version: {current_schema_version}") seen_deltas = False - bad_files = [] + bad_delta_files = [] + changed_delta_files = [] for diff in diffs: - if not diff.new_file or diff.b_path is None: + if diff.b_path is None: + # We don't lint deleted files. continue match = SCHEMA_FILE_REGEX.match(diff.b_path) if not match: continue + changed_delta_files.append(diff.b_path) + + if not diff.new_file: + continue + seen_deltas = True _, delta_version, _ = match.groups() if delta_version != str(current_schema_version): - bad_files.append(diff.b_path) + bad_delta_files.append(diff.b_path) if not seen_deltas: click.secho( @@ -92,41 +111,91 @@ def main(force_colors: bool) -> None: ) return - if not bad_files: + if bad_delta_files: + bad_delta_files.sort() + + click.secho( + "Found deltas in the wrong folder!", + fg="red", + bold=True, + color=force_colors, + ) + + for f in bad_delta_files: + click.secho( + f"\t{f}", + fg="red", + bold=True, + color=force_colors, + ) + + click.secho() + click.secho( + f"Please move these files to delta/{current_schema_version}/", + fg="red", + bold=True, + color=force_colors, + ) + + else: click.secho( f"All deltas are in the correct folder: {current_schema_version}!", fg="green", bold=True, color=force_colors, ) - return - bad_files.sort() + # Make sure we process them in order. This sort works because deltas are numbered + # and delta files are also numbered in order. + changed_delta_files.sort() - click.secho( - "Found deltas in the wrong folder!", - fg="red", - bold=True, - color=force_colors, - ) + # Now check that we're not trying to create or drop indices. If we want to + # do that they should be in background updates. The exception is when we + # create indices on tables we've just created. + created_tables = set() + for delta_file in changed_delta_files: + with open(delta_file) as fd: + delta_lines = fd.readlines() - for f in bad_files: - click.secho( - f"\t{f}", - fg="red", - bold=True, - color=force_colors, - ) + for line in delta_lines: + # Strip SQL comments + line = line.split("--", maxsplit=1)[0] - click.secho() - click.secho( - f"Please move these files to delta/{current_schema_version}/", - fg="red", - bold=True, - color=force_colors, - ) + # Check and track any tables we create + match = TABLE_CREATION_REGEX.search(line) + if match: + table_name = match.group(1) + created_tables.add(table_name) - click.get_current_context().exit(1) + # Check for dropping indices, these are always banned + match = INDEX_DELETION_REGEX.search(line) + if match: + clause = match.group() + + click.secho( + f"Found delta with index deletion: '{clause}' in {delta_file}\nThese should be in background updates.", + fg="red", + bold=True, + color=force_colors, + ) + return_code = 1 + + # Check for index creation, which is only allowed for tables we've + # created. + match = INDEX_CREATION_REGEX.search(line) + if match: + clause = match.group() + table_name = match.group(1) + if table_name not in created_tables: + click.secho( + f"Found delta with index creation: '{clause}' in {delta_file}\nThese should be in background updates.", + fg="red", + bold=True, + color=force_colors, + ) + return_code = 1 + + click.get_current_context().exit(return_code) if __name__ == "__main__": diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index b6dcb96e2c..c4d678b142 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -195,6 +195,10 @@ if [ -z "$skip_docker_build" ]; then # Build the unified Complement image (from the worker Synapse image we just built). echo_if_github "::group::Build Docker image: complement/Dockerfile" $CONTAINER_RUNTIME build -t complement-synapse \ + `# This is the tag we end up pushing to the registry (see` \ + `# .github/workflows/push_complement_image.yml) so let's just label it now` \ + `# so people can reference it by the same name locally.` \ + -t ghcr.io/element-hq/synapse/complement-synapse \ -f "docker/complement/Dockerfile" "docker/complement" echo_if_github "::endgroup::" @@ -225,6 +229,8 @@ test_packages=( ./tests/msc3902 ./tests/msc3967 ./tests/msc4140 + ./tests/msc4155 + ./tests/msc4306 ) # Enable dirty runs, so tests will reuse the same container where possible. diff --git a/scripts-dev/gen_config_documentation.py b/scripts-dev/gen_config_documentation.py new file mode 100755 index 0000000000..9a49c07a34 --- /dev/null +++ b/scripts-dev/gen_config_documentation.py @@ -0,0 +1,510 @@ +#!/usr/bin/env python3 +"""Generate Synapse documentation from JSON Schema file.""" + +import json +import re +import sys +from typing import Any, Optional + +import yaml + +HEADER = """ + +# Configuring Synapse + +This is intended as a guide to the Synapse configuration. The behavior of a Synapse instance can be modified +through the many configuration settings documented here — each config option is explained, +including what the default is, how to change the default and what sort of behaviour the setting governs. +Also included is an example configuration for each setting. If you don't want to spend a lot of time +thinking about options, the config as generated sets sensible defaults for all values. Do note however that the +database defaults to SQLite, which is not recommended for production usage. You can read more on this subject +[here](../../setup/installation.md#using-postgresql). + +## Config Conventions + +Configuration options that take a time period can be set using a number +followed by a letter. Letters have the following meanings: + +* `s` = second +* `m` = minute +* `h` = hour +* `d` = day +* `w` = week +* `y` = year + +For example, setting `redaction_retention_period: 5m` would remove redacted +messages from the database after 5 minutes, rather than 5 months. + +In addition, configuration options referring to size use the following suffixes: + +* `K` = KiB, or 1024 bytes +* `M` = MiB, or 1,048,576 bytes +* `G` = GiB, or 1,073,741,824 bytes +* `T` = TiB, or 1,099,511,627,776 bytes + +For example, setting `max_avatar_size: 10M` means that Synapse will not accept files larger than 10,485,760 bytes +for a user avatar. + +## Config Validation + +The configuration file can be validated with the following command: +```bash +python -m synapse.config read -c +``` + +To validate the entire file, omit `read `: +```bash +python -m synapse.config -c +``` + +To see how to set other options, check the help reference: +```bash +python -m synapse.config --help +``` + +### YAML +The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules +apply if you want your config file to be read properly. A few helpful things to know: +* `#` before any option in the config will comment out that setting and either a default (if available) will + be applied or Synapse will ignore the setting. Thus, in example #1 below, the setting will be read and + applied, but in example #2 the setting will not be read and a default will be applied. + + Example #1: + ```yaml + pid_file: DATADIR/homeserver.pid + ``` + Example #2: + ```yaml + #pid_file: DATADIR/homeserver.pid + ``` +* Indentation matters! The indentation before a setting + will determine whether a given setting is read as part of another + setting, or considered on its own. Thus, in example #1, the `enabled` setting + is read as a sub-option of the `presence` setting, and will be properly applied. + + However, the lack of indentation before the `enabled` setting in example #2 means + that when reading the config, Synapse will consider both `presence` and `enabled` as + different settings. In this case, `presence` has no value, and thus a default applied, and `enabled` + is an option that Synapse doesn't recognize and thus ignores. + + Example #1: + ```yaml + presence: + enabled: false + ``` + Example #2: + ```yaml + presence: + enabled: false + ``` + In this manual, all top-level settings (ones with no indentation) are identified + at the beginning of their section (i.e. "### `example_setting`") and + the sub-options, if any, are identified and listed in the body of the section. + In addition, each setting has an example of its usage, with the proper indentation + shown. +""" +SECTION_HEADERS = { + "modules": { + "title": "Modules", + "description": ( + "Server admins can expand Synapse's functionality with external " + "modules.\n\n" + "See [here](../../modules/index.md) for more documentation on how " + "to configure or create custom modules for Synapse." + ), + }, + "server_name": { + "title": "Server", + "description": "Define your homeserver name and other base options.", + }, + "admin_contact": { + "title": "Homeserver blocking", + "description": "Useful options for Synapse admins.", + }, + "tls_certificate_path": { + "title": "TLS", + "description": "Options related to TLS.", + }, + "federation_domain_whitelist": { + "title": "Federation", + "description": "Options related to federation.", + }, + "event_cache_size": { + "title": "Caching", + "description": "Options related to caching.", + }, + "database": { + "title": "Database", + "description": "Config options related to database settings.", + }, + "log_config": { + "title": "Logging", + "description": ("Config options related to logging."), + }, + "rc_message": { + "title": "Ratelimiting", + "description": ( + "Options related to ratelimiting in Synapse.\n\n" + "Each ratelimiting configuration is made of two parameters:\n" + "- `per_second`: number of requests a client can send per second.\n" + "- `burst_count`: number of requests a client can send before " + "being throttled." + ), + }, + "enable_authenticated_media": { + "title": "Media Store", + "description": "Config options related to Synapse's media store.", + }, + "recaptcha_public_key": { + "title": "Captcha", + "description": ( + "See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha." + ), + }, + "turn_uris": { + "title": "TURN", + "description": ("Options related to adding a TURN server to Synapse."), + }, + "enable_registration": { + "title": "Registration", + "description": ( + "Registration can be rate-limited using the parameters in the " + "[Ratelimiting](#ratelimiting) section of this manual." + ), + }, + "session_lifetime": { + "title": "User session management", + "description": ("Config options related to user session management."), + }, + "enable_metrics": { + "title": "Metrics", + "description": ("Config options related to metrics."), + }, + "room_prejoin_state": { + "title": "API Configuration", + "description": ("Config settings related to the client/server API."), + }, + "signing_key_path": { + "title": "Signing Keys", + "description": ("Config options relating to signing keys."), + }, + "saml2_config": { + "title": "Single sign-on integration", + "description": ( + "The following settings can be used to make Synapse use a single sign-on provider for authentication, instead of its internal password database.\n\n" + "You will probably also want to set the following options to `false` to disable the regular login/registration flows:\n" + "* [`enable_registration`](#enable_registration)\n" + "* [`password_config.enabled`](#password_config)" + ), + }, + "push": { + "title": "Push", + "description": ("Configuration settings related to push notifications."), + }, + "encryption_enabled_by_default_for_room_type": { + "title": "Rooms", + "description": ("Config options relating to rooms."), + }, + "opentracing": { + "title": "Opentracing", + "description": ("Configuration options related to Opentracing support."), + }, + "worker_replication_secret": { + "title": "Coordinating workers", + "description": ( + "Configuration options related to workers which belong in the main config file (usually called `homeserver.yaml`). A Synapse deployment can scale horizontally by running multiple Synapse processes called _workers_. Incoming requests are distributed between workers to handle higher loads. Some workers are privileged and can accept requests from other workers.\n\n" + "As a result, the worker configuration is divided into two parts.\n\n" + "1. The first part (in this section of the manual) defines which shardable tasks are delegated to privileged workers. This allows unprivileged workers to make requests to a privileged worker to act on their behalf.\n" + "2. [The second part](#individual-worker-configuration) controls the behaviour of individual workers in isolation.\n\n" + "For guidance on setting up workers, see the [worker documentation](../../workers.md)." + ), + }, + "worker_app": { + "title": "Individual worker configuration", + "description": ( + "These options configure an individual worker, in its worker configuration file. They should be not be provided when configuring the main process.\n\n" + "Note also the configuration above for [coordinating a cluster of workers](#coordinating-workers).\n\n" + "For guidance on setting up workers, see the [worker documentation](../../workers.md)." + ), + }, + "background_updates": { + "title": "Background Updates", + "description": ("Configuration settings related to background updates."), + }, + "auto_accept_invites": { + "title": "Auto Accept Invites", + "description": ( + "Configuration settings related to automatically accepting invites." + ), + }, +} +INDENT = " " + + +has_error = False + + +def error(text: str) -> None: + global has_error + print(f"ERROR: {text}", file=sys.stderr) + has_error = True + + +def indent(text: str, first_line: bool = True) -> str: + """Indents each non-empty line of the given text.""" + text = re.sub(r"(\n)([^\n])", r"\1" + INDENT + r"\2", text) + if first_line: + text = re.sub(r"^([^\n])", INDENT + r"\1", text) + + return text + + +def em(s: Optional[str]) -> str: + """Add emphasis to text.""" + return f"*{s}*" if s else "" + + +def a(s: Optional[str], suffix: str = " ") -> str: + """Appends a space if the given string is not empty.""" + return s + suffix if s else "" + + +def p(s: Optional[str], prefix: str = " ") -> str: + """Prepend a space if the given string is not empty.""" + return prefix + s if s else "" + + +def resolve_local_refs(schema: dict) -> dict: + """Returns the given schema with local $ref properties replaced by their keywords. + + Crude approximation that will override keywords. + """ + defs = schema["$defs"] + + def replace_ref(d: Any) -> Any: + if isinstance(d, dict): + the_def = {} + if "$ref" in d: + # Found a "$ref" key. + def_name = d["$ref"].removeprefix("#/$defs/") + del d["$ref"] + the_def = defs[def_name] + + new_dict = {k: replace_ref(v) for k, v in d.items()} + if common_keys := (new_dict.keys() & the_def.keys()) - {"properties"}: + print( + f"WARN: '{def_name}' overrides keys '{common_keys}'", + file=sys.stderr, + ) + + new_dict_props = new_dict.get("properties", {}) + the_def_props = the_def.get("properties", {}) + if common_props := new_dict_props.keys() & the_def_props.keys(): + print( + f"WARN: '{def_name}' overrides properties '{common_props}'", + file=sys.stderr, + ) + if merged_props := {**new_dict_props, **the_def_props}: + return {**new_dict, **the_def, "properties": merged_props} + else: + return {**new_dict, **the_def} + + elif isinstance(d, list): + return [replace_ref(v) for v in d] + else: + return d + + return replace_ref(schema) + + +def sep(values: dict) -> str: + """Separator between parts of the description.""" + # If description is multiple paragraphs already, add new ones. Otherwise + # append to same paragraph. + return "\n\n" if "\n\n" in values.get("description", "") else " " + + +def type_str(values: dict) -> str: + """Type of the current value.""" + if t := values.get("io.element.type_name"): + # Allow custom overrides for the type name, for documentation clarity + return f"({t})" + if not (t := values.get("type")): + return "" + if not isinstance(t, list): + t = [t] + joined = "|".join(t) + return f"({joined})" + + +def items(values: dict) -> str: + """A block listing properties of array items.""" + if not (items := values.get("items")): + return "" + if not (item_props := items.get("properties")): + return "" + return "\nOptions for each entry include:\n\n" + "\n".join( + sub_section(k, v) for k, v in item_props.items() + ) + + +def properties(values: dict) -> str: + """A block listing object properties.""" + if not (properties := values.get("properties")): + return "" + return "\nThis setting has the following sub-options:\n\n" + "\n".join( + sub_section(k, v) for k, v in properties.items() + ) + + +def sub_section(prop: str, values: dict) -> str: + """Formats a bullet point about the given sub-property.""" + sep = lambda: globals()["sep"](values) + type_str = lambda: globals()["type_str"](values) + items = lambda: globals()["items"](values) + properties = lambda: globals()["properties"](values) + + def default() -> str: + try: + default = values["default"] + return f"Defaults to `{json.dumps(default)}`." + except KeyError: + return "" + + def description() -> str: + if not (description := values.get("description")): + error(f"missing description for {prop}") + return "MISSING DESCRIPTION\n" + + return f"{description}{p(default(), sep())}\n" + + return ( + f"* `{prop}`{p(type_str())}: " + + f"{indent(description(), first_line=False)}" + + indent(items()) + + indent(properties()) + ) + + +def section(prop: str, values: dict) -> str: + """Formats a section about the given property.""" + sep = lambda: globals()["sep"](values) + type_str = lambda: globals()["type_str"](values) + items = lambda: globals()["items"](values) + properties = lambda: globals()["properties"](values) + + def is_simple_default() -> bool: + """Whether the given default is simple enough for a one-liner.""" + if not (d := values.get("default")): + return True + return not isinstance(d, dict) and not isinstance(d, list) + + def default_str() -> str: + try: + default = values["default"] + except KeyError: + t = values.get("type", []) + if "object" == t or "object" in t: + # Skip objects as they probably have child defaults. + return "" + return "There is no default for this option." + + if not is_simple_default(): + # Show complex defaults as a code block instead. + return "" + return f"Defaults to `{json.dumps(default)}`." + + def header() -> str: + try: + title = SECTION_HEADERS[prop]["title"] + description = SECTION_HEADERS[prop]["description"] + return f"## {title}\n\n{description}\n\n---\n" + except KeyError: + return "" + + def title() -> str: + return f"### `{prop}`\n" + + def description() -> str: + if not (description := values.get("description")): + error(f"missing description for {prop}") + return "MISSING DESCRIPTION\n" + return f"\n{a(em(type_str()))}{description}{p(default_str(), sep())}\n" + + def example_str(example: Any) -> str: + return "```yaml\n" + f"{yaml.dump({prop: example}, sort_keys=False)}" + "```\n" + + def default_example() -> str: + if is_simple_default(): + return "" + default_cfg = example_str(values["default"]) + return f"\nDefault configuration:\n{default_cfg}" + + def examples() -> str: + if not (examples := values.get("examples")): + return "" + + examples_str = "\n".join(example_str(e) for e in examples) + + if len(examples) >= 2: + return f"\nExample configurations:\n{examples_str}" + else: + return f"\nExample configuration:\n{examples_str}" + + def post_description() -> str: + # Sometimes it's helpful to have a description after the list of fields, + # e.g. with a subsection that consists only of text. + # This helps with that. + if not (description := values.get("io.element.post_description")): + return "" + return f"\n{description}\n\n" + + return ( + "---\n" + + header() + + title() + + description() + + items() + + properties() + + default_example() + + examples() + + post_description() + ) + + +def main() -> None: + # For Windows: reconfigure the terminal to be UTF-8 for `print()` calls. + if sys.platform == "win32": + sys.stdout.reconfigure(encoding="utf-8") + + def usage(err_msg: str) -> int: + script_name = (sys.argv[:1] or ["__main__.py"])[0] + print(err_msg, file=sys.stderr) + print(f"Usage: {script_name} ", file=sys.stderr) + print(f"\n{__doc__}", file=sys.stderr) + exit(1) + + def read_json_file_arg() -> Any: + if len(sys.argv) > 2: + exit(usage("Too many arguments.")) + if not (filepath := (sys.argv[1:] or [""])[0]): + exit(usage("No schema file provided.")) + with open(filepath, "r", encoding="utf-8") as f: + # Note: Windows requires that we specify the encoding otherwise it uses + # things like CP-1251, which can cause explosions. + # See https://github.com/yaml/pyyaml/issues/123 for more info. + return yaml.safe_load(f) + + schema = read_json_file_arg() + schema = resolve_local_refs(schema) + + sections = (section(k, v) for k, v in schema["properties"].items()) + print(HEADER + "".join(sections), end="") + + if has_error: + print("There were errors.", file=sys.stderr) + exit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index c656047729..7096100a3e 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -139,3 +139,6 @@ cargo-fmt # Ensure type hints are correct. mypy + +# Generate configuration documentation from the JSON Schema +./scripts-dev/gen_config_documentation.py schema/synapse-config.schema.yaml > docs/usage/configuration/config_documentation.md diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index a15c3c005c..610dec415a 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -23,28 +23,195 @@ can crop up, e.g the cache descriptors. """ -from typing import Callable, Optional, Tuple, Type, Union +import enum +from typing import Callable, Mapping, Optional, Tuple, Type, Union +import attr import mypy.types from mypy.erasetype import remove_instance_last_known_values from mypy.errorcodes import ErrorCode -from mypy.nodes import ARG_NAMED_OPT, TempNode, Var -from mypy.plugin import FunctionSigContext, MethodSigContext, Plugin +from mypy.nodes import ARG_NAMED_OPT, ListExpr, NameExpr, TempNode, TupleExpr, Var +from mypy.plugin import ( + ClassDefContext, + Context, + FunctionLike, + FunctionSigContext, + MethodSigContext, + MypyFile, + Plugin, +) from mypy.typeops import bind_self from mypy.types import ( AnyType, CallableType, Instance, NoneType, + Options, TupleType, TypeAliasType, TypeVarType, UninhabitedType, UnionType, ) +from mypy_zope import plugin as mypy_zope_plugin +from pydantic.mypy import plugin as mypy_pydantic_plugin + +PROMETHEUS_METRIC_MISSING_SERVER_NAME_LABEL = ErrorCode( + "missing-server-name-label", + "`SERVER_NAME_LABEL` required in metric", + category="per-homeserver-tenant-metrics", +) + +PROMETHEUS_METRIC_MISSING_FROM_LIST_TO_CHECK = ErrorCode( + "metric-type-missing-from-list", + "Every Prometheus metric type must be included in the `prometheus_metric_fullname_to_label_arg_map`.", + category="per-homeserver-tenant-metrics", +) + + +class Sentinel(enum.Enum): + # defining a sentinel in this way allows mypy to correctly handle the + # type of a dictionary lookup and subsequent type narrowing. + UNSET_SENTINEL = object() + + +@attr.s(auto_attribs=True) +class ArgLocation: + keyword_name: str + """ + The keyword argument name for this argument + """ + position: int + """ + The 0-based positional index of this argument + """ + + +prometheus_metric_fullname_to_label_arg_map: Mapping[str, Optional[ArgLocation]] = { + # `Collector` subclasses: + "prometheus_client.metrics.MetricWrapperBase": ArgLocation("labelnames", 2), + "prometheus_client.metrics.Counter": ArgLocation("labelnames", 2), + "prometheus_client.metrics.Histogram": ArgLocation("labelnames", 2), + "prometheus_client.metrics.Gauge": ArgLocation("labelnames", 2), + "prometheus_client.metrics.Summary": ArgLocation("labelnames", 2), + "prometheus_client.metrics.Info": ArgLocation("labelnames", 2), + "prometheus_client.metrics.Enum": ArgLocation("labelnames", 2), + "synapse.metrics.LaterGauge": ArgLocation("labelnames", 2), + "synapse.metrics.InFlightGauge": ArgLocation("labels", 2), + "synapse.metrics.GaugeBucketCollector": ArgLocation("labelnames", 2), + "prometheus_client.registry.Collector": None, + "prometheus_client.registry._EmptyCollector": None, + "prometheus_client.registry.CollectorRegistry": None, + "prometheus_client.process_collector.ProcessCollector": None, + "prometheus_client.platform_collector.PlatformCollector": None, + "prometheus_client.gc_collector.GCCollector": None, + "synapse.metrics._gc.GCCounts": None, + "synapse.metrics._gc.PyPyGCStats": None, + "synapse.metrics._reactor_metrics.ReactorLastSeenMetric": None, + "synapse.metrics.CPUMetrics": None, + "synapse.metrics.jemalloc.JemallocCollector": None, + "synapse.util.metrics.DynamicCollectorRegistry": None, + "synapse.metrics.background_process_metrics._Collector": None, + # + # `Metric` subclasses: + "prometheus_client.metrics_core.Metric": None, + "prometheus_client.metrics_core.UnknownMetricFamily": ArgLocation("labels", 3), + "prometheus_client.metrics_core.CounterMetricFamily": ArgLocation("labels", 3), + "prometheus_client.metrics_core.GaugeMetricFamily": ArgLocation("labels", 3), + "prometheus_client.metrics_core.SummaryMetricFamily": ArgLocation("labels", 3), + "prometheus_client.metrics_core.InfoMetricFamily": ArgLocation("labels", 3), + "prometheus_client.metrics_core.HistogramMetricFamily": ArgLocation("labels", 3), + "prometheus_client.metrics_core.GaugeHistogramMetricFamily": ArgLocation( + "labels", 4 + ), + "prometheus_client.metrics_core.StateSetMetricFamily": ArgLocation("labels", 3), + "synapse.metrics.GaugeHistogramMetricFamilyWithLabels": ArgLocation( + "labelnames", 4 + ), +} +""" +Map from the fullname of the Prometheus `Metric`/`Collector` classes to the keyword +argument name and positional index of the label names. This map is useful because +different metrics have different signatures for passing in label names and we just need +to know where to look. + +This map should include any metrics that we collect with Prometheus. Which corresponds +to anything that inherits from `prometheus_client.registry.Collector` +(`synapse.metrics._types.Collector`) or `prometheus_client.metrics_core.Metric`. The +exhaustiveness of this list is enforced by `analyze_prometheus_metric_classes`. + +The entries with `None` always fail the lint because they don't have a `labelnames` +argument (therefore, no `SERVER_NAME_LABEL`), but we include them here so that people +can notice and manually allow via a type ignore comment as the source of truth +should be in the source code. +""" + +# Unbound at this point because we don't know the mypy version yet. +# This is set in the `plugin(...)` function below. +MypyPydanticPluginClass: Type[Plugin] +MypyZopePluginClass: Type[Plugin] class SynapsePlugin(Plugin): + def __init__(self, options: Options): + super().__init__(options) + self.mypy_pydantic_plugin = MypyPydanticPluginClass(options) + self.mypy_zope_plugin = MypyZopePluginClass(options) + + def set_modules(self, modules: dict[str, MypyFile]) -> None: + """ + This is called by mypy internals. We have to override this to ensure it's also + called for any other plugins that we're manually handling. + + Here is how mypy describes it: + + > [`self._modules`] can't be set in `__init__` because it is executed too soon + > in `build.py`. Therefore, `build.py` *must* set it later before graph processing + > starts by calling `set_modules()`. + """ + super().set_modules(modules) + self.mypy_pydantic_plugin.set_modules(modules) + self.mypy_zope_plugin.set_modules(modules) + + def get_base_class_hook( + self, fullname: str + ) -> Optional[Callable[[ClassDefContext], None]]: + def _get_base_class_hook(ctx: ClassDefContext) -> None: + # Run any `get_base_class_hook` checks from other plugins first. + # + # Unfortunately, because mypy only chooses the first plugin that returns a + # non-None value (known-limitation, c.f. + # https://github.com/python/mypy/issues/19524), we workaround this by + # putting our custom plugin first in the plugin order and then calling the + # other plugin's hook manually followed by our own checks. + if callback := self.mypy_pydantic_plugin.get_base_class_hook(fullname): + callback(ctx) + if callback := self.mypy_zope_plugin.get_base_class_hook(fullname): + callback(ctx) + + # Now run our own checks + analyze_prometheus_metric_classes(ctx) + + return _get_base_class_hook + + def get_function_signature_hook( + self, fullname: str + ) -> Optional[Callable[[FunctionSigContext], FunctionLike]]: + # Strip off the unique identifier for classes that are dynamically created inside + # functions. ex. `synapse.metrics.jemalloc.JemallocCollector@185` (this is the line + # number) + if "@" in fullname: + fullname = fullname.split("@", 1)[0] + + # Look for any Prometheus metrics to make sure they have the `SERVER_NAME_LABEL` + # label. + if fullname in prometheus_metric_fullname_to_label_arg_map.keys(): + # Because it's difficult to determine the `fullname` of the function in the + # callback, let's just pass it in while we have it. + return lambda ctx: check_prometheus_metric_instantiation(ctx, fullname) + + return None + def get_method_signature_hook( self, fullname: str ) -> Optional[Callable[[MethodSigContext], CallableType]]: @@ -65,6 +232,157 @@ class SynapsePlugin(Plugin): return None +def analyze_prometheus_metric_classes(ctx: ClassDefContext) -> None: + """ + Cross-check the list of Prometheus metric classes against the + `prometheus_metric_fullname_to_label_arg_map` to ensure the list is exhaustive and + up-to-date. + """ + + fullname = ctx.cls.fullname + # Strip off the unique identifier for classes that are dynamically created inside + # functions. ex. `synapse.metrics.jemalloc.JemallocCollector@185` (this is the line + # number) + if "@" in fullname: + fullname = fullname.split("@", 1)[0] + + if any( + ancestor_type.fullname + in ( + # All of the Prometheus metric classes inherit from the `Collector`. + "prometheus_client.registry.Collector", + "synapse.metrics._types.Collector", + # And custom metrics that inherit from `Metric`. + "prometheus_client.metrics_core.Metric", + ) + for ancestor_type in ctx.cls.info.mro + ): + if fullname not in prometheus_metric_fullname_to_label_arg_map: + ctx.api.fail( + f"Expected {fullname} to be in `prometheus_metric_fullname_to_label_arg_map`, " + f"but it was not found. This is a problem with our custom mypy plugin. " + f"Please add it to the map.", + Context(), + code=PROMETHEUS_METRIC_MISSING_FROM_LIST_TO_CHECK, + ) + + +def check_prometheus_metric_instantiation( + ctx: FunctionSigContext, fullname: str +) -> CallableType: + """ + Ensure that the `prometheus_client` metrics include the `SERVER_NAME_LABEL` label + when instantiated. + + This is important because we support multiple Synapse instances running in the same + process, where all metrics share a single global `REGISTRY`. The `server_name` label + ensures metrics are correctly separated by homeserver. + + There are also some metrics that apply at the process level, such as CPU usage, + Python garbage collection, and Twisted reactor tick time, which shouldn't have the + `SERVER_NAME_LABEL`. In those cases, use a type ignore comment to disable the + check, e.g. `# type: ignore[missing-server-name-label]`. + + Args: + ctx: The `FunctionSigContext` from mypy. + fullname: The fully qualified name of the function being called, + e.g. `"prometheus_client.metrics.Counter"` + """ + # The true signature, this isn't being modified so this is what will be returned. + signature = ctx.default_signature + + # Find where the label names argument is in the function signature. + arg_location = prometheus_metric_fullname_to_label_arg_map.get( + fullname, Sentinel.UNSET_SENTINEL + ) + assert arg_location is not Sentinel.UNSET_SENTINEL, ( + f"Expected to find {fullname} in `prometheus_metric_fullname_to_label_arg_map`, " + f"but it was not found. This is a problem with our custom mypy plugin. " + f"Please add it to the map. Context: {ctx.context}" + ) + # People should be using `# type: ignore[missing-server-name-label]` for + # process-level metrics that should not have the `SERVER_NAME_LABEL`. + if arg_location is None: + ctx.api.fail( + f"{signature.name} does not have a `labelnames`/`labels` argument " + "(if this is untrue, update `prometheus_metric_fullname_to_label_arg_map` " + "in our custom mypy plugin) and should probably have a type ignore comment, " + "e.g. `# type: ignore[missing-server-name-label]`. The reason we don't " + "automatically ignore this is the source of truth should be in the source code.", + ctx.context, + code=PROMETHEUS_METRIC_MISSING_SERVER_NAME_LABEL, + ) + return signature + + # Sanity check the arguments are still as expected in this version of + # `prometheus_client`. ex. `Counter(name, documentation, labelnames, ...)` + # + # `signature.arg_names` should be: ["name", "documentation", "labelnames", ...] + if ( + len(signature.arg_names) < (arg_location.position + 1) + or signature.arg_names[arg_location.position] != arg_location.keyword_name + ): + ctx.api.fail( + f"Expected argument number {arg_location.position + 1} of {signature.name} to be `labelnames`/`labels`, " + f"but got {signature.arg_names[arg_location.position]}", + ctx.context, + ) + return signature + + # Ensure mypy is passing the correct number of arguments because we are doing some + # dirty indexing into `ctx.args` later on. + assert len(ctx.args) == len(signature.arg_names), ( + f"Expected the list of arguments in the {signature.name} signature ({len(signature.arg_names)})" + f"to match the number of arguments from the function signature context ({len(ctx.args)})" + ) + + # Check if the `labelnames` argument includes `SERVER_NAME_LABEL` + # + # `ctx.args` should look like this: + # ``` + # [ + # [StrExpr("name")], + # [StrExpr("documentation")], + # [ListExpr([StrExpr("label1"), StrExpr("label2")])] + # ... + # ] + # ``` + labelnames_arg_expression = ( + ctx.args[arg_location.position][0] + if len(ctx.args[arg_location.position]) > 0 + else None + ) + if isinstance(labelnames_arg_expression, (ListExpr, TupleExpr)): + # Check if the `labelnames` argument includes the `server_name` label (`SERVER_NAME_LABEL`). + for labelname_expression in labelnames_arg_expression.items: + if ( + isinstance(labelname_expression, NameExpr) + and labelname_expression.fullname == "synapse.metrics.SERVER_NAME_LABEL" + ): + # Found the `SERVER_NAME_LABEL`, all good! + break + else: + ctx.api.fail( + f"Expected {signature.name} to include `SERVER_NAME_LABEL` in the list of labels. " + "If this is a process-level metric (vs homeserver-level), use a type ignore comment " + "to disable this check.", + ctx.context, + code=PROMETHEUS_METRIC_MISSING_SERVER_NAME_LABEL, + ) + else: + ctx.api.fail( + f"Expected the `labelnames` argument of {signature.name} to be a list of label names " + f"(including `SERVER_NAME_LABEL`), but got {labelnames_arg_expression}. " + "If this is a process-level metric (vs homeserver-level), use a type ignore comment " + "to disable this check.", + ctx.context, + code=PROMETHEUS_METRIC_MISSING_SERVER_NAME_LABEL, + ) + return signature + + return signature + + def _get_true_return_type(signature: CallableType) -> mypy.types.Type: """ Get the "final" return type of a callable which might return an Awaitable/Deferred. @@ -372,10 +690,13 @@ def is_cacheable( def plugin(version: str) -> Type[SynapsePlugin]: + global MypyPydanticPluginClass, MypyZopePluginClass # This is the entry point of the plugin, and lets us deal with the fact # that the mypy plugin interface is *not* stable by looking at the version # string. # # However, since we pin the version of mypy Synapse uses in CI, we don't # really care. + MypyPydanticPluginClass = mypy_pydantic_plugin(version) + MypyZopePluginClass = mypy_zope_plugin(version) return SynapsePlugin diff --git a/scripts-dev/release.py b/scripts-dev/release.py index b14b61c705..73a4e7b7a9 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -36,11 +36,11 @@ from typing import Any, List, Match, Optional, Union import attr import click -import commonmark import git from click.exceptions import ClickException from git import GitCommandError, Repo from github import BadCredentialsException, Github +from markdown_it import MarkdownIt from packaging import version @@ -254,6 +254,12 @@ def _prepare() -> None: # Update the version specified in pyproject.toml. subprocess.check_output(["poetry", "version", new_version]) + # Update config schema $id. + schema_file = "schema/synapse-config.schema.yaml" + major_minor_version = ".".join(new_version.split(".")[:2]) + url = f"https://element-hq.github.io/synapse/schema/synapse/v{major_minor_version}/synapse-config.schema.json" + subprocess.check_output(["sed", "-i", f"0,/^\\$id: .*/s||$id: {url}|", schema_file]) + # Generate changelogs. generate_and_write_changelog(synapse_repo, current_version, new_version) @@ -592,7 +598,7 @@ def _wait_for_actions(gh_token: Optional[str]) -> None: if all( workflow["status"] != "in_progress" for workflow in resp["workflow_runs"] ): - success = ( + success = all( workflow["status"] == "completed" for workflow in resp["workflow_runs"] ) if success: @@ -845,7 +851,7 @@ def get_changes_for_version(wanted_version: version.Version) -> str: # First we parse the changelog so that we can split it into sections based # on the release headings. - ast = commonmark.Parser().parse(changes) + tokens = MarkdownIt().parse(changes) @attr.s(auto_attribs=True) class VersionSection: @@ -856,19 +862,22 @@ def get_changes_for_version(wanted_version: version.Version) -> str: end_line: Optional[int] = None # Is none if its the last entry headings: List[VersionSection] = [] - for node, _ in ast.walker(): - # We look for all text nodes that are in a level 1 heading. - if node.t != "text": + for i, token in enumerate(tokens): + # We look for level 1 headings (h1 tags). + if token.type != "heading_open" or token.tag != "h1": continue - if node.parent.t != "heading" or node.parent.level != 1: - continue + # The next token should be an inline token containing the heading text + if i + 1 < len(tokens) and tokens[i + 1].type == "inline": + heading_text = tokens[i + 1].content + # The map property contains [line_begin, line_end] (0-based) + start_line = token.map[0] if token.map else 0 - # If we have a previous heading then we update its `end_line`. - if headings: - headings[-1].end_line = node.parent.sourcepos[0][0] - 1 + # If we have a previous heading then we update its `end_line`. + if headings: + headings[-1].end_line = start_line - headings.append(VersionSection(node.literal, node.parent.sourcepos[0][0] - 1)) + headings.append(VersionSection(heading_text, start_line)) changes_by_line = changes.split("\n") diff --git a/synapse/__init__.py b/synapse/__init__.py index e7784ac5d7..3bd1b3307e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -45,16 +45,6 @@ if py_version < (3, 9): # Allow using the asyncio reactor via env var. if strtobool(os.environ.get("SYNAPSE_ASYNC_IO_REACTOR", "0")): - from incremental import Version - - import twisted - - # We need a bugfix that is included in Twisted 21.2.0: - # https://twistedmatrix.com/trac/ticket/9787 - if twisted.version < Version("Twisted", 21, 2, 0): - print("Using asyncio reactor requires Twisted>=21.2.0") - sys.exit(1) - import asyncio from twisted.internet import asyncioreactor diff --git a/synapse/_pydantic_compat.py b/synapse/_pydantic_compat.py index f0eedf5c6d..a520c0e897 100644 --- a/synapse/_pydantic_compat.py +++ b/synapse/_pydantic_compat.py @@ -34,9 +34,11 @@ HAS_PYDANTIC_V2: bool = Version(pydantic_version).major == 2 if TYPE_CHECKING or HAS_PYDANTIC_V2: from pydantic.v1 import ( + AnyHttpUrl, BaseModel, Extra, Field, + FilePath, MissingError, PydanticValueError, StrictBool, @@ -48,15 +50,18 @@ if TYPE_CHECKING or HAS_PYDANTIC_V2: conint, constr, parse_obj_as, + root_validator, validator, ) from pydantic.v1.error_wrappers import ErrorWrapper from pydantic.v1.typing import get_args else: from pydantic import ( + AnyHttpUrl, BaseModel, Extra, Field, + FilePath, MissingError, PydanticValueError, StrictBool, @@ -68,6 +73,7 @@ else: conint, constr, parse_obj_as, + root_validator, validator, ) from pydantic.error_wrappers import ErrorWrapper @@ -75,6 +81,7 @@ else: __all__ = ( "HAS_PYDANTIC_V2", + "AnyHttpUrl", "BaseModel", "constr", "conbytes", @@ -83,6 +90,7 @@ __all__ = ( "ErrorWrapper", "Extra", "Field", + "FilePath", "get_args", "MissingError", "parse_obj_as", @@ -92,4 +100,5 @@ __all__ = ( "StrictStr", "ValidationError", "validator", + "root_validator", ) diff --git a/synapse/_scripts/generate_workers_map.py b/synapse/_scripts/generate_workers_map.py index 09feb8cf30..8878e364e2 100755 --- a/synapse/_scripts/generate_workers_map.py +++ b/synapse/_scripts/generate_workers_map.py @@ -153,9 +153,13 @@ def get_registered_paths_for_default( """ hs = MockHomeserver(base_config, worker_app) + # TODO We only do this to avoid an error, but don't need the database etc hs.setup() - return get_registered_paths_for_hs(hs) + registered_paths = get_registered_paths_for_hs(hs) + hs.cleanup() + + return registered_paths def elide_http_methods_if_unconflicting( diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py index 14cb21c7fb..4897fa94b0 100644 --- a/synapse/_scripts/register_new_matrix_user.py +++ b/synapse/_scripts/register_new_matrix_user.py @@ -30,6 +30,7 @@ from typing import Any, Callable, Dict, Optional import requests import yaml +from typing_extensions import Never _CONFLICTING_SHARED_SECRET_OPTS_ERROR = """\ Conflicting options 'registration_shared_secret' and 'registration_shared_secret_path' @@ -40,6 +41,10 @@ _NO_SHARED_SECRET_OPTS_ERROR = """\ No 'registration_shared_secret' or 'registration_shared_secret_path' defined in config. """ +_EMPTY_SHARED_SECRET_PATH_OPTS_ERROR = """\ +The secret given via `registration_shared_secret_path` must not be empty. +""" + _DEFAULT_SERVER_URL = "http://localhost:8008" @@ -170,6 +175,12 @@ def register_new_user( ) +def bail(err_msg: str) -> Never: + """Prints the given message to stderr and exits.""" + print(err_msg, file=sys.stderr) + sys.exit(1) + + def main() -> None: logging.captureWarnings(True) @@ -262,15 +273,20 @@ def main() -> None: assert config is not None secret = config.get("registration_shared_secret") + if not isinstance(secret, (str, type(None))): + bail("registration_shared_secret is not a string.") secret_file = config.get("registration_shared_secret_path") - if secret_file: - if secret: - print(_CONFLICTING_SHARED_SECRET_OPTS_ERROR, file=sys.stderr) - sys.exit(1) + if not isinstance(secret_file, (str, type(None))): + bail("registration_shared_secret_path is not a string.") + + if not secret and not secret_file: + bail(_NO_SHARED_SECRET_OPTS_ERROR) + elif secret and secret_file: + bail(_CONFLICTING_SHARED_SECRET_OPTS_ERROR) + elif not secret and secret_file: secret = _read_file(secret_file, "registration_shared_secret_path").strip() - if not secret: - print(_NO_SHARED_SECRET_OPTS_ERROR, file=sys.stderr) - sys.exit(1) + if not secret: + bail(_EMPTY_SHARED_SECRET_PATH_OPTS_ERROR) if args.password_file: password = _read_file(args.password_file, "password-file").strip() diff --git a/synapse/_scripts/review_recent_signups.py b/synapse/_scripts/review_recent_signups.py index 62723c539d..0ff7fae567 100644 --- a/synapse/_scripts/review_recent_signups.py +++ b/synapse/_scripts/review_recent_signups.py @@ -29,19 +29,21 @@ import attr from synapse.config._base import ( Config, + ConfigError, RootConfig, find_config_files, read_config_files, ) from synapse.config.database import DatabaseConfig +from synapse.config.server import ServerConfig from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn from synapse.storage.engines import create_engine class ReviewConfig(RootConfig): - "A config class that just pulls out the database config" + "A config class that just pulls out the server and database config" - config_classes = [DatabaseConfig] + config_classes = [ServerConfig, DatabaseConfig] @attr.s(auto_attribs=True) @@ -148,6 +150,10 @@ def main() -> None: config_dict = read_config_files(config_files) config.parse_config_dict(config_dict, "", "") + server_name = config.server.server_name + if not isinstance(server_name, str): + raise ConfigError("Must be a string", ("server_name",)) + since_ms = time.time() * 1000 - Config.parse_duration(config_args.since) exclude_users_with_email = config_args.exclude_emails exclude_users_with_appservice = config_args.exclude_app_service @@ -159,7 +165,12 @@ def main() -> None: engine = create_engine(database_config.config) - with make_conn(database_config, engine, "review_recent_signups") as db_conn: + with make_conn( + db_config=database_config, + engine=engine, + default_txn_name="review_recent_signups", + server_name=server_name, + ) as db_conn: # This generates a type of Cursor, not LoggingTransaction. user_infos = get_recent_users( db_conn.cursor(), diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index d8f6f8ebdc..a81db3cfbf 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -42,12 +42,12 @@ from typing import ( Set, Tuple, Type, + TypedDict, TypeVar, cast, ) import yaml -from typing_extensions import TypedDict from twisted.internet import defer, reactor as reactor_ @@ -99,6 +99,7 @@ from synapse.storage.engines import create_engine from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor from synapse.util import SYNAPSE_VERSION, Clock +from synapse.util.stringutils import random_string # Cast safety: Twisted does some naughty magic which replaces the # twisted.internet.reactor module with a Reactor instance at runtime. @@ -128,6 +129,7 @@ BOOLEAN_COLUMNS = { "pushers": ["enabled"], "redactions": ["have_censored"], "remote_media_cache": ["authenticated"], + "room_memberships": ["participant"], "room_stats_state": ["is_federatable"], "rooms": ["is_public", "has_auth_chain_index"], "sliding_sync_joined_rooms": ["is_encrypted"], @@ -135,6 +137,7 @@ BOOLEAN_COLUMNS = { "has_known_state", "is_encrypted", ], + "thread_subscriptions": ["subscribed", "automatic"], "users": ["shadow_banned", "approved", "locked", "suspended"], "un_partial_stated_event_stream": ["rejection_status_changed"], "users_who_share_rooms": ["share_private"], @@ -189,6 +192,16 @@ APPEND_ONLY_TABLES = [ "users", ] +# These tables declare their id column with "PRIMARY KEY AUTOINCREMENT" on sqlite side +# and with "PRIMARY KEY GENERATED ALWAYS AS IDENTITY" on postgres side. This creates an +# implicit sequence that needs its value to be migrated separately. Additionally, +# inserting on postgres side needs to use the "OVERRIDING SYSTEM VALUE" modifier. +AUTOINCREMENT_TABLES = { + "sliding_sync_connections", + "sliding_sync_connection_positions", + "sliding_sync_connection_required_state", + "state_groups_pending_deletion", +} IGNORED_TABLES = { # We don't port these tables, as they're a faff and we can regenerate @@ -216,6 +229,15 @@ IGNORED_TABLES = { } +# These background updates will not be applied upon creation of the postgres database. +IGNORED_BACKGROUND_UPDATES = { + # Reapplying this background update to the postgres database is unnecessary after + # already having waited for the SQLite database to complete all running background + # updates. + "mark_unreferenced_state_groups_for_deletion_bg_update", +} + + # Error returned by the run function. Used at the top-level part of the script to # handle errors and return codes. end_error: Optional[str] = None @@ -269,11 +291,17 @@ class Store( return self.db_pool.runInteraction("execute_sql", r) def insert_many_txn( - self, txn: LoggingTransaction, table: str, headers: List[str], rows: List[Tuple] + self, + txn: LoggingTransaction, + table: str, + headers: List[str], + rows: List[Tuple], + override_system_value: bool = False, ) -> None: - sql = "INSERT INTO %s (%s) VALUES (%s)" % ( + sql = "INSERT INTO %s (%s) %s VALUES (%s)" % ( table, ", ".join(k for k in headers), + "OVERRIDING SYSTEM VALUE" if override_system_value else "", ", ".join("%s" for _ in headers), ) @@ -296,6 +324,7 @@ class MockHomeserver: self.config = config self.hostname = config.server.server_name self.version_string = SYNAPSE_VERSION + self.instance_id = random_string(5) def get_clock(self) -> Clock: return self.clock @@ -303,6 +332,9 @@ class MockHomeserver: def get_reactor(self) -> ISynapseReactor: return reactor + def get_instance_id(self) -> str: + return self.instance_id + def get_instance_name(self) -> str: return "master" @@ -517,7 +549,13 @@ class Porter: def insert(txn: LoggingTransaction) -> None: assert headers is not None - self.postgres_store.insert_many_txn(txn, table, headers[1:], rows) + self.postgres_store.insert_many_txn( + txn, + table, + headers[1:], + rows, + override_system_value=table in AUTOINCREMENT_TABLES, + ) self.postgres_store.db_pool.simple_update_one_txn( txn, @@ -639,14 +677,28 @@ class Porter: engine = create_engine(db_config.config) hs = MockHomeserver(self.hs_config) + server_name = hs.hostname - with make_conn(db_config, engine, "portdb") as db_conn: + with make_conn( + db_config=db_config, + engine=engine, + default_txn_name="portdb", + server_name=server_name, + ) as db_conn: engine.check_database( db_conn, allow_outdated_version=allow_outdated_version ) prepare_database(db_conn, engine, config=self.hs_config) # Type safety: ignore that we're using Mock homeservers here. - store = Store(DatabasePool(hs, db_config, engine), db_conn, hs) # type: ignore[arg-type] + store = Store( + DatabasePool( + hs, # type: ignore[arg-type] + db_config, + engine, + ), + db_conn, + hs, # type: ignore[arg-type] + ) db_conn.commit() return store @@ -687,6 +739,20 @@ class Porter: # 0 means off. 1 means full. 2 means incremental. return autovacuum_setting != 0 + async def remove_ignored_background_updates_from_database(self) -> None: + def _remove_delete_unreferenced_state_groups_bg_updates( + txn: LoggingTransaction, + ) -> None: + txn.execute( + "DELETE FROM background_updates WHERE update_name = ANY(?)", + (list(IGNORED_BACKGROUND_UPDATES),), + ) + + await self.postgres_store.db_pool.runInteraction( + "remove_delete_unreferenced_state_groups_bg_updates", + _remove_delete_unreferenced_state_groups_bg_updates, + ) + async def run(self) -> None: """Ports the SQLite database to a PostgreSQL database. @@ -732,6 +798,8 @@ class Porter: self.hs_config.database.get_single_database() ) + await self.remove_ignored_background_updates_from_database() + await self.run_background_updates_on_postgres() self.progress.set_state("Creating port tables") @@ -853,6 +921,19 @@ class Porter: ], ) + await self._setup_autoincrement_sequence( + "sliding_sync_connection_positions", "connection_position" + ) + await self._setup_autoincrement_sequence( + "sliding_sync_connection_required_state", "required_state_id" + ) + await self._setup_autoincrement_sequence( + "sliding_sync_connections", "connection_key" + ) + await self._setup_autoincrement_sequence( + "state_groups_pending_deletion", "sequence_number" + ) + # Step 3. Get tables. self.progress.set_state("Fetching tables") sqlite_tables = await self.sqlite_store.db_pool.simple_select_onecol( @@ -1034,7 +1115,7 @@ class Porter: def get_sent_table_size(txn: LoggingTransaction) -> int: txn.execute( - "SELECT count(*) FROM sent_transactions" " WHERE ts >= ?", (yesterday,) + "SELECT count(*) FROM sent_transactions WHERE ts >= ?", (yesterday,) ) result = txn.fetchone() assert result is not None @@ -1185,6 +1266,49 @@ class Porter: "_setup_%s" % (sequence_name,), r ) + async def _setup_autoincrement_sequence( + self, + sqlite_table_name: str, + sqlite_id_column_name: str, + ) -> None: + """Set a sequence to the correct value. Use where id column was declared with PRIMARY KEY AUTOINCREMENT.""" + seq_name = await self._pg_get_serial_sequence( + sqlite_table_name, sqlite_id_column_name + ) + if seq_name is None: + raise Exception( + "implicit sequence not found for table " + sqlite_table_name + ) + + seq_value = await self.sqlite_store.db_pool.simple_select_one_onecol( + table="sqlite_sequence", + keyvalues={"name": sqlite_table_name}, + retcol="seq", + allow_none=True, + ) + if seq_value is None: + return + + def r(txn: LoggingTransaction) -> None: + sql = "ALTER SEQUENCE %s RESTART WITH" % (seq_name,) + txn.execute(sql + " %s", (seq_value + 1,)) + + await self.postgres_store.db_pool.runInteraction("_setup_%s" % (seq_name,), r) + + async def _pg_get_serial_sequence(self, table: str, column: str) -> Optional[str]: + """Returns the name of the postgres sequence associated with a column, or NULL.""" + + def r(txn: LoggingTransaction) -> Optional[str]: + txn.execute("SELECT pg_get_serial_sequence('%s', '%s')" % (table, column)) + result = txn.fetchone() + if not result: + return None + return result[0] + + return await self.postgres_store.db_pool.runInteraction( + "_pg_get_serial_sequence", r + ) + async def _setup_auth_chain_sequence(self) -> None: curr_chain_id: Optional[ int diff --git a/synapse/_scripts/synctl.py b/synapse/_scripts/synctl.py index 688df9485c..2e2aa27a17 100755 --- a/synapse/_scripts/synctl.py +++ b/synapse/_scripts/synctl.py @@ -292,9 +292,9 @@ def main() -> None: for key in worker_config: if key == "worker_app": # But we allow worker_app continue - assert not key.startswith( - "worker_" - ), "Main process cannot use worker_* config" + assert not key.startswith("worker_"), ( + "Main process cannot use worker_* config" + ) else: worker_pidfile = worker_config["worker_pid_file"] worker_cache_factor = worker_config.get("synctl_cache_factor") diff --git a/synapse/_scripts/update_synapse_database.py b/synapse/_scripts/update_synapse_database.py index d8b4dbd6c6..3624db3544 100644 --- a/synapse/_scripts/update_synapse_database.py +++ b/synapse/_scripts/update_synapse_database.py @@ -53,6 +53,7 @@ class MockHomeserver(HomeServer): def run_background_updates(hs: HomeServer) -> None: + server_name = hs.hostname main = hs.get_datastores().main state = hs.get_datastores().state @@ -66,7 +67,11 @@ def run_background_updates(hs: HomeServer) -> None: def run() -> None: # Apply all background updates on the database. defer.ensureDeferred( - run_as_background_process("background_updates", run_background_updates) + run_as_background_process( + "background_updates", + server_name, + run_background_updates, + ) ) reactor.callWhenRunning(run) @@ -115,6 +120,13 @@ def main() -> None: # DB. hs.setup() + # This will cause all of the relevant storage classes to be instantiated and call + # `register_background_update_handler(...)`, + # `register_background_index_update(...)`, + # `register_background_validate_constraint(...)`, etc so they are available to use + # if we are asked to run those background updates. + hs.get_storage_controllers() + if args.run_background_updates: run_background_updates(hs) diff --git a/synapse/api/auth/__init__.py b/synapse/api/auth/__init__.py index d5241afe73..d253938329 100644 --- a/synapse/api/auth/__init__.py +++ b/synapse/api/auth/__init__.py @@ -18,14 +18,15 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional, Protocol, Tuple -from typing_extensions import Protocol +from prometheus_client import Histogram from twisted.web.server import Request from synapse.appservice import ApplicationService from synapse.http.site import SynapseRequest +from synapse.metrics import SERVER_NAME_LABEL from synapse.types import Requester if TYPE_CHECKING: @@ -35,6 +36,13 @@ if TYPE_CHECKING: GUEST_DEVICE_ID = "guest_device" +introspection_response_timer = Histogram( + "synapse_api_auth_delegated_introspection_response", + "Time taken to get a response for an introspection request", + labelnames=["code", SERVER_NAME_LABEL], +) + + class Auth(Protocol): """The interface that an auth provider must implement.""" diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index fc1c0cc903..f97a71caf7 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -37,7 +37,9 @@ from synapse.appservice import ApplicationService from synapse.http import get_request_user_agent from synapse.http.site import SynapseRequest from synapse.logging.opentracing import trace +from synapse.state import CREATE_KEY, POWER_KEY from synapse.types import Requester, create_requester +from synapse.types.state import StateFilter from synapse.util.cancellation import cancellable if TYPE_CHECKING: @@ -170,7 +172,7 @@ class BaseAuth: """ # It's ok if the app service is trying to use the sender from their registration - if app_service.sender == user_id: + if app_service.sender.to_string() == user_id: pass # Check to make sure the app service is allowed to control the user elif not app_service.is_interested_in_user(user_id): @@ -216,18 +218,20 @@ class BaseAuth: # by checking if they would (theoretically) be able to change the # m.room.canonical_alias events - power_level_event = ( - await self._storage_controllers.state.get_current_state_event( - room_id, EventTypes.PowerLevels, "" - ) + auth_events = await self._storage_controllers.state.get_current_state( + room_id, + StateFilter.from_types( + [ + POWER_KEY, + CREATE_KEY, + ] + ), ) - auth_events = {} - if power_level_event: - auth_events[(EventTypes.PowerLevels, "")] = power_level_event - send_level = event_auth.get_send_level( - EventTypes.CanonicalAlias, "", power_level_event + EventTypes.CanonicalAlias, + "", + auth_events.get(POWER_KEY), ) user_level = event_auth.get_user_power_level( requester.user.to_string(), auth_events diff --git a/synapse/api/auth/internal.py b/synapse/api/auth/internal.py index 9fd4db68e1..b33384c13f 100644 --- a/synapse/api/auth/internal.py +++ b/synapse/api/auth/internal.py @@ -29,6 +29,7 @@ from synapse.api.errors import ( InvalidClientTokenError, MissingClientTokenError, UnrecognizedRequestError, + UserLockedError, ) from synapse.http.site import SynapseRequest from synapse.logging.opentracing import active_span, force_tracing, start_active_span @@ -162,12 +163,7 @@ class InternalAuth(BaseAuth): if not allow_locked and await self.store.get_user_locked_status( requester.user.to_string() ): - raise AuthError( - 401, - "User account has been locked", - errcode=Codes.USER_LOCKED, - additional_fields={"soft_logout": True}, - ) + raise UserLockedError() # Deny the request if the user account has expired. # This check is only done for regular users, not appservice ones. @@ -300,4 +296,4 @@ class InternalAuth(BaseAuth): Returns: True if the user is an admin """ - return await self.store.is_server_admin(requester.user) + return await self.store.is_server_admin(requester.user.to_string()) diff --git a/synapse/api/auth/mas.py b/synapse/api/auth/mas.py new file mode 100644 index 0000000000..40b4a5bd34 --- /dev/null +++ b/synapse/api/auth/mas.py @@ -0,0 +1,438 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +import logging +from typing import TYPE_CHECKING, Optional, Set +from urllib.parse import urlencode + +from synapse._pydantic_compat import ( + BaseModel, + Extra, + StrictBool, + StrictInt, + StrictStr, + ValidationError, +) +from synapse.api.auth.base import BaseAuth +from synapse.api.errors import ( + AuthError, + HttpResponseException, + InvalidClientTokenError, + SynapseError, + UnrecognizedRequestError, +) +from synapse.http.site import SynapseRequest +from synapse.logging.context import PreserveLoggingContext +from synapse.logging.opentracing import ( + active_span, + force_tracing, + inject_request_headers, + start_active_span, +) +from synapse.metrics import SERVER_NAME_LABEL +from synapse.synapse_rust.http_client import HttpClient +from synapse.types import JsonDict, Requester, UserID, create_requester +from synapse.util import json_decoder +from synapse.util.caches.cached_call import RetryOnExceptionCachedCall +from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext + +from . import introspection_response_timer + +if TYPE_CHECKING: + from synapse.rest.admin.experimental_features import ExperimentalFeature + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + +# Scope as defined by MSC2967 +# https://github.com/matrix-org/matrix-spec-proposals/pull/2967 +UNSTABLE_SCOPE_MATRIX_API = "urn:matrix:org.matrix.msc2967.client:api:*" +UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:org.matrix.msc2967.client:device:" +STABLE_SCOPE_MATRIX_API = "urn:matrix:client:api:*" +STABLE_SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:client:device:" + + +class ServerMetadata(BaseModel): + class Config: + extra = Extra.allow + + issuer: StrictStr + account_management_uri: StrictStr + + +class IntrospectionResponse(BaseModel): + retrieved_at_ms: StrictInt + active: StrictBool + scope: Optional[StrictStr] + username: Optional[StrictStr] + sub: Optional[StrictStr] + device_id: Optional[StrictStr] + expires_in: Optional[StrictInt] + + class Config: + extra = Extra.allow + + def get_scope_set(self) -> set[str]: + if not self.scope: + return set() + + return {token for token in self.scope.split(" ") if token} + + def is_active(self, now_ms: int) -> bool: + if not self.active: + return False + + # Compatibility tokens don't expire and don't have an 'expires_in' field + if self.expires_in is None: + return True + + absolute_expiry_ms = self.expires_in * 1000 + self.retrieved_at_ms + return now_ms < absolute_expiry_ms + + +class MasDelegatedAuth(BaseAuth): + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.server_name = hs.hostname + self._clock = hs.get_clock() + self._config = hs.config.mas + + self._http_client = hs.get_proxied_http_client() + self._rust_http_client = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + self._server_metadata = RetryOnExceptionCachedCall[ServerMetadata]( + self._load_metadata + ) + self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users + + # # Token Introspection Cache + # This remembers what users/devices are represented by which access tokens, + # in order to reduce overall system load: + # - on Synapse (as requests are relatively expensive) + # - on the network + # - on MAS + # + # Since there is no invalidation mechanism currently, + # the entries expire after 2 minutes. + # This does mean tokens can be treated as valid by Synapse + # for longer than reality. + # + # Ideally, tokens should logically be invalidated in the following circumstances: + # - If a session logout happens. + # In this case, MAS will delete the device within Synapse + # anyway and this is good enough as an invalidation. + # - If the client refreshes their token in MAS. + # In this case, the device still exists and it's not the end of the world for + # the old access token to continue working for a short time. + self._introspection_cache: ResponseCache[str] = ResponseCache( + clock=self._clock, + name="mas_token_introspection", + server_name=self.server_name, + timeout_ms=120_000, + # don't log because the keys are access tokens + enable_logging=False, + ) + + @property + def _metadata_url(self) -> str: + return f"{self._config.endpoint.rstrip('/')}/.well-known/openid-configuration" + + @property + def _introspection_endpoint(self) -> str: + return f"{self._config.endpoint.rstrip('/')}/oauth2/introspect" + + async def _load_metadata(self) -> ServerMetadata: + response = await self._http_client.get_json(self._metadata_url) + metadata = ServerMetadata(**response) + return metadata + + async def issuer(self) -> str: + metadata = await self._server_metadata.get() + return metadata.issuer + + async def account_management_url(self) -> str: + metadata = await self._server_metadata.get() + return metadata.account_management_uri + + async def auth_metadata(self) -> JsonDict: + metadata = await self._server_metadata.get() + return metadata.dict() + + def is_request_using_the_shared_secret(self, request: SynapseRequest) -> bool: + """ + Check if the request is using the shared secret. + + Args: + request: The request to check. + + Returns: + True if the request is using the shared secret, False otherwise. + """ + access_token = self.get_access_token_from_request(request) + shared_secret = self._config.secret() + if not shared_secret: + return False + + return access_token == shared_secret + + async def _introspect_token( + self, token: str, cache_context: ResponseCacheContext[str] + ) -> IntrospectionResponse: + """ + Send a token to the introspection endpoint and returns the introspection response + + Parameters: + token: The token to introspect + + Raises: + HttpResponseException: If the introspection endpoint returns a non-2xx response + ValueError: If the introspection endpoint returns an invalid JSON response + JSONDecodeError: If the introspection endpoint returns a non-JSON response + Exception: If the HTTP request fails + + Returns: + The introspection response + """ + + # By default, we shouldn't cache the result unless we know it's valid + cache_context.should_cache = False + raw_headers: dict[str, str] = { + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json", + "Authorization": f"Bearer {self._config.secret()}", + # Tell MAS that we support reading the device ID as an explicit + # value, not encoded in the scope. This is supported by MAS 0.15+ + "X-MAS-Supports-Device-Id": "1", + } + + args = {"token": token, "token_type_hint": "access_token"} + body = urlencode(args, True) + + # Do the actual request + + logger.debug("Fetching token from MAS") + start_time = self._clock.time() + try: + with start_active_span("mas-introspect-token"): + inject_request_headers(raw_headers) + with PreserveLoggingContext(): + resp_body = await self._rust_http_client.post( + url=self._introspection_endpoint, + response_limit=1 * 1024 * 1024, + headers=raw_headers, + request_body=body, + ) + except HttpResponseException as e: + end_time = self._clock.time() + introspection_response_timer.labels( + code=e.code, **{SERVER_NAME_LABEL: self.server_name} + ).observe(end_time - start_time) + raise + except Exception: + end_time = self._clock.time() + introspection_response_timer.labels( + code="ERR", **{SERVER_NAME_LABEL: self.server_name} + ).observe(end_time - start_time) + raise + + logger.debug("Fetched token from MAS") + + end_time = self._clock.time() + introspection_response_timer.labels( + code=200, **{SERVER_NAME_LABEL: self.server_name} + ).observe(end_time - start_time) + + raw_response = json_decoder.decode(resp_body.decode("utf-8")) + try: + response = IntrospectionResponse( + retrieved_at_ms=self._clock.time_msec(), + **raw_response, + ) + except ValidationError as e: + raise ValueError( + "The introspection endpoint returned an invalid JSON response" + ) from e + + # We had a valid response, so we can cache it + cache_context.should_cache = True + return response + + async def is_server_admin(self, requester: Requester) -> bool: + return "urn:synapse:admin:*" in requester.scope + + async def get_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, + ) -> Requester: + parent_span = active_span() + with start_active_span("get_user_by_req"): + access_token = self.get_access_token_from_request(request) + + requester = await self.get_appservice_user(request, access_token) + if not requester: + requester = await self.get_user_by_access_token( + token=access_token, + allow_expired=allow_expired, + ) + + await self._record_request(request, requester) + + request.requester = requester + + if parent_span: + if requester.authenticated_entity in self._force_tracing_for_users: + # request tracing is enabled for this user, so we need to force it + # tracing on for the parent span (which will be the servlet span). + # + # It's too late for the get_user_by_req span to inherit the setting, + # so we also force it on for that. + force_tracing() + force_tracing(parent_span) + parent_span.set_tag( + "authenticated_entity", requester.authenticated_entity + ) + parent_span.set_tag("user_id", requester.user.to_string()) + if requester.device_id is not None: + parent_span.set_tag("device_id", requester.device_id) + if requester.app_service is not None: + parent_span.set_tag("appservice_id", requester.app_service.id) + return requester + + async def get_user_by_access_token( + self, + token: str, + allow_expired: bool = False, + ) -> Requester: + try: + introspection_result = await self._introspection_cache.wrap( + token, self._introspect_token, token, cache_context=True + ) + except Exception: + logger.exception("Failed to introspect token") + raise SynapseError(503, "Unable to introspect the access token") + + logger.debug("Introspection result: %r", introspection_result) + if not introspection_result.is_active(self._clock.time_msec()): + raise InvalidClientTokenError("Token is not active") + + # Let's look at the scope + scope = introspection_result.get_scope_set() + + # Determine type of user based on presence of particular scopes + if ( + UNSTABLE_SCOPE_MATRIX_API not in scope + and STABLE_SCOPE_MATRIX_API not in scope + ): + raise InvalidClientTokenError( + "Token doesn't grant access to the Matrix C-S API" + ) + + if introspection_result.username is None: + raise AuthError( + 500, + "Invalid username claim in the introspection result", + ) + + user_id = UserID( + localpart=introspection_result.username, + domain=self.server_name, + ) + + # Try to find a user from the username claim + user_info = await self.store.get_user_by_id(user_id=user_id.to_string()) + if user_info is None: + raise AuthError( + 500, + "User not found", + ) + + # MAS will give us the device ID as an explicit value for *compatibility* sessions + # If present, we get it from here, if not we get it in the scope for next-gen sessions + device_id = introspection_result.device_id + if device_id is None: + # Find device_ids in scope + # We only allow a single device_id in the scope, so we find them all in the + # scope list, and raise if there are more than one. The OIDC server should be + # the one enforcing valid scopes, so we raise a 500 if we find an invalid scope. + device_ids: Set[str] = set() + for tok in scope: + if tok.startswith(UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX): + device_ids.add(tok[len(UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX) :]) + elif tok.startswith(STABLE_SCOPE_MATRIX_DEVICE_PREFIX): + device_ids.add(tok[len(STABLE_SCOPE_MATRIX_DEVICE_PREFIX) :]) + + if len(device_ids) > 1: + raise AuthError( + 500, + "Multiple device IDs in scope", + ) + + device_id = next(iter(device_ids), None) + + if device_id is not None: + # Sanity check the device_id + if len(device_id) > 255 or len(device_id) < 1: + raise AuthError( + 500, + "Invalid device ID in introspection result", + ) + + # Make sure the device exists. This helps with introspection cache + # invalidation: if we log out, the device gets deleted by MAS + device = await self.store.get_device( + user_id=user_id.to_string(), + device_id=device_id, + ) + if device is None: + # Invalidate the introspection cache, the device was deleted + self._introspection_cache.unset(token) + raise InvalidClientTokenError("Token is not active") + + return create_requester( + user_id=user_id, + device_id=device_id, + scope=scope, + ) + + async def get_user_by_req_experimental_feature( + self, + request: SynapseRequest, + feature: "ExperimentalFeature", + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, + ) -> Requester: + try: + requester = await self.get_user_by_req( + request, + allow_guest=allow_guest, + allow_expired=allow_expired, + allow_locked=allow_locked, + ) + if await self.store.is_feature_enabled(requester.user.to_string(), feature): + return requester + + raise UnrecognizedRequestError(code=404) + except (AuthError, InvalidClientTokenError): + if feature.is_globally_enabled(self.hs.config): + # If its globally enabled then return the auth error + raise + + raise UnrecognizedRequestError(code=404) diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 53907c01d4..c406c683e7 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -19,7 +19,8 @@ # # import logging -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set from urllib.parse import urlencode from authlib.oauth2 import ClientAuth @@ -27,26 +28,31 @@ from authlib.oauth2.auth import encode_client_secret_basic, encode_client_secret from authlib.oauth2.rfc7523 import ClientSecretJWT, PrivateKeyJWT, private_key_jwt_sign from authlib.oauth2.rfc7662 import IntrospectionToken from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url -from prometheus_client import Histogram - -from twisted.web.client import readBody -from twisted.web.http_headers import Headers from synapse.api.auth.base import BaseAuth from synapse.api.errors import ( AuthError, HttpResponseException, InvalidClientTokenError, - OAuthInsufficientScopeError, - StoreError, SynapseError, UnrecognizedRequestError, ) from synapse.http.site import SynapseRequest -from synapse.logging.context import make_deferred_yieldable +from synapse.logging.context import PreserveLoggingContext +from synapse.logging.opentracing import ( + active_span, + force_tracing, + inject_request_headers, + start_active_span, +) +from synapse.metrics import SERVER_NAME_LABEL +from synapse.synapse_rust.http_client import HttpClient from synapse.types import Requester, UserID, create_requester from synapse.util import json_decoder from synapse.util.caches.cached_call import RetryOnExceptionCachedCall +from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext + +from . import introspection_response_timer if TYPE_CHECKING: from synapse.rest.admin.experimental_features import ExperimentalFeature @@ -54,18 +60,12 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -introspection_response_timer = Histogram( - "synapse_api_auth_delegated_introspection_response", - "Time taken to get a response for an introspection request", - ["code"], -) - - # Scope as defined by MSC2967 # https://github.com/matrix-org/matrix-spec-proposals/pull/2967 -SCOPE_MATRIX_API = "urn:matrix:org.matrix.msc2967.client:api:*" -SCOPE_MATRIX_GUEST = "urn:matrix:org.matrix.msc2967.client:api:guest" -SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:org.matrix.msc2967.client:device:" +UNSTABLE_SCOPE_MATRIX_API = "urn:matrix:org.matrix.msc2967.client:api:*" +UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:org.matrix.msc2967.client:device:" +STABLE_SCOPE_MATRIX_API = "urn:matrix:client:api:*" +STABLE_SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:client:device:" # Scope which allows access to the Synapse admin API SCOPE_SYNAPSE_ADMIN = "urn:synapse:admin:*" @@ -76,6 +76,61 @@ def scope_to_list(scope: str) -> List[str]: return scope.strip().split(" ") +@dataclass +class IntrospectionResult: + _inner: IntrospectionToken + + # when we retrieved this token, + # in milliseconds since the Unix epoch + retrieved_at_ms: int + + def is_active(self, now_ms: int) -> bool: + if not self._inner.get("active"): + return False + + expires_in = self._inner.get("expires_in") + if expires_in is None: + return True + if not isinstance(expires_in, int): + raise InvalidClientTokenError("token `expires_in` is not an int") + + absolute_expiry_ms = expires_in * 1000 + self.retrieved_at_ms + return now_ms < absolute_expiry_ms + + def get_scope_list(self) -> List[str]: + value = self._inner.get("scope") + if not isinstance(value, str): + return [] + return scope_to_list(value) + + def get_sub(self) -> Optional[str]: + value = self._inner.get("sub") + if not isinstance(value, str): + return None + return value + + def get_username(self) -> Optional[str]: + value = self._inner.get("username") + if not isinstance(value, str): + return None + return value + + def get_name(self) -> Optional[str]: + value = self._inner.get("name") + if not isinstance(value, str): + return None + return value + + def get_device_id(self) -> Optional[str]: + value = self._inner.get("device_id") + if value is not None and not isinstance(value, str): + raise AuthError( + 500, + "Invalid device ID in introspection result", + ) + return value + + class PrivateKeyJWTWithKid(PrivateKeyJWT): # type: ignore[misc] """An implementation of the private_key_jwt client auth method that includes a kid header. @@ -116,10 +171,45 @@ class MSC3861DelegatedAuth(BaseAuth): assert self._config.client_id, "No client_id provided" assert auth_method is not None, "Invalid client_auth_method provided" + self.server_name = hs.hostname self._clock = hs.get_clock() self._http_client = hs.get_proxied_http_client() self._hostname = hs.hostname - self._admin_token = self._config.admin_token + self._admin_token: Callable[[], Optional[str]] = self._config.admin_token + self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users + + self._rust_http_client = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # # Token Introspection Cache + # This remembers what users/devices are represented by which access tokens, + # in order to reduce overall system load: + # - on Synapse (as requests are relatively expensive) + # - on the network + # - on MAS + # + # Since there is no invalidation mechanism currently, + # the entries expire after 2 minutes. + # This does mean tokens can be treated as valid by Synapse + # for longer than reality. + # + # Ideally, tokens should logically be invalidated in the following circumstances: + # - If a session logout happens. + # In this case, MAS will delete the device within Synapse + # anyway and this is good enough as an invalidation. + # - If the client refreshes their token in MAS. + # In this case, the device still exists and it's not the end of the world for + # the old access token to continue working for a short time. + self._introspection_cache: ResponseCache[str] = ResponseCache( + clock=self._clock, + name="token_introspection", + server_name=self.server_name, + timeout_ms=120_000, + # don't log because the keys are access tokens + enable_logging=False, + ) self._issuer_metadata = RetryOnExceptionCachedCall[OpenIDProviderMetadata]( self._load_metadata @@ -133,9 +223,10 @@ class MSC3861DelegatedAuth(BaseAuth): ) else: # Else use the client secret - assert self._config.client_secret, "No client_secret provided" + client_secret = self._config.client_secret() + assert client_secret, "No client_secret provided" self._client_auth = ClientAuth( - self._config.client_id, self._config.client_secret, auth_method + self._config.client_id, client_secret, auth_method ) async def _load_metadata(self) -> OpenIDProviderMetadata: @@ -174,6 +265,12 @@ class MSC3861DelegatedAuth(BaseAuth): logger.warning("Failed to load metadata:", exc_info=True) return None + async def auth_metadata(self) -> Dict[str, Any]: + """ + Returns the auth metadata dict + """ + return await self._issuer_metadata.get() + async def _introspection_endpoint(self) -> str: """ Returns the introspection endpoint of the issuer @@ -186,7 +283,9 @@ class MSC3861DelegatedAuth(BaseAuth): metadata = await self._issuer_metadata.get() return metadata.get("introspection_endpoint") - async def _introspect_token(self, token: str) -> IntrospectionToken: + async def _introspect_token( + self, token: str, cache_context: ResponseCacheContext[str] + ) -> IntrospectionResult: """ Send a token to the introspection endpoint and returns the introspection response @@ -202,11 +301,15 @@ class MSC3861DelegatedAuth(BaseAuth): Returns: The introspection response """ + # By default, we shouldn't cache the result unless we know it's valid + cache_context.should_cache = False introspection_endpoint = await self._introspection_endpoint() raw_headers: Dict[str, str] = { "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": str(self._http_client.user_agent, "utf-8"), "Accept": "application/json", + # Tell MAS that we support reading the device ID as an explicit + # value, not encoded in the scope. This is supported by MAS 0.15+ + "X-MAS-Supports-Device-Id": "1", } args = {"token": token, "token_type_hint": "access_token"} @@ -216,38 +319,40 @@ class MSC3861DelegatedAuth(BaseAuth): uri, raw_headers, body = self._client_auth.prepare( method="POST", uri=introspection_endpoint, headers=raw_headers, body=body ) - headers = Headers({k: [v] for (k, v) in raw_headers.items()}) # Do the actual request - # We're not using the SimpleHttpClient util methods as we don't want to - # check the HTTP status code, and we do the body encoding ourselves. + logger.debug("Fetching token from MAS") start_time = self._clock.time() try: - response = await self._http_client.request( - method="POST", - uri=uri, - data=body.encode("utf-8"), - headers=headers, - ) - - resp_body = await make_deferred_yieldable(readBody(response)) + with start_active_span("mas-introspect-token"): + inject_request_headers(raw_headers) + with PreserveLoggingContext(): + resp_body = await self._rust_http_client.post( + url=uri, + response_limit=1 * 1024 * 1024, + headers=raw_headers, + request_body=body, + ) + except HttpResponseException as e: + end_time = self._clock.time() + introspection_response_timer.labels( + code=e.code, **{SERVER_NAME_LABEL: self.server_name} + ).observe(end_time - start_time) + raise except Exception: end_time = self._clock.time() - introspection_response_timer.labels("ERR").observe(end_time - start_time) + introspection_response_timer.labels( + code="ERR", **{SERVER_NAME_LABEL: self.server_name} + ).observe(end_time - start_time) raise - end_time = self._clock.time() - introspection_response_timer.labels(response.code).observe( - end_time - start_time - ) + logger.debug("Fetched token from MAS") - if response.code < 200 or response.code >= 300: - raise HttpResponseException( - response.code, - response.phrase.decode("ascii", errors="replace"), - resp_body, - ) + end_time = self._clock.time() + introspection_response_timer.labels( + code=200, **{SERVER_NAME_LABEL: self.server_name} + ).observe(end_time - start_time) resp = json_decoder.decode(resp_body.decode("utf-8")) @@ -256,17 +361,76 @@ class MSC3861DelegatedAuth(BaseAuth): "The introspection endpoint returned an invalid JSON response." ) - return IntrospectionToken(**resp) + # We had a valid response, so we can cache it + cache_context.should_cache = True + return IntrospectionResult( + IntrospectionToken(**resp), retrieved_at_ms=self._clock.time_msec() + ) async def is_server_admin(self, requester: Requester) -> bool: return "urn:synapse:admin:*" in requester.scope + def _is_access_token_the_admin_token(self, token: str) -> bool: + admin_token = self._admin_token() + if admin_token is None: + return False + return token == admin_token + async def get_user_by_req( self, request: SynapseRequest, allow_guest: bool = False, allow_expired: bool = False, allow_locked: bool = False, + ) -> Requester: + """Get a registered user's ID. + + Args: + request: An HTTP request with an access_token query parameter. + allow_guest: If False, will raise an AuthError if the user making the + request is a guest. + allow_expired: If True, allow the request through even if the account + is expired, or session token lifetime has ended. Note that + /login will deliver access tokens regardless of expiration. + + Returns: + Resolves to the requester + Raises: + InvalidClientCredentialsError if no user by that token exists or the token + is invalid. + AuthError if access is denied for the user in the access token + """ + parent_span = active_span() + with start_active_span("get_user_by_req"): + requester = await self._wrapped_get_user_by_req( + request, allow_guest, allow_expired, allow_locked + ) + + if parent_span: + if requester.authenticated_entity in self._force_tracing_for_users: + # request tracing is enabled for this user, so we need to force it + # tracing on for the parent span (which will be the servlet span). + # + # It's too late for the get_user_by_req span to inherit the setting, + # so we also force it on for that. + force_tracing() + force_tracing(parent_span) + parent_span.set_tag( + "authenticated_entity", requester.authenticated_entity + ) + parent_span.set_tag("user_id", requester.user.to_string()) + if requester.device_id is not None: + parent_span.set_tag("device_id", requester.device_id) + if requester.app_service is not None: + parent_span.set_tag("appservice_id", requester.app_service.id) + return requester + + async def _wrapped_get_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, ) -> Requester: access_token = self.get_access_token_from_request(request) @@ -277,12 +441,9 @@ class MSC3861DelegatedAuth(BaseAuth): requester = await self.get_user_by_access_token(access_token, allow_expired) # Do not record requests from MAS using the virtual `__oidc_admin` user. - if access_token != self._admin_token: + if not self._is_access_token_the_admin_token(access_token): await self._record_request(request, requester) - if not allow_guest and requester.is_guest: - raise OAuthInsufficientScopeError([SCOPE_MATRIX_API]) - request.requester = requester return requester @@ -313,16 +474,29 @@ class MSC3861DelegatedAuth(BaseAuth): raise UnrecognizedRequestError(code=404) + def is_request_using_the_admin_token(self, request: SynapseRequest) -> bool: + """ + Check if the request is using the admin token. + + Args: + request: The request to check. + + Returns: + True if the request is using the admin token, False otherwise. + """ + access_token = self.get_access_token_from_request(request) + return self._is_access_token_the_admin_token(access_token) + async def get_user_by_access_token( self, token: str, allow_expired: bool = False, ) -> Requester: - if self._admin_token is not None and token == self._admin_token: + if self._is_access_token_the_admin_token(token): # XXX: This is a temporary solution so that the admin API can be called by # the OIDC provider. This will be removed once we have OIDC client # credentials grant support in matrix-authentication-service. - logging.info("Admin toked used") + logger.info("Admin token used") # XXX: that user doesn't exist and won't be provisioned. # This is mostly fine for admin calls, but we should also think about doing # requesters without a user_id. @@ -333,7 +507,9 @@ class MSC3861DelegatedAuth(BaseAuth): ) try: - introspection_result = await self._introspect_token(token) + introspection_result = await self._introspection_cache.wrap( + token, self._introspect_token, token, cache_context=True + ) except Exception: logger.exception("Failed to introspect token") raise SynapseError(503, "Unable to introspect the access token") @@ -342,21 +518,22 @@ class MSC3861DelegatedAuth(BaseAuth): # TODO: introspection verification should be more extensive, especially: # - verify the audience - if not introspection_result.get("active"): + if not introspection_result.is_active(self._clock.time_msec()): raise InvalidClientTokenError("Token is not active") # Let's look at the scope - scope: List[str] = scope_to_list(introspection_result.get("scope", "")) + scope: List[str] = introspection_result.get_scope_list() # Determine type of user based on presence of particular scopes - has_user_scope = SCOPE_MATRIX_API in scope - has_guest_scope = SCOPE_MATRIX_GUEST in scope + has_user_scope = ( + UNSTABLE_SCOPE_MATRIX_API in scope or STABLE_SCOPE_MATRIX_API in scope + ) - if not has_user_scope and not has_guest_scope: + if not has_user_scope: raise InvalidClientTokenError("No scope in token granting user rights") # Match via the sub claim - sub: Optional[str] = introspection_result.get("sub") + sub = introspection_result.get_sub() if sub is None: raise InvalidClientTokenError( "Invalid sub claim in the introspection result" @@ -369,29 +546,20 @@ class MSC3861DelegatedAuth(BaseAuth): # If we could not find a user via the external_id, it either does not exist, # or the external_id was never recorded - # TODO: claim mapping should be configurable - username: Optional[str] = introspection_result.get("username") - if username is None or not isinstance(username, str): + username = introspection_result.get_username() + if username is None: raise AuthError( 500, "Invalid username claim in the introspection result", ) user_id = UserID(username, self._hostname) - # First try to find a user from the username claim + # Try to find a user from the username claim user_info = await self.store.get_user_by_id(user_id=user_id.to_string()) if user_info is None: - # If the user does not exist, we should create it on the fly - # TODO: we could use SCIM to provision users ahead of time and listen - # for SCIM SET events if those ever become standard: - # https://datatracker.ietf.org/doc/html/draft-hunt-scim-notify-00 - - # TODO: claim mapping should be configurable - # If present, use the name claim as the displayname - name: Optional[str] = introspection_result.get("name") - - await self.store.register_user( - user_id=user_id.to_string(), create_profile_with_displayname=name + raise AuthError( + 500, + "User not found", ) # And record the sub as external_id @@ -401,42 +569,41 @@ class MSC3861DelegatedAuth(BaseAuth): else: user_id = UserID.from_string(user_id_str) - # Find device_ids in scope - # We only allow a single device_id in the scope, so we find them all in the - # scope list, and raise if there are more than one. The OIDC server should be - # the one enforcing valid scopes, so we raise a 500 if we find an invalid scope. - device_ids = [ - tok[len(SCOPE_MATRIX_DEVICE_PREFIX) :] - for tok in scope - if tok.startswith(SCOPE_MATRIX_DEVICE_PREFIX) - ] + # MAS 0.15+ will give us the device ID as an explicit value for compatibility sessions + # If present, we get it from here, if not we get it in thee scope + device_id = introspection_result.get_device_id() + if device_id is None: + # Find device_ids in scope + # We only allow a single device_id in the scope, so we find them all in the + # scope list, and raise if there are more than one. The OIDC server should be + # the one enforcing valid scopes, so we raise a 500 if we find an invalid scope. + device_ids: Set[str] = set() + for tok in scope: + if tok.startswith(UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX): + device_ids.add(tok[len(UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX) :]) + elif tok.startswith(STABLE_SCOPE_MATRIX_DEVICE_PREFIX): + device_ids.add(tok[len(STABLE_SCOPE_MATRIX_DEVICE_PREFIX) :]) - if len(device_ids) > 1: - raise AuthError( - 500, - "Multiple device IDs in scope", - ) + if len(device_ids) > 1: + raise AuthError( + 500, + "Multiple device IDs in scope", + ) + + device_id = next(iter(device_ids), None) - device_id = device_ids[0] if device_ids else None if device_id is not None: # Sanity check the device_id if len(device_id) > 255 or len(device_id) < 1: raise AuthError( 500, - "Invalid device ID in scope", + "Invalid device ID in introspection result", ) - # Create the device on the fly if it does not exist - try: - await self.store.get_device( - user_id=user_id.to_string(), device_id=device_id - ) - except StoreError: - await self.store.store_device( - user_id=user_id.to_string(), - device_id=device_id, - initial_device_display_name="OIDC-native client", - ) + # Make sure the device exists + await self.store.get_device( + user_id=user_id.to_string(), device_id=device_id + ) # TODO: there is a few things missing in the requester here, which still need # to be figured out, like: @@ -449,5 +616,4 @@ class MSC3861DelegatedAuth(BaseAuth): user_id=user_id, device_id=device_id, scope=scope, - is_guest=(has_guest_scope and not has_user_scope), ) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 8db302b3d8..7a8f546d6b 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -29,8 +29,13 @@ from typing import Final # the max size of a (canonical-json-encoded) event MAX_PDU_SIZE = 65536 -# the "depth" field on events is limited to 2**63 - 1 -MAX_DEPTH = 2**63 - 1 +# Max/min size of ints in canonical JSON +CANONICALJSON_MAX_INT = (2**53) - 1 +CANONICALJSON_MIN_INT = -CANONICALJSON_MAX_INT + +# the "depth" field on events is limited to the same as what +# canonicaljson accepts +MAX_DEPTH = CANONICALJSON_MAX_INT # the maximum length for a room alias is 255 characters MAX_ALIAS_LENGTH = 255 @@ -41,6 +46,9 @@ MAX_USERID_LENGTH = 255 # Constant value used for the pseudo-thread which is the main timeline. MAIN_TIMELINE: Final = "main" +# MAX_INT + 1, so it always trumps any PL in canonical JSON. +CREATOR_POWER_LEVEL = 2**53 + class Membership: """Represents the membership states of a user in a room.""" @@ -180,12 +188,18 @@ ServerNoticeLimitReached: Final = "m.server_notice.usage_limit_reached" class UserTypes: """Allows for user type specific behaviour. With the benefit of hindsight - 'admin' and 'guest' users should also be UserTypes. Normal users are type None + 'admin' and 'guest' users should also be UserTypes. Extra user types can be + added in the configuration. Normal users are type None or one of the extra + user types (if configured). """ SUPPORT: Final = "support" BOT: Final = "bot" - ALL_USER_TYPES: Final = (SUPPORT, BOT) + ALL_BUILTIN_USER_TYPES: Final = (SUPPORT, BOT) + """ + The user types that are built-in to Synapse. Extra user types can be + added in the configuration. + """ class RelationTypes: @@ -224,6 +238,8 @@ class EventContentFields: # # This is deprecated in MSC2175. ROOM_CREATOR: Final = "creator" + # MSC4289 + ADDITIONAL_CREATORS: Final = "additional_creators" # The version of the room for `m.room.create` events. ROOM_VERSION: Final = "room_version" @@ -231,6 +247,8 @@ class EventContentFields: ROOM_NAME: Final = "name" MEMBERSHIP: Final = "membership" + MEMBERSHIP_DISPLAYNAME: Final = "displayname" + MEMBERSHIP_AVATAR_URL: Final = "avatar_url" # Used in m.room.guest_access events. GUEST_ACCESS: Final = "guest_access" @@ -249,6 +267,11 @@ class EventContentFields: TOMBSTONE_SUCCESSOR_ROOM: Final = "replacement_room" + # Used in m.room.topic events. + TOPIC: Final = "topic" + M_TOPIC: Final = "m.topic" + M_TEXT: Final = "m.text" + class EventUnsignedContentFields: """Fields found inside the 'unsigned' data on events""" @@ -257,6 +280,13 @@ class EventUnsignedContentFields: MEMBERSHIP: Final = "membership" +class MTextFields: + """Fields found inside m.text content blocks.""" + + BODY: Final = "body" + MIMETYPE: Final = "mimetype" + + class RoomTypes: """Understood values of the room_type field of m.room.create events.""" @@ -273,6 +303,13 @@ class AccountDataTypes: IGNORED_USER_LIST: Final = "m.ignored_user_list" TAG: Final = "m.tag" PUSH_RULES: Final = "m.push_rules" + # MSC4155: Invite filtering + MSC4155_INVITE_PERMISSION_CONFIG: Final = ( + "org.matrix.msc4155.invite_permission_config" + ) + # Synapse-specific behaviour. See "Client-Server API Extensions" documentation + # in Admin API for more information. + SYNAPSE_ADMIN_CLIENT_CONFIG: Final = "io.element.synapse.admin_client_config" class HistoryVisibility: @@ -318,3 +355,8 @@ class ApprovalNoticeMedium: class Direction(enum.Enum): BACKWARDS = "b" FORWARDS = "f" + + +class ProfileFields: + DISPLAYNAME: Final = "displayname" + AVATAR_URL: Final = "avatar_url" diff --git a/synapse/api/errors.py b/synapse/api/errors.py index e6efa7a424..ec4d707b7b 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -70,6 +70,7 @@ class Codes(str, Enum): THREEPID_NOT_FOUND = "M_THREEPID_NOT_FOUND" THREEPID_DENIED = "M_THREEPID_DENIED" INVALID_USERNAME = "M_INVALID_USERNAME" + THREEPID_MEDIUM_NOT_SUPPORTED = "M_THREEPID_MEDIUM_NOT_SUPPORTED" SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED" CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN" CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" @@ -87,8 +88,7 @@ class Codes(str, Enum): WEAK_PASSWORD = "M_WEAK_PASSWORD" INVALID_SIGNATURE = "M_INVALID_SIGNATURE" USER_DEACTIVATED = "M_USER_DEACTIVATED" - # USER_LOCKED = "M_USER_LOCKED" - USER_LOCKED = "ORG_MATRIX_MSC3939_USER_LOCKED" + USER_LOCKED = "M_USER_LOCKED" NOT_YET_UPLOADED = "M_NOT_YET_UPLOADED" CANNOT_OVERWRITE_MEDIA = "M_CANNOT_OVERWRITE_MEDIA" @@ -101,8 +101,9 @@ class Codes(str, Enum): # The account has been suspended on the server. # By opposition to `USER_DEACTIVATED`, this is a reversible measure # that can possibly be appealed and reverted. - # Part of MSC3823. - USER_ACCOUNT_SUSPENDED = "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" + # Introduced by MSC3823 + # https://github.com/matrix-org/matrix-spec-proposals/pull/3823 + USER_ACCOUNT_SUSPENDED = "M_USER_SUSPENDED" BAD_ALIAS = "M_BAD_ALIAS" # For restricted join rules. @@ -132,6 +133,19 @@ class Codes(str, Enum): # connection. UNKNOWN_POS = "M_UNKNOWN_POS" + # Part of MSC4133 + PROFILE_TOO_LARGE = "M_PROFILE_TOO_LARGE" + KEY_TOO_LARGE = "M_KEY_TOO_LARGE" + + # Part of MSC4155 + INVITE_BLOCKED = "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED" + + # Part of MSC4306: Thread Subscriptions + MSC4306_CONFLICTING_UNSUBSCRIPTION = ( + "IO.ELEMENT.MSC4306.M_CONFLICTING_UNSUBSCRIPTION" + ) + MSC4306_NOT_IN_THREAD = "IO.ELEMENT.MSC4306.M_NOT_IN_THREAD" + class CodeMessageException(RuntimeError): """An exception with integer code, a message string attributes and optional headers. @@ -298,6 +312,20 @@ class UserDeactivatedError(SynapseError): ) +class UserLockedError(SynapseError): + """The error returned to the client when the user attempted to access an + authenticated endpoint, but the account has been locked. + """ + + def __init__(self) -> None: + super().__init__( + code=HTTPStatus.UNAUTHORIZED, + msg="User account has been locked", + errcode=Codes.USER_LOCKED, + additional_fields={"soft_logout": True}, + ) + + class FederationDeniedError(SynapseError): """An error raised when the server tries to federate with a server which is not on its federation whitelist. @@ -519,7 +547,11 @@ class InvalidCaptchaError(SynapseError): class LimitExceededError(SynapseError): - """A client has sent too many requests and is being throttled.""" + """A client has sent too many requests and is being throttled. + + Args: + pause: Optional time in seconds to pause before responding to the client. + """ def __init__( self, @@ -527,6 +559,7 @@ class LimitExceededError(SynapseError): code: int = 429, retry_after_ms: Optional[int] = None, errcode: str = Codes.LIMIT_EXCEEDED, + pause: Optional[float] = None, ): # Use HTTP header Retry-After to enable library-assisted retry handling. headers = ( @@ -537,6 +570,7 @@ class LimitExceededError(SynapseError): super().__init__(code, "Too Many Requests", errcode, headers=headers) self.retry_after_ms = retry_after_ms self.limiter_name = limiter_name + self.pause = pause def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict": return cs_error(self.msg, self.errcode, retry_after_ms=self.retry_after_ms) diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index b80630c5d3..509ef6b2c1 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -20,8 +20,7 @@ # # -from collections import OrderedDict -from typing import Hashable, Optional, Tuple +from typing import TYPE_CHECKING, Dict, Hashable, Optional, Tuple from synapse.api.errors import LimitExceededError from synapse.config.ratelimiting import RatelimitSettings @@ -29,6 +28,12 @@ from synapse.storage.databases.main import DataStore from synapse.types import Requester from synapse.util import Clock +if TYPE_CHECKING: + # To avoid circular imports: + from synapse.module_api.callbacks.ratelimit_callbacks import ( + RatelimitModuleApiCallbacks, + ) + class Ratelimiter: """ @@ -73,19 +78,23 @@ class Ratelimiter: store: DataStore, clock: Clock, cfg: RatelimitSettings, + ratelimit_callbacks: Optional["RatelimitModuleApiCallbacks"] = None, ): self.clock = clock self.rate_hz = cfg.per_second self.burst_count = cfg.burst_count self.store = store self._limiter_name = cfg.key + self._ratelimit_callbacks = ratelimit_callbacks - # An ordered dictionary representing the token buckets tracked by this rate + # A dictionary representing the token buckets tracked by this rate # limiter. Each entry maps a key of arbitrary type to a tuple representing: # * The number of tokens currently in the bucket, # * The time point when the bucket was last completely empty, and # * The rate_hz (leak rate) of this particular bucket. - self.actions: OrderedDict[Hashable, Tuple[float, float, float]] = OrderedDict() + self.actions: Dict[Hashable, Tuple[float, float, float]] = {} + + self.clock.looping_call(self._prune_message_counts, 60 * 1000) def _get_key( self, requester: Optional[Requester], key: Optional[Hashable] @@ -164,14 +173,25 @@ class Ratelimiter: if override and not override.messages_per_second: return True, -1.0 + if requester and self._ratelimit_callbacks: + # Check if the user has a custom rate limit for this specific limiter + # as returned by the module API. + module_override = ( + await self._ratelimit_callbacks.get_ratelimit_override_for_user( + requester.user.to_string(), + self._limiter_name, + ) + ) + + if module_override: + rate_hz = module_override.per_second + burst_count = module_override.burst_count + # Override default values if set time_now_s = _time_now_s if _time_now_s is not None else self.clock.time() rate_hz = rate_hz if rate_hz is not None else self.rate_hz burst_count = burst_count if burst_count is not None else self.burst_count - # Remove any expired entries - self._prune_message_counts(time_now_s) - # Check if there is an existing count entry for this key action_count, time_start, _ = self._get_action_counts(key, time_now_s) @@ -246,13 +266,12 @@ class Ratelimiter: action_count, time_start, rate_hz = self._get_action_counts(key, time_now_s) self.actions[key] = (action_count + n_actions, time_start, rate_hz) - def _prune_message_counts(self, time_now_s: float) -> None: + def _prune_message_counts(self) -> None: """Remove message count entries that have not exceeded their defined rate_hz limit - - Args: - time_now_s: The current time """ + time_now_s = self.clock.time() + # We create a copy of the key list here as the dictionary is modified during # the loop for key in list(self.actions.keys()): @@ -275,6 +294,7 @@ class Ratelimiter: update: bool = True, n_actions: int = 1, _time_now_s: Optional[float] = None, + pause: Optional[float] = 0.5, ) -> None: """Checks if an action can be performed. If not, raises a LimitExceededError @@ -298,6 +318,8 @@ class Ratelimiter: at all. _time_now_s: The current time. Optional, defaults to the current time according to self.clock. Only used by tests. + pause: Time in seconds to pause when an action is being limited. Defaults to 0.5 + to stop clients from "tight-looping" on retrying their request. Raises: LimitExceededError: If an action could not be performed, along with the time in @@ -316,13 +338,10 @@ class Ratelimiter: ) if not allowed: - # We pause for a bit here to stop clients from "tight-looping" on - # retrying their request. - await self.clock.sleep(0.5) - raise LimitExceededError( limiter_name=self._limiter_name, retry_after_ms=int(1000 * (time_allowed - time_now_s)), + pause=pause, ) diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index 4bde385f78..71ef5952c3 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -36,12 +36,14 @@ class EventFormatVersions: ROOM_V1_V2 = 1 # $id:server event id format: used for room v1 and v2 ROOM_V3 = 2 # MSC1659-style $hash event id format: used for room v3 ROOM_V4_PLUS = 3 # MSC1884-style $hash format: introduced for room v4 + ROOM_V11_HYDRA_PLUS = 4 # MSC4291 room IDs as hashes: introduced for room HydraV11 KNOWN_EVENT_FORMAT_VERSIONS = { EventFormatVersions.ROOM_V1_V2, EventFormatVersions.ROOM_V3, EventFormatVersions.ROOM_V4_PLUS, + EventFormatVersions.ROOM_V11_HYDRA_PLUS, } @@ -50,6 +52,7 @@ class StateResolutionVersions: V1 = 1 # room v1 state res V2 = 2 # MSC1442 state res: room v2 and later + V2_1 = 3 # MSC4297 state res class RoomDisposition: @@ -109,6 +112,10 @@ class RoomVersion: msc3931_push_features: Tuple[str, ...] # values from PushRuleRoomFlag # MSC3757: Restricting who can overwrite a state event msc3757_enabled: bool + # MSC4289: Creator power enabled + msc4289_creator_power_enabled: bool + # MSC4291: Room IDs as hashes of the create event + msc4291_room_ids_as_hashes: bool class RoomVersions: @@ -131,6 +138,8 @@ class RoomVersions: enforce_int_power_levels=False, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V2 = RoomVersion( "2", @@ -151,6 +160,8 @@ class RoomVersions: enforce_int_power_levels=False, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V3 = RoomVersion( "3", @@ -171,6 +182,8 @@ class RoomVersions: enforce_int_power_levels=False, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V4 = RoomVersion( "4", @@ -191,6 +204,8 @@ class RoomVersions: enforce_int_power_levels=False, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V5 = RoomVersion( "5", @@ -211,6 +226,8 @@ class RoomVersions: enforce_int_power_levels=False, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V6 = RoomVersion( "6", @@ -231,6 +248,8 @@ class RoomVersions: enforce_int_power_levels=False, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V7 = RoomVersion( "7", @@ -251,6 +270,8 @@ class RoomVersions: enforce_int_power_levels=False, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V8 = RoomVersion( "8", @@ -271,6 +292,8 @@ class RoomVersions: enforce_int_power_levels=False, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V9 = RoomVersion( "9", @@ -291,6 +314,8 @@ class RoomVersions: enforce_int_power_levels=False, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V10 = RoomVersion( "10", @@ -311,6 +336,8 @@ class RoomVersions: enforce_int_power_levels=True, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) MSC1767v10 = RoomVersion( # MSC1767 (Extensible Events) based on room version "10" @@ -332,6 +359,8 @@ class RoomVersions: enforce_int_power_levels=True, msc3931_push_features=(PushRuleRoomFlag.EXTENSIBLE_EVENTS,), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) MSC3757v10 = RoomVersion( # MSC3757 (Restricting who can overwrite a state event) based on room version "10" @@ -353,6 +382,8 @@ class RoomVersions: enforce_int_power_levels=True, msc3931_push_features=(), msc3757_enabled=True, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) V11 = RoomVersion( "11", @@ -373,6 +404,8 @@ class RoomVersions: enforce_int_power_levels=True, msc3931_push_features=(), msc3757_enabled=False, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, ) MSC3757v11 = RoomVersion( # MSC3757 (Restricting who can overwrite a state event) based on room version "11" @@ -394,6 +427,52 @@ class RoomVersions: enforce_int_power_levels=True, msc3931_push_features=(), msc3757_enabled=True, + msc4289_creator_power_enabled=False, + msc4291_room_ids_as_hashes=False, + ) + HydraV11 = RoomVersion( + "org.matrix.hydra.11", + RoomDisposition.UNSTABLE, + EventFormatVersions.ROOM_V11_HYDRA_PLUS, + StateResolutionVersions.V2_1, # Changed from v11 + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + implicit_room_creator=True, # Used by MSC3820 + updated_redaction_rules=True, # Used by MSC3820 + restricted_join_rule=True, + restricted_join_rule_fix=True, + knock_join_rule=True, + msc3389_relation_redactions=False, + knock_restricted_join_rule=True, + enforce_int_power_levels=True, + msc3931_push_features=(), + msc3757_enabled=False, + msc4289_creator_power_enabled=True, # Changed from v11 + msc4291_room_ids_as_hashes=True, # Changed from v11 + ) + V12 = RoomVersion( + "12", + RoomDisposition.STABLE, + EventFormatVersions.ROOM_V11_HYDRA_PLUS, + StateResolutionVersions.V2_1, # Changed from v11 + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + implicit_room_creator=True, # Used by MSC3820 + updated_redaction_rules=True, # Used by MSC3820 + restricted_join_rule=True, + restricted_join_rule_fix=True, + knock_join_rule=True, + msc3389_relation_redactions=False, + knock_restricted_join_rule=True, + enforce_int_power_levels=True, + msc3931_push_features=(), + msc3757_enabled=False, + msc4289_creator_power_enabled=True, # Changed from v11 + msc4291_room_ids_as_hashes=True, # Changed from v11 ) @@ -411,8 +490,10 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = { RoomVersions.V9, RoomVersions.V10, RoomVersions.V11, + RoomVersions.V12, RoomVersions.MSC3757v10, RoomVersions.MSC3757v11, + RoomVersions.HydraV11, ) } diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 03a3e96f28..baa6e2d390 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -22,8 +22,10 @@ """Contains the URL paths to prefix various aspects of the server with.""" import hmac +import urllib.parse from hashlib import sha256 -from urllib.parse import urlencode +from typing import Optional +from urllib.parse import urlencode, urljoin from synapse.config import ConfigError from synapse.config.homeserver import HomeServerConfig @@ -66,3 +68,52 @@ class ConsentURIBuilder: urlencode({"u": user_id, "h": mac}), ) return consent_uri + + +class LoginSSORedirectURIBuilder: + def __init__(self, hs_config: HomeServerConfig): + self._public_baseurl = hs_config.server.public_baseurl + + def build_login_sso_redirect_uri( + self, *, idp_id: Optional[str], client_redirect_url: str + ) -> str: + """Build a `/login/sso/redirect` URI for the given identity provider. + + Builds `/_matrix/client/v3/login/sso/redirect/{idpId}?redirectUrl=xxx` when `idp_id` is specified. + Otherwise, builds `/_matrix/client/v3/login/sso/redirect?redirectUrl=xxx` when `idp_id` is `None`. + + Args: + idp_id: Optional ID of the identity provider + client_redirect_url: URL to redirect the user to after login + + Returns + The URI to follow when choosing a specific identity provider. + """ + base_url = urljoin( + self._public_baseurl, + f"{CLIENT_API_PREFIX}/v3/login/sso/redirect", + ) + + serialized_query_parameters = urlencode({"redirectUrl": client_redirect_url}) + + if idp_id: + # Since this is a user-controlled string, make it safe to include in a URL path. + url_encoded_idp_id = urllib.parse.quote( + idp_id, + # Since this defaults to `safe="/"`, we have to override it. We're + # working with an individual URL path parameter so there shouldn't be + # any slashes in it which could change the request path. + safe="", + encoding="utf8", + ) + + resultant_url = urljoin( + # We have to add a trailing slash to the base URL to ensure that the + # last path segment is not stripped away when joining with another path. + f"{base_url}/", + f"{url_encoded_idp_id}?{serialized_query_parameters}", + ) + else: + resultant_url = f"{base_url}?{serialized_query_parameters}" + + return resultant_url diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 4cc260d551..cf3d260e65 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -72,10 +72,10 @@ from synapse.events.auto_accept_invites import InviteAutoAccepter from synapse.events.presence_router import load_legacy_presence_router from synapse.handlers.auth import load_legacy_password_auth_providers from synapse.http.site import SynapseSite -from synapse.logging.context import PreserveLoggingContext +from synapse.logging.context import LoggingContext, PreserveLoggingContext from synapse.logging.opentracing import init_tracer from synapse.metrics import install_gc_manager, register_threadpool -from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.metrics.jemalloc import setup_jemalloc_stats from synapse.module_api.callbacks.spamchecker_callbacks import load_legacy_spam_checkers from synapse.module_api.callbacks.third_party_event_rules_callbacks import ( @@ -183,25 +183,23 @@ def start_reactor( if gc_thresholds: gc.set_threshold(*gc_thresholds) install_gc_manager() - run_command() - # make sure that we run the reactor with the sentinel log context, - # otherwise other PreserveLoggingContext instances will get confused - # and complain when they see the logcontext arbitrarily swapping - # between the sentinel and `run` logcontexts. - # - # We also need to drop the logcontext before forking if we're daemonizing, - # otherwise the cputime metrics get confused about the per-thread resource usage - # appearing to go backwards. - with PreserveLoggingContext(): - if daemonize: - assert pid_file is not None + # Reset the logging context when we start the reactor (whenever we yield control + # to the reactor, the `sentinel` logging context needs to be set so we don't + # leak the current logging context and erroneously apply it to the next task the + # reactor event loop picks up) + with PreserveLoggingContext(): + run_command() - if print_pidfile: - print(pid_file) + if daemonize: + assert pid_file is not None - daemonize_process(pid_file, logger) - run() + if print_pidfile: + print(pid_file) + + daemonize_process(pid_file, logger) + + run() def quit_with_error(error_string: str) -> NoReturn: @@ -286,6 +284,16 @@ def register_start( def listen_metrics(bind_addresses: StrCollection, port: int) -> None: """ Start Prometheus metrics server. + + This method runs the metrics server on a different port, in a different thread to + Synapse. This can make it more resilient to heavy load in Synapse causing metric + requests to be slow or timeout. + + Even though `start_http_server_prometheus(...)` uses `threading.Thread` behind the + scenes (where all threads share the GIL and only one thread can execute Python + bytecode at a time), this still works because the metrics thread can preempt the + Twisted reactor thread between bytecode boundaries and the metrics thread gets + scheduled with roughly equal priority to the Twisted reactor thread. """ from prometheus_client import start_http_server as start_http_server_prometheus @@ -293,32 +301,9 @@ def listen_metrics(bind_addresses: StrCollection, port: int) -> None: for host in bind_addresses: logger.info("Starting metrics listener on %s:%d", host, port) - _set_prometheus_client_use_created_metrics(False) start_http_server_prometheus(port, addr=host, registry=RegistryProxy) -def _set_prometheus_client_use_created_metrics(new_value: bool) -> None: - """ - Sets whether prometheus_client should expose `_created`-suffixed metrics for - all gauges, histograms and summaries. - There is no programmatic way to disable this without poking at internals; - the proper way is to use an environment variable which prometheus_client - loads at import time. - - The motivation for disabling these `_created` metrics is that they're - a waste of space as they're not useful but they take up space in Prometheus. - """ - - import prometheus_client.metrics - - if hasattr(prometheus_client.metrics, "_use_created"): - prometheus_client.metrics._use_created = new_value - else: - logger.error( - "Can't disable `_created` metrics in prometheus_client (brittle hack broken?)" - ) - - def listen_manhole( bind_addresses: StrCollection, port: int, @@ -445,8 +430,8 @@ def listen_http( # getHost() returns a UNIXAddress which contains an instance variable of 'name' # encoded as a byte string. Decode as utf-8 so pretty. logger.info( - "Synapse now listening on Unix Socket at: " - f"{ports[0].getHost().name.decode('utf-8')}" + "Synapse now listening on Unix Socket at: %s", + ports[0].getHost().name.decode("utf-8"), ) return ports @@ -525,6 +510,7 @@ async def start(hs: "HomeServer") -> None: Args: hs: homeserver instance """ + server_name = hs.hostname reactor = hs.get_reactor() # We want to use a separate thread pool for the resolver so that large @@ -537,22 +523,34 @@ async def start(hs: "HomeServer") -> None: ) # Register the threadpools with our metrics. - register_threadpool("default", reactor.getThreadPool()) - register_threadpool("gai_resolver", resolver_threadpool) + register_threadpool( + name="default", server_name=server_name, threadpool=reactor.getThreadPool() + ) + register_threadpool( + name="gai_resolver", server_name=server_name, threadpool=resolver_threadpool + ) # Set up the SIGHUP machinery. if hasattr(signal, "SIGHUP"): - @wrap_as_background_process("sighup") - async def handle_sighup(*args: Any, **kwargs: Any) -> None: - # Tell systemd our state, if we're using it. This will silently fail if - # we're not using systemd. - sdnotify(b"RELOADING=1") + def handle_sighup(*args: Any, **kwargs: Any) -> "defer.Deferred[None]": + async def _handle_sighup(*args: Any, **kwargs: Any) -> None: + # Tell systemd our state, if we're using it. This will silently fail if + # we're not using systemd. + sdnotify(b"RELOADING=1") - for i, args, kwargs in _sighup_callbacks: - i(*args, **kwargs) + for i, args, kwargs in _sighup_callbacks: + i(*args, **kwargs) - sdnotify(b"READY=1") + sdnotify(b"READY=1") + + return run_as_background_process( + "sighup", + server_name, + _handle_sighup, + *args, + **kwargs, + ) # We defer running the sighup handlers until next reactor tick. This # is so that we're in a sane state, e.g. flushing the logs may fail @@ -601,18 +599,38 @@ async def start(hs: "HomeServer") -> None: hs.get_datastores().main.db_pool.start_profiling() hs.get_pusherpool().start() + def log_shutdown() -> None: + with LoggingContext("log_shutdown"): + logger.info("Shutting down...") + # Log when we start the shut down process. - hs.get_reactor().addSystemEventTrigger( - "before", "shutdown", logger.info, "Shutting down..." - ) + hs.get_reactor().addSystemEventTrigger("before", "shutdown", log_shutdown) setup_sentry(hs) setup_sdnotify(hs) - # If background tasks are running on the main process or this is the worker in - # charge of them, start collecting the phone home stats and shared usage metrics. + # Register background tasks required by this server. This must be done + # somewhat manually due to the background tasks not being registered + # unless handlers are instantiated. + # + # While we could "start" these before the reactor runs, nothing will happen until + # the reactor is running, so we may as well do it here in `start`. + # + # Additionally, this means we also start them after we daemonize and fork the + # process which means we can avoid any potential problems with cputime metrics + # getting confused about the per-thread resource usage appearing to go backwards + # because we're comparing the resource usage (`rusage`) from the original process to + # the forked process. if hs.config.worker.run_background_tasks: + hs.start_background_tasks() + + # TODO: This should be moved to same pattern we use for other background tasks: + # Add to `REQUIRED_ON_BACKGROUND_TASK_STARTUP` and rely on + # `start_background_tasks` to start it. await hs.get_common_usage_metrics_manager().setup() + + # TODO: This feels like another pattern that should refactored as one of the + # `REQUIRED_ON_BACKGROUND_TASK_STARTUP` start_phone_stats_home(hs) # We now freeze all allocated objects in the hopes that (almost) diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index a528c3890d..543b26d8ba 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -51,8 +51,7 @@ from synapse.http.server import JsonResource, OptionsResource from synapse.logging.context import LoggingContext from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource -from synapse.rest import ClientRestResource -from synapse.rest.admin import register_servlets_for_media_repo +from synapse.rest import ClientRestResource, admin from synapse.rest.health import HealthResource from synapse.rest.key.v2 import KeyResource from synapse.rest.synapse.client import build_synapse_client_resource_tree @@ -105,6 +104,9 @@ from synapse.storage.databases.main.stats import StatsStore from synapse.storage.databases.main.stream import StreamWorkerStore from synapse.storage.databases.main.tags import TagsWorkerStore from synapse.storage.databases.main.task_scheduler import TaskSchedulerWorkerStore +from synapse.storage.databases.main.thread_subscriptions import ( + ThreadSubscriptionsWorkerStore, +) from synapse.storage.databases.main.transactions import TransactionWorkerStore from synapse.storage.databases.main.ui_auth import UIAuthWorkerStore from synapse.storage.databases.main.user_directory import UserDirectoryStore @@ -119,7 +121,6 @@ class GenericWorkerStore( # FIXME(https://github.com/matrix-org/synapse/issues/3714): We need to add # UserDirectoryStore as we write directly rather than going via the correct worker. UserDirectoryStore, - StatsStore, UIAuthWorkerStore, EndToEndRoomKeyStore, PresenceStore, @@ -134,6 +135,7 @@ class GenericWorkerStore( KeyStore, RoomWorkerStore, DirectoryWorkerStore, + ThreadSubscriptionsWorkerStore, PushRulesWorkerStore, ApplicationServiceTransactionWorkerStore, ApplicationServiceWorkerStore, @@ -155,6 +157,7 @@ class GenericWorkerStore( StreamWorkerStore, EventsWorkerStore, RegistrationWorkerStore, + StatsStore, SearchStore, TransactionWorkerStore, LockStore, @@ -176,8 +179,13 @@ class GenericWorkerServer(HomeServer): def _listen_http(self, listener_config: ListenerConfig) -> None: assert listener_config.http_options is not None - # We always include a health resource. - resources: Dict[str, Resource] = {"/health": HealthResource()} + # We always include an admin resource that we populate with servlets as needed + admin_resource = JsonResource(self, canonical_json=False) + resources: Dict[str, Resource] = { + # We always include a health resource. + "/health": HealthResource(), + "/_synapse/admin": admin_resource, + } for res in listener_config.http_options.resources: for name in res.names: @@ -190,6 +198,7 @@ class GenericWorkerServer(HomeServer): resources.update(build_synapse_client_resource_tree(self)) resources["/.well-known"] = well_known_resource(self) + admin.register_servlets(self, admin_resource) elif name == "federation": resources[FEDERATION_PREFIX] = TransportLayerServer(self) @@ -199,15 +208,13 @@ class GenericWorkerServer(HomeServer): # We need to serve the admin servlets for media on the # worker. - admin_resource = JsonResource(self, canonical_json=False) - register_servlets_for_media_repo(self, admin_resource) + admin.register_servlets_for_media_repo(self, admin_resource) resources.update( { MEDIA_R0_PREFIX: media_repo, MEDIA_V3_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo, - "/_synapse/admin": admin_resource, } ) @@ -284,8 +291,7 @@ class GenericWorkerServer(HomeServer): elif listener.type == "metrics": if not self.config.metrics.enable_metrics: logger.warning( - "Metrics listener configured, but " - "enable_metrics is not True!" + "Metrics listener configured, but enable_metrics is not True!" ) else: if isinstance(listener, TCPListenerConfig): @@ -349,7 +355,12 @@ def start(config_options: List[str]) -> None: except Exception as e: handle_startup_exception(e) - register_start(_base.start, hs) + async def start() -> None: + # Re-establish log context now that we're back from the reactor + with LoggingContext("start"): + await _base.start(hs) + + register_start(start) # redirect stdio to the logs, if configured. if not hs.config.logging.no_redirect_stdio: diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2a824e8457..dfc4a00719 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -54,6 +54,7 @@ from synapse.config.server import ListenerConfig, TCPListenerConfig from synapse.federation.transport.server import TransportLayerServer from synapse.http.additional_resource import AdditionalResource from synapse.http.server import ( + JsonResource, OptionsResource, RootOptionsRedirectResource, StaticResource, @@ -61,8 +62,7 @@ from synapse.http.server import ( from synapse.logging.context import LoggingContext from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource -from synapse.rest import ClientRestResource -from synapse.rest.admin import AdminRestResource +from synapse.rest import ClientRestResource, admin from synapse.rest.health import HealthResource from synapse.rest.key.v2 import KeyResource from synapse.rest.synapse.client import build_synapse_client_resource_tree @@ -180,11 +180,14 @@ class SynapseHomeServer(HomeServer): if compress: client_resource = gz_wrap(client_resource) + admin_resource = JsonResource(self, canonical_json=False) + admin.register_servlets(self, admin_resource) + resources.update( { CLIENT_API_PREFIX: client_resource, "/.well-known": well_known_resource(self), - "/_synapse/admin": AdminRestResource(self), + "/_synapse/admin": admin_resource, **build_synapse_client_resource_tree(self), } ) @@ -286,8 +289,7 @@ class SynapseHomeServer(HomeServer): elif listener.type == "metrics": if not self.config.metrics.enable_metrics: logger.warning( - "Metrics listener configured, but " - "enable_metrics is not True!" + "Metrics listener configured, but enable_metrics is not True!" ) else: if isinstance(listener, TCPListenerConfig): @@ -375,15 +377,17 @@ def setup(config_options: List[str]) -> SynapseHomeServer: handle_startup_exception(e) async def start() -> None: - # Load the OIDC provider metadatas, if OIDC is enabled. - if hs.config.oidc.oidc_enabled: - oidc = hs.get_oidc_handler() - # Loading the provider metadata also ensures the provider config is valid. - await oidc.load_metadata() + # Re-establish log context now that we're back from the reactor + with LoggingContext("start"): + # Load the OIDC provider metadatas, if OIDC is enabled. + if hs.config.oidc.oidc_enabled: + oidc = hs.get_oidc_handler() + # Loading the provider metadata also ensures the provider config is valid. + await oidc.load_metadata() - await _base.start(hs) + await _base.start(hs) - hs.get_datastores().main.db_pool.updates.start_doing_background_updates() + hs.get_datastores().main.db_pool.updates.start_doing_background_updates() register_start(start) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index f602bbbeea..69d3ac78fd 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -26,151 +26,190 @@ from typing import TYPE_CHECKING, List, Mapping, Sized, Tuple from prometheus_client import Gauge -from synapse.metrics.background_process_metrics import wrap_as_background_process +from twisted.internet import defer + +from synapse.metrics import SERVER_NAME_LABEL +from synapse.metrics.background_process_metrics import ( + run_as_background_process, +) from synapse.types import JsonDict +from synapse.util.constants import ONE_HOUR_SECONDS, ONE_MINUTE_SECONDS if TYPE_CHECKING: from synapse.server import HomeServer logger = logging.getLogger("synapse.app.homeserver") +MILLISECONDS_PER_SECOND = 1000 + +INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS = 5 * ONE_MINUTE_SECONDS +""" +We wait 5 minutes to send the first set of stats as the server can be quite busy the +first few minutes +""" + +PHONE_HOME_INTERVAL_SECONDS = 3 * ONE_HOUR_SECONDS +""" +Phone home stats are sent every 3 hours +""" + # Contains the list of processes we will be monitoring # currently either 0 or 1 _stats_process: List[Tuple[int, "resource.struct_rusage"]] = [] # Gauges to expose monthly active user control metrics -current_mau_gauge = Gauge("synapse_admin_mau_current", "Current MAU") +current_mau_gauge = Gauge( + "synapse_admin_mau_current", + "Current MAU", + labelnames=[SERVER_NAME_LABEL], +) current_mau_by_service_gauge = Gauge( "synapse_admin_mau_current_mau_by_service", "Current MAU by service", - ["app_service"], + labelnames=["app_service", SERVER_NAME_LABEL], +) +max_mau_gauge = Gauge( + "synapse_admin_mau_max", + "MAU Limit", + labelnames=[SERVER_NAME_LABEL], ) -max_mau_gauge = Gauge("synapse_admin_mau_max", "MAU Limit") registered_reserved_users_mau_gauge = Gauge( "synapse_admin_mau_registered_reserved_users", "Registered users with reserved threepids", + labelnames=[SERVER_NAME_LABEL], ) -@wrap_as_background_process("phone_stats_home") -async def phone_stats_home( +def phone_stats_home( hs: "HomeServer", stats: JsonDict, stats_process: List[Tuple[int, "resource.struct_rusage"]] = _stats_process, -) -> None: - """Collect usage statistics and send them to the configured endpoint. +) -> "defer.Deferred[None]": + server_name = hs.hostname - Args: - hs: the HomeServer object to use for gathering usage data. - stats: the dict in which to store the statistics sent to the configured - endpoint. Mostly used in tests to figure out the data that is supposed to - be sent. - stats_process: statistics about resource usage of the process. - """ + async def _phone_stats_home( + hs: "HomeServer", + stats: JsonDict, + stats_process: List[Tuple[int, "resource.struct_rusage"]] = _stats_process, + ) -> None: + """Collect usage statistics and send them to the configured endpoint. - logger.info("Gathering stats for reporting") - now = int(hs.get_clock().time()) - # Ensure the homeserver has started. - assert hs.start_time is not None - uptime = int(now - hs.start_time) - if uptime < 0: - uptime = 0 + Args: + hs: the HomeServer object to use for gathering usage data. + stats: the dict in which to store the statistics sent to the configured + endpoint. Mostly used in tests to figure out the data that is supposed to + be sent. + stats_process: statistics about resource usage of the process. + """ - # - # Performance statistics. Keep this early in the function to maintain reliability of `test_performance_100` test. - # - old = stats_process[0] - new = (now, resource.getrusage(resource.RUSAGE_SELF)) - stats_process[0] = new + logger.info("Gathering stats for reporting") + now = int(hs.get_clock().time()) + # Ensure the homeserver has started. + assert hs.start_time is not None + uptime = int(now - hs.start_time) + if uptime < 0: + uptime = 0 - # Get RSS in bytes - stats["memory_rss"] = new[1].ru_maxrss + # + # Performance statistics. Keep this early in the function to maintain reliability of `test_performance_100` test. + # + old = stats_process[0] + new = (now, resource.getrusage(resource.RUSAGE_SELF)) + stats_process[0] = new - # Get CPU time in % of a single core, not % of all cores - used_cpu_time = (new[1].ru_utime + new[1].ru_stime) - ( - old[1].ru_utime + old[1].ru_stime - ) - if used_cpu_time == 0 or new[0] == old[0]: - stats["cpu_average"] = 0 - else: - stats["cpu_average"] = math.floor(used_cpu_time / (new[0] - old[0]) * 100) + # Get RSS in bytes + stats["memory_rss"] = new[1].ru_maxrss - # - # General statistics - # - - store = hs.get_datastores().main - common_metrics = await hs.get_common_usage_metrics_manager().get_metrics() - - stats["homeserver"] = hs.config.server.server_name - stats["server_context"] = hs.config.server.server_context - stats["timestamp"] = now - stats["uptime_seconds"] = uptime - version = sys.version_info - stats["python_version"] = "{}.{}.{}".format( - version.major, version.minor, version.micro - ) - stats["total_users"] = await store.count_all_users() - - total_nonbridged_users = await store.count_nonbridged_users() - stats["total_nonbridged_users"] = total_nonbridged_users - - daily_user_type_results = await store.count_daily_user_type() - for name, count in daily_user_type_results.items(): - stats["daily_user_type_" + name] = count - - room_count = await store.get_room_count() - stats["total_room_count"] = room_count - - stats["daily_active_users"] = common_metrics.daily_active_users - stats["monthly_active_users"] = await store.count_monthly_users() - daily_active_e2ee_rooms = await store.count_daily_active_e2ee_rooms() - stats["daily_active_e2ee_rooms"] = daily_active_e2ee_rooms - stats["daily_e2ee_messages"] = await store.count_daily_e2ee_messages() - daily_sent_e2ee_messages = await store.count_daily_sent_e2ee_messages() - stats["daily_sent_e2ee_messages"] = daily_sent_e2ee_messages - stats["daily_active_rooms"] = await store.count_daily_active_rooms() - stats["daily_messages"] = await store.count_daily_messages() - daily_sent_messages = await store.count_daily_sent_messages() - stats["daily_sent_messages"] = daily_sent_messages - - r30v2_results = await store.count_r30v2_users() - for name, count in r30v2_results.items(): - stats["r30v2_users_" + name] = count - - stats["cache_factor"] = hs.config.caches.global_factor - stats["event_cache_size"] = hs.config.caches.event_cache_size - - # - # Database version - # - - # This only reports info about the *main* database. - stats["database_engine"] = store.db_pool.engine.module.__name__ - stats["database_server_version"] = store.db_pool.engine.server_version - - # - # Logging configuration - # - synapse_logger = logging.getLogger("synapse") - log_level = synapse_logger.getEffectiveLevel() - stats["log_level"] = logging.getLevelName(log_level) - - logger.info( - "Reporting stats to %s: %s" % (hs.config.metrics.report_stats_endpoint, stats) - ) - try: - await hs.get_proxied_http_client().put_json( - hs.config.metrics.report_stats_endpoint, stats + # Get CPU time in % of a single core, not % of all cores + used_cpu_time = (new[1].ru_utime + new[1].ru_stime) - ( + old[1].ru_utime + old[1].ru_stime ) - except Exception as e: - logger.warning("Error reporting stats: %s", e) + if used_cpu_time == 0 or new[0] == old[0]: + stats["cpu_average"] = 0 + else: + stats["cpu_average"] = math.floor(used_cpu_time / (new[0] - old[0]) * 100) + + # + # General statistics + # + + store = hs.get_datastores().main + common_metrics = await hs.get_common_usage_metrics_manager().get_metrics() + + stats["homeserver"] = hs.config.server.server_name + stats["server_context"] = hs.config.server.server_context + stats["timestamp"] = now + stats["uptime_seconds"] = uptime + version = sys.version_info + stats["python_version"] = "{}.{}.{}".format( + version.major, version.minor, version.micro + ) + stats["total_users"] = await store.count_all_users() + + total_nonbridged_users = await store.count_nonbridged_users() + stats["total_nonbridged_users"] = total_nonbridged_users + + daily_user_type_results = await store.count_daily_user_type() + for name, count in daily_user_type_results.items(): + stats["daily_user_type_" + name] = count + + room_count = await store.get_room_count() + stats["total_room_count"] = room_count + + stats["daily_active_users"] = common_metrics.daily_active_users + stats["monthly_active_users"] = await store.count_monthly_users() + daily_active_e2ee_rooms = await store.count_daily_active_e2ee_rooms() + stats["daily_active_e2ee_rooms"] = daily_active_e2ee_rooms + stats["daily_e2ee_messages"] = await store.count_daily_e2ee_messages() + daily_sent_e2ee_messages = await store.count_daily_sent_e2ee_messages() + stats["daily_sent_e2ee_messages"] = daily_sent_e2ee_messages + stats["daily_active_rooms"] = await store.count_daily_active_rooms() + stats["daily_messages"] = await store.count_daily_messages() + daily_sent_messages = await store.count_daily_sent_messages() + stats["daily_sent_messages"] = daily_sent_messages + + r30v2_results = await store.count_r30v2_users() + for name, count in r30v2_results.items(): + stats["r30v2_users_" + name] = count + + stats["cache_factor"] = hs.config.caches.global_factor + stats["event_cache_size"] = hs.config.caches.event_cache_size + + # + # Database version + # + + # This only reports info about the *main* database. + stats["database_engine"] = store.db_pool.engine.module.__name__ + stats["database_server_version"] = store.db_pool.engine.server_version + + # + # Logging configuration + # + synapse_logger = logging.getLogger("synapse") + log_level = synapse_logger.getEffectiveLevel() + stats["log_level"] = logging.getLevelName(log_level) + + logger.info( + "Reporting stats to %s: %s", hs.config.metrics.report_stats_endpoint, stats + ) + try: + await hs.get_proxied_http_client().put_json( + hs.config.metrics.report_stats_endpoint, stats + ) + except Exception as e: + logger.warning("Error reporting stats: %s", e) + + return run_as_background_process( + "phone_stats_home", server_name, _phone_stats_home, hs, stats, stats_process + ) def start_phone_stats_home(hs: "HomeServer") -> None: """ Start the background tasks which report phone home stats. """ + server_name = hs.hostname clock = hs.get_clock() stats: JsonDict = {} @@ -185,34 +224,50 @@ def start_phone_stats_home(hs: "HomeServer") -> None: # If you increase the loop period, the accuracy of user_daily_visits # table will decrease clock.looping_call( - hs.get_datastores().main.generate_user_daily_visits, 5 * 60 * 1000 + hs.get_datastores().main.generate_user_daily_visits, + 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND, ) # monthly active user limiting functionality clock.looping_call( - hs.get_datastores().main.reap_monthly_active_users, 1000 * 60 * 60 + hs.get_datastores().main.reap_monthly_active_users, + ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, ) hs.get_datastores().main.reap_monthly_active_users() - @wrap_as_background_process("generate_monthly_active_users") - async def generate_monthly_active_users() -> None: - current_mau_count = 0 - current_mau_count_by_service: Mapping[str, int] = {} - reserved_users: Sized = () - store = hs.get_datastores().main - if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only: - current_mau_count = await store.get_monthly_active_count() - current_mau_count_by_service = ( - await store.get_monthly_active_count_by_service() + def generate_monthly_active_users() -> "defer.Deferred[None]": + async def _generate_monthly_active_users() -> None: + current_mau_count = 0 + current_mau_count_by_service: Mapping[str, int] = {} + reserved_users: Sized = () + store = hs.get_datastores().main + if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only: + current_mau_count = await store.get_monthly_active_count() + current_mau_count_by_service = ( + await store.get_monthly_active_count_by_service() + ) + reserved_users = await store.get_registered_reserved_users() + current_mau_gauge.labels(**{SERVER_NAME_LABEL: server_name}).set( + float(current_mau_count) ) - reserved_users = await store.get_registered_reserved_users() - current_mau_gauge.set(float(current_mau_count)) - for app_service, count in current_mau_count_by_service.items(): - current_mau_by_service_gauge.labels(app_service).set(float(count)) + for app_service, count in current_mau_count_by_service.items(): + current_mau_by_service_gauge.labels( + app_service=app_service, **{SERVER_NAME_LABEL: server_name} + ).set(float(count)) - registered_reserved_users_mau_gauge.set(float(len(reserved_users))) - max_mau_gauge.set(float(hs.config.server.max_mau_value)) + registered_reserved_users_mau_gauge.labels( + **{SERVER_NAME_LABEL: server_name} + ).set(float(len(reserved_users))) + max_mau_gauge.labels(**{SERVER_NAME_LABEL: server_name}).set( + float(hs.config.server.max_mau_value) + ) + + return run_as_background_process( + "generate_monthly_active_users", + server_name, + _generate_monthly_active_users, + ) if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only: generate_monthly_active_users() @@ -221,7 +276,12 @@ def start_phone_stats_home(hs: "HomeServer") -> None: if hs.config.metrics.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") - clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000, hs, stats) + clock.looping_call( + phone_stats_home, + PHONE_HOME_INTERVAL_SECONDS * MILLISECONDS_PER_SECOND, + hs, + stats, + ) # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process @@ -229,4 +289,6 @@ def start_phone_stats_home(hs: "HomeServer") -> None: # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes - clock.call_later(5 * 60, phone_stats_home, hs, stats) + clock.call_later( + INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, phone_stats_home, hs, stats + ) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index a96cdbf1e7..2d8d382e68 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -78,7 +78,7 @@ class ApplicationService: self, token: str, id: str, - sender: str, + sender: UserID, url: Optional[str] = None, namespaces: Optional[JsonDict] = None, hs_token: Optional[str] = None, @@ -87,6 +87,7 @@ class ApplicationService: ip_range_whitelist: Optional[IPSet] = None, supports_ephemeral: bool = False, msc3202_transaction_extensions: bool = False, + msc4190_device_management: bool = False, ): self.token = token self.url = ( @@ -95,11 +96,14 @@ class ApplicationService: self.hs_token = hs_token # The full Matrix ID for this application service's sender. self.sender = sender + # The application service user should be part of the server's domain. + self.server_name = sender.domain # nb must be called this for @cached self.namespaces = self._check_namespaces(namespaces) self.id = id self.ip_range_whitelist = ip_range_whitelist self.supports_ephemeral = supports_ephemeral self.msc3202_transaction_extensions = msc3202_transaction_extensions + self.msc4190_device_management = msc4190_device_management if "|" in self.id: raise Exception("application service ID cannot contain '|' character") @@ -221,7 +225,7 @@ class ApplicationService: """ return ( # User is the appservice's configured sender_localpart user - user_id == self.sender + user_id == self.sender.to_string() # User is in the appservice's user namespace or self.is_user_in_namespace(user_id) ) @@ -345,7 +349,7 @@ class ApplicationService: def is_exclusive_user(self, user_id: str) -> bool: return ( self._is_exclusive(ApplicationService.NS_USERS, user_id) - or user_id == self.sender + or user_id == self.sender.to_string() ) def is_interested_in_protocol(self, protocol: str) -> bool: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 19322471dc..55069cc5d3 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -48,6 +48,7 @@ from synapse.events import EventBase from synapse.events.utils import SerializeEventConfig, serialize_event from synapse.http.client import SimpleHttpClient, is_unknown_endpoint from synapse.logging import opentracing +from synapse.metrics import SERVER_NAME_LABEL from synapse.types import DeviceListUpdates, JsonDict, JsonMapping, ThirdPartyInstanceID from synapse.util.caches.response_cache import ResponseCache @@ -59,29 +60,31 @@ logger = logging.getLogger(__name__) sent_transactions_counter = Counter( "synapse_appservice_api_sent_transactions", "Number of /transactions/ requests sent", - ["service"], + labelnames=["service", SERVER_NAME_LABEL], ) failed_transactions_counter = Counter( "synapse_appservice_api_failed_transactions", "Number of /transactions/ requests that failed to send", - ["service"], + labelnames=["service", SERVER_NAME_LABEL], ) sent_events_counter = Counter( - "synapse_appservice_api_sent_events", "Number of events sent to the AS", ["service"] + "synapse_appservice_api_sent_events", + "Number of events sent to the AS", + labelnames=["service", SERVER_NAME_LABEL], ) sent_ephemeral_counter = Counter( "synapse_appservice_api_sent_ephemeral", "Number of ephemeral events sent to the AS", - ["service"], + labelnames=["service", SERVER_NAME_LABEL], ) sent_todevice_counter = Counter( "synapse_appservice_api_sent_todevice", "Number of todevice messages sent to the AS", - ["service"], + labelnames=["service", SERVER_NAME_LABEL], ) HOUR_IN_MS = 60 * 60 * 1000 @@ -126,11 +129,15 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.clock = hs.get_clock() self.config = hs.config.appservice self.protocol_meta_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "as_protocol_meta", timeout_ms=HOUR_IN_MS + clock=hs.get_clock(), + name="as_protocol_meta", + server_name=self.server_name, + timeout_ms=HOUR_IN_MS, ) def _get_headers(self, service: "ApplicationService") -> Dict[bytes, List[bytes]]: @@ -378,6 +385,7 @@ class ApplicationServiceApi(SimpleHttpClient): "left": list(device_list_summary.left), } + labels = {"service": service.id, SERVER_NAME_LABEL: self.server_name} try: args = None if self.config.use_appservice_legacy_authorization: @@ -395,10 +403,10 @@ class ApplicationServiceApi(SimpleHttpClient): service.url, [event.get("event_id") for event in events], ) - sent_transactions_counter.labels(service.id).inc() - sent_events_counter.labels(service.id).inc(len(serialized_events)) - sent_ephemeral_counter.labels(service.id).inc(len(ephemeral)) - sent_todevice_counter.labels(service.id).inc(len(to_device_messages)) + sent_transactions_counter.labels(**labels).inc() + sent_events_counter.labels(**labels).inc(len(serialized_events)) + sent_ephemeral_counter.labels(**labels).inc(len(ephemeral)) + sent_todevice_counter.labels(**labels).inc(len(to_device_messages)) return True except CodeMessageException as e: logger.warning( @@ -417,7 +425,7 @@ class ApplicationServiceApi(SimpleHttpClient): ex.args, exc_info=logger.isEnabledFor(logging.DEBUG), ) - failed_transactions_counter.labels(service.id).inc() + failed_transactions_counter.labels(**labels).inc() return False async def claim_client_keys( @@ -555,6 +563,9 @@ class ApplicationServiceApi(SimpleHttpClient): ) and service.is_interested_in_user(e.state_key) ), + # Appservices are considered 'trusted' by the admin and should have + # applicable metadata on their events. + include_admin_metadata=True, ), ) for e in events diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 7994da0868..01f77c4cb6 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -2,7 +2,7 @@ # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright 2015, 2016 OpenMarket Ltd -# Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2023, 2025 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -70,6 +70,8 @@ from typing import ( Tuple, ) +from twisted.internet.interfaces import IDelayedCall + from synapse.appservice import ( ApplicationService, ApplicationServiceState, @@ -101,18 +103,16 @@ MAX_TO_DEVICE_MESSAGES_PER_TRANSACTION = 100 class ApplicationServiceScheduler: - """Public facing API for this module. Does the required DI to tie the - components together. This also serves as the "event_pool", which in this + """ + Public facing API for this module. Does the required dependency injection (DI) to + tie the components together. This also serves as the "event_pool", which in this case is a simple array. """ def __init__(self, hs: "HomeServer"): - self.clock = hs.get_clock() + self.txn_ctrl = _TransactionController(hs) self.store = hs.get_datastores().main - self.as_api = hs.get_application_service_api() - - self.txn_ctrl = _TransactionController(self.clock, self.store, self.as_api) - self.queuer = _ServiceQueuer(self.txn_ctrl, self.clock, hs) + self.queuer = _ServiceQueuer(self.txn_ctrl, hs) async def start(self) -> None: logger.info("Starting appservice scheduler") @@ -182,9 +182,7 @@ class _ServiceQueuer: appservice at a given time. """ - def __init__( - self, txn_ctrl: "_TransactionController", clock: Clock, hs: "HomeServer" - ): + def __init__(self, txn_ctrl: "_TransactionController", hs: "HomeServer"): # dict of {service_id: [events]} self.queued_events: Dict[str, List[EventBase]] = {} # dict of {service_id: [events]} @@ -197,10 +195,11 @@ class _ServiceQueuer: # the appservices which currently have a transaction in flight self.requests_in_flight: Set[str] = set() self.txn_ctrl = txn_ctrl - self.clock = clock self._msc3202_transaction_extensions_enabled: bool = ( hs.config.experimental.msc3202_transaction_extensions ) + self.server_name = hs.hostname + self.clock = hs.get_clock() self._store = hs.get_datastores().main def start_background_request(self, service: ApplicationService) -> None: @@ -208,7 +207,9 @@ class _ServiceQueuer: if service.id in self.requests_in_flight: return - run_as_background_process("as-sender", self._send_request, service) + run_as_background_process( + "as-sender", self.server_name, self._send_request, service + ) async def _send_request(self, service: ApplicationService) -> None: # sanity-check: we shouldn't get here if this service already has a sender @@ -317,7 +318,7 @@ class _ServiceQueuer: users: Set[str] = set() # The sender is always included - users.add(service.sender) + users.add(service.sender.to_string()) # All AS users that would receive the PDUs or EDUs sent to these rooms # are classed as 'interesting'. @@ -357,10 +358,11 @@ class _TransactionController: (Note we have only have one of these in the homeserver.) """ - def __init__(self, clock: Clock, store: DataStore, as_api: ApplicationServiceApi): - self.clock = clock - self.store = store - self.as_api = as_api + def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname + self.clock = hs.get_clock() + self.store = hs.get_datastores().main + self.as_api = hs.get_application_service_api() # map from service id to recoverer instance self.recoverers: Dict[str, "_Recoverer"] = {} @@ -444,12 +446,31 @@ class _TransactionController: logger.info("Starting recoverer for AS ID %s", service.id) assert service.id not in self.recoverers recoverer = self.RECOVERER_CLASS( - self.clock, self.store, self.as_api, service, self.on_recovered + self.server_name, + self.clock, + self.store, + self.as_api, + service, + self.on_recovered, ) self.recoverers[service.id] = recoverer recoverer.recover() logger.info("Now %i active recoverers", len(self.recoverers)) + def force_retry(self, service: ApplicationService) -> None: + """Forces a Recoverer to attempt delivery of transations immediately. + + Args: + service: + """ + recoverer = self.recoverers.get(service.id) + if not recoverer: + # No need to force a retry on a happy AS. + logger.info("%s is not in recovery, not forcing retry", service.id) + return + + recoverer.force_retry() + async def _is_service_up(self, service: ApplicationService) -> bool: state = await self.store.get_appservice_state(service) return state == ApplicationServiceState.UP or state is None @@ -461,33 +482,41 @@ class _Recoverer: We have one of these for each appservice which is currently considered DOWN. Args: - clock (synapse.util.Clock): - store (synapse.storage.DataStore): - as_api (synapse.appservice.api.ApplicationServiceApi): - service (synapse.appservice.ApplicationService): the service we are managing - callback (callable[_Recoverer]): called once the service recovers. + server_name: the homeserver name (used to label metrics) (this should be `hs.hostname`). + clock: + store: + as_api: + service: the service we are managing + callback: called once the service recovers. """ def __init__( self, + server_name: str, clock: Clock, store: DataStore, as_api: ApplicationServiceApi, service: ApplicationService, callback: Callable[["_Recoverer"], Awaitable[None]], ): + self.server_name = server_name self.clock = clock self.store = store self.as_api = as_api self.service = service self.callback = callback self.backoff_counter = 1 + self.scheduled_recovery: Optional[IDelayedCall] = None def recover(self) -> None: delay = 2**self.backoff_counter logger.info("Scheduling retries on %s in %fs", self.service.id, delay) - self.clock.call_later( - delay, run_as_background_process, "as-recoverer", self.retry + self.scheduled_recovery = self.clock.call_later( + delay, + run_as_background_process, + "as-recoverer", + self.server_name, + self.retry, ) def _backoff(self) -> None: @@ -496,6 +525,22 @@ class _Recoverer: self.backoff_counter += 1 self.recover() + def force_retry(self) -> None: + """Cancels the existing timer and forces an immediate retry in the background. + + Args: + service: + """ + # Prevent the existing backoff from occuring + if self.scheduled_recovery: + self.clock.cancel_call_later(self.scheduled_recovery) + # Run a retry, which will resechedule a recovery if it fails. + run_as_background_process( + "retry", + self.server_name, + self.retry, + ) + async def retry(self) -> None: logger.info("Starting retries on %s", self.service.id) try: diff --git a/synapse/config/_base.py b/synapse/config/_base.py index adce34c03a..191253ddda 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -22,6 +22,7 @@ import argparse import errno +import importlib.resources as importlib_resources import logging import os import re @@ -46,7 +47,6 @@ from typing import ( import attr import jinja2 -import pkg_resources import yaml from synapse.types import StrSequence @@ -170,12 +170,12 @@ class Config: section: ClassVar[str] - def __init__(self, root_config: "RootConfig" = None): + def __init__(self, root_config: "RootConfig"): self.root = root_config # Get the path to the default Synapse template directory - self.default_template_dir = pkg_resources.resource_filename( - "synapse", "res/templates" + self.default_template_dir = str( + importlib_resources.files("synapse").joinpath("res").joinpath("templates") ) @staticmethod @@ -221,9 +221,13 @@ class Config: The number of milliseconds in the duration. Raises: - TypeError, if given something other than an integer or a string + TypeError: if given something other than an integer or a string, or the + duration is using an incorrect suffix. ValueError: if given a string not of the form described above. """ + # For integers, we prefer to use `type(value) is int` instead of + # `isinstance(value, int)` because we want to exclude subclasses of int, such as + # bool. if type(value) is int: # noqa: E721 return value elif isinstance(value, str): @@ -246,9 +250,20 @@ class Config: if suffix in sizes: value = value[:-1] size = sizes[suffix] + elif suffix.isdigit(): + # No suffix is treated as milliseconds. + value = value + size = 1 + else: + raise TypeError( + f"Bad duration suffix {value} (expected no suffix or one of these suffixes: {sizes.keys()})" + ) + return int(value) * size else: - raise TypeError(f"Bad duration {value!r}") + raise TypeError( + f"Bad duration type {value!r} (expected int or string duration)" + ) @staticmethod def abspath(file_path: str) -> str: @@ -430,7 +445,7 @@ class RootConfig: return res @classmethod - def invoke_all_static(cls, func_name: str, *args: Any, **kwargs: any) -> None: + def invoke_all_static(cls, func_name: str, *args: Any, **kwargs: Any) -> None: """ Invoke a static function on config objects this RootConfig is configured to use. @@ -574,6 +589,14 @@ class RootConfig: " Defaults to the directory containing the last config file", ) + config_parser.add_argument( + "--no-secrets-in-config", + dest="secrets_in_config", + action="store_false", + default=True, + help="Reject config options that expect an in-line secret as value.", + ) + cls.invoke_all_static("add_arguments", config_parser) @classmethod @@ -611,7 +634,10 @@ class RootConfig: config_dict = read_config_files(config_files) obj.parse_config_dict( - config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path + config_dict, + config_dir_path=config_dir_path, + data_dir_path=data_dir_path, + allow_secrets_in_config=config_args.secrets_in_config, ) obj.invoke_all("read_arguments", config_args) @@ -638,6 +664,13 @@ class RootConfig: help="Specify config file. Can be given multiple times and" " may specify directories containing *.yaml files.", ) + parser.add_argument( + "--no-secrets-in-config", + dest="secrets_in_config", + action="store_false", + default=True, + help="Reject config options that expect an in-line secret as value.", + ) # we nest the mutually-exclusive group inside another group so that the help # text shows them in their own group. @@ -806,14 +839,21 @@ class RootConfig: return None obj.parse_config_dict( - config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path + config_dict, + config_dir_path=config_dir_path, + data_dir_path=data_dir_path, + allow_secrets_in_config=config_args.secrets_in_config, ) obj.invoke_all("read_arguments", config_args) return obj def parse_config_dict( - self, config_dict: Dict[str, Any], config_dir_path: str, data_dir_path: str + self, + config_dict: Dict[str, Any], + config_dir_path: str, + data_dir_path: str, + allow_secrets_in_config: bool = True, ) -> None: """Read the information from the config dict into this Config object. @@ -831,6 +871,7 @@ class RootConfig: config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path, + allow_secrets_in_config=allow_secrets_in_config, ) def generate_missing_files( @@ -868,7 +909,10 @@ class RootConfig: def read_config_files(config_files: Iterable[str]) -> Dict[str, Any]: - """Read the config files into a dict + """Read the config files and shallowly merge them into a dict. + + Successive configurations are shallowly merged into ones provided earlier, + i.e., entirely replacing top-level sections of the configuration. Args: config_files: A list of the config files to read @@ -1006,7 +1050,7 @@ class RoutableShardedWorkerHandlingConfig(ShardedWorkerHandlingConfig): return self._get_instance(key) -def read_file(file_path: Any, config_path: Iterable[str]) -> str: +def read_file(file_path: Any, config_path: StrSequence) -> str: """Check the given file exists, and read it into a string If it does not, emit an error indicating the problem diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index d9cb0da38b..5e03635206 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -36,6 +36,7 @@ from synapse.config import ( # noqa: F401 jwt, key, logger, + mas, metrics, modules, oembed, @@ -59,6 +60,7 @@ from synapse.config import ( # noqa: F401 tls, tracer, user_directory, + user_types, voip, workers, ) @@ -122,6 +124,8 @@ class RootConfig: retention: retention.RetentionConfig background_updates: background_updates.BackgroundUpdateConfig auto_accept_invites: auto_accept_invites.AutoAcceptInvitesConfig + user_types: user_types.UserTypesConfig + mas: mas.MasConfig config_classes: List[Type["Config"]] = ... config_files: List[str] @@ -132,7 +136,11 @@ class RootConfig: @classmethod def invoke_all_static(cls, func_name: str, *args: Any, **kwargs: Any) -> None: ... def parse_config_dict( - self, config_dict: Dict[str, Any], config_dir_path: str, data_dir_path: str + self, + config_dict: Dict[str, Any], + config_dir_path: str, + data_dir_path: str, + allow_secrets_in_config: bool = ..., ) -> None: ... def generate_config( self, @@ -175,7 +183,7 @@ class RootConfig: class Config: root: RootConfig default_template_dir: str - def __init__(self, root_config: Optional[RootConfig] = ...) -> None: ... + def __init__(self, root_config: RootConfig = ...) -> None: ... @staticmethod def parse_size(value: Union[str, int]) -> int: ... @staticmethod @@ -208,4 +216,4 @@ class ShardedWorkerHandlingConfig: class RoutableShardedWorkerHandlingConfig(ShardedWorkerHandlingConfig): def get_instance(self, key: str) -> str: ... # noqa: F811 -def read_file(file_path: Any, config_path: Iterable[str]) -> str: ... +def read_file(file_path: Any, config_path: StrSequence) -> str: ... diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 6ff00e1ff8..81dbd330cc 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -122,8 +122,7 @@ def _load_appservice( localpart = as_info["sender_localpart"] if urlparse.quote(localpart) != localpart: raise ValueError("sender_localpart needs characters which are not URL encoded.") - user = UserID(localpart, hostname) - user_id = user.to_string() + user_id = UserID(localpart, hostname) # Rate limiting for users of this AS is on by default (excludes sender) rate_limited = as_info.get("rate_limited") @@ -183,6 +182,18 @@ def _load_appservice( "The `org.matrix.msc3202` option should be true or false if specified." ) + # Opt-in flag for the MSC4190 behaviours. + # When enabled, the following C-S API endpoints change for appservices: + # - POST /register does not return an access token + # - PUT /devices/{device_id} creates a new device if one does not exist + # - DELETE /devices/{device_id} no longer requires UIA + # - POST /delete_devices/{device_id} no longer requires UIA + msc4190_enabled = as_info.get("io.element.msc4190", False) + if not isinstance(msc4190_enabled, bool): + raise ValueError( + "The `io.element.msc4190` option should be true or false if specified." + ) + return ApplicationService( token=as_info["as_token"], url=as_info["url"], @@ -195,4 +206,5 @@ def _load_appservice( ip_range_whitelist=ip_range_whitelist, supports_ephemeral=supports_ephemeral, msc3202_transaction_extensions=msc3202_transaction_extensions, + msc4190_device_management=msc4190_enabled, ) diff --git a/synapse/config/auth.py b/synapse/config/auth.py index 9246fd6430..31b332dc09 100644 --- a/synapse/config/auth.py +++ b/synapse/config/auth.py @@ -36,13 +36,14 @@ class AuthConfig(Config): if password_config is None: password_config = {} - # The default value of password_config.enabled is True, unless msc3861 is enabled. - msc3861_enabled = ( - (config.get("experimental_features") or {}) - .get("msc3861", {}) - .get("enabled", False) - ) - passwords_enabled = password_config.get("enabled", not msc3861_enabled) + auth_delegated = (config.get("experimental_features") or {}).get( + "msc3861", {} + ).get("enabled", False) or ( + config.get("matrix_authentication_service") or {} + ).get("enabled", False) + + # The default value of password_config.enabled is True, unless auth is delegated + passwords_enabled = password_config.get("enabled", not auth_delegated) # 'only_for_reauth' allows users who have previously set a password to use it, # even though passwords would otherwise be disabled. diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 84897c09c5..ea1a5e0b32 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -23,14 +23,38 @@ from typing import Any from synapse.types import JsonDict -from ._base import Config, ConfigError +from ._base import Config, ConfigError, read_file + +CONFLICTING_RECAPTCHA_PRIVATE_KEY_OPTS_ERROR = """\ +You have configured both `recaptcha_private_key` and +`recaptcha_private_key_path`. These are mutually incompatible. +""" + +CONFLICTING_RECAPTCHA_PUBLIC_KEY_OPTS_ERROR = """\ +You have configured both `recaptcha_public_key` and `recaptcha_public_key_path`. +These are mutually incompatible. +""" class CaptchaConfig(Config): section = "captcha" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: recaptcha_private_key = config.get("recaptcha_private_key") + if recaptcha_private_key and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("recaptcha_private_key",), + ) + recaptcha_private_key_path = config.get("recaptcha_private_key_path") + if recaptcha_private_key_path: + if recaptcha_private_key: + raise ConfigError(CONFLICTING_RECAPTCHA_PRIVATE_KEY_OPTS_ERROR) + recaptcha_private_key = read_file( + recaptcha_private_key_path, ("recaptcha_private_key_path",) + ).strip() if recaptcha_private_key is not None and not isinstance( recaptcha_private_key, str ): @@ -38,6 +62,18 @@ class CaptchaConfig(Config): self.recaptcha_private_key = recaptcha_private_key recaptcha_public_key = config.get("recaptcha_public_key") + if recaptcha_public_key and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("recaptcha_public_key",), + ) + recaptcha_public_key_path = config.get("recaptcha_public_key_path") + if recaptcha_public_key_path: + if recaptcha_public_key: + raise ConfigError(CONFLICTING_RECAPTCHA_PUBLIC_KEY_OPTS_ERROR) + recaptcha_public_key = read_file( + recaptcha_public_key_path, ("recaptcha_public_key_path",) + ).strip() if recaptcha_public_key is not None and not isinstance( recaptcha_public_key, str ): diff --git a/synapse/config/cas.py b/synapse/config/cas.py index fa59c350c1..60d66d7019 100644 --- a/synapse/config/cas.py +++ b/synapse/config/cas.py @@ -20,7 +20,7 @@ # # -from typing import Any, List +from typing import Any, List, Optional from synapse.config.sso import SsoAttributeRequirement from synapse.types import JsonDict @@ -42,11 +42,16 @@ class CasConfig(Config): self.cas_enabled = cas_config and cas_config.get("enabled", True) if self.cas_enabled: + if not isinstance(cas_config, dict): + raise ConfigError("Must be a dictionary", ("cas_config",)) + self.cas_server_url = cas_config["server_url"] # TODO Update this to a _synapse URL. public_baseurl = self.root.server.public_baseurl - self.cas_service_url = public_baseurl + "_matrix/client/r0/login/cas/ticket" + self.cas_service_url: Optional[str] = ( + public_baseurl + "_matrix/client/r0/login/cas/ticket" + ) self.cas_protocol_version = cas_config.get("protocol_version") if ( diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 8033fa2e52..c3a3e05a82 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -110,6 +110,7 @@ class EmailConfig(Config): raise ConfigError( "email.require_transport_security requires email.enable_tls to be true" ) + self.email_tlsname = email_config.get("tlsname", None) if "app_name" in email_config: self.email_app_name = email_config["app_name"] diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 3411179a2a..d086deab3f 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -20,6 +20,7 @@ # import enum +from functools import cache from typing import TYPE_CHECKING, Any, Optional import attr @@ -27,8 +28,8 @@ import attr.validators from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config import ConfigError -from synapse.config._base import Config, RootConfig -from synapse.types import JsonDict +from synapse.config._base import Config, RootConfig, read_file +from synapse.types import JsonDict, StrSequence # Determine whether authlib is installed. try: @@ -43,6 +44,12 @@ if TYPE_CHECKING: from authlib.jose.rfc7517 import JsonWebKey +@cache +def read_secret_from_file_once(file_path: Any, config_path: StrSequence) -> str: + """Returns the memoized secret read from file.""" + return read_file(file_path, config_path).strip() + + class ClientAuthMethod(enum.Enum): """List of supported client auth methods.""" @@ -63,6 +70,40 @@ def _parse_jwks(jwks: Optional[JsonDict]) -> Optional["JsonWebKey"]: return JsonWebKey.import_key(jwks) +def _check_client_secret( + instance: "MSC3861", _attribute: attr.Attribute, _value: Optional[str] +) -> None: + if instance._client_secret and instance._client_secret_path: + raise ConfigError( + ( + "You have configured both " + "`experimental_features.msc3861.client_secret` and " + "`experimental_features.msc3861.client_secret_path`. " + "These are mutually incompatible." + ), + ("experimental", "msc3861", "client_secret"), + ) + # Check client secret can be retrieved + instance.client_secret() + + +def _check_admin_token( + instance: "MSC3861", _attribute: attr.Attribute, _value: Optional[str] +) -> None: + if instance._admin_token and instance._admin_token_path: + raise ConfigError( + ( + "You have configured both " + "`experimental_features.msc3861.admin_token` and " + "`experimental_features.msc3861.admin_token_path`. " + "These are mutually incompatible." + ), + ("experimental", "msc3861", "admin_token"), + ) + # Check client secret can be retrieved + instance.admin_token() + + @attr.s(slots=True, frozen=True) class MSC3861: """Configuration for MSC3861: Matrix architecture change to delegate authentication via OIDC""" @@ -97,15 +138,30 @@ class MSC3861: ) """The auth method used when calling the introspection endpoint.""" - client_secret: Optional[str] = attr.ib( + _client_secret: Optional[str] = attr.ib( default=None, - validator=attr.validators.optional(attr.validators.instance_of(str)), + validator=[ + attr.validators.optional(attr.validators.instance_of(str)), + _check_client_secret, + ], ) """ The client secret to use when calling the introspection endpoint, when using any of the client_secret_* client auth methods. """ + _client_secret_path: Optional[str] = attr.ib( + default=None, + validator=[ + attr.validators.optional(attr.validators.instance_of(str)), + _check_client_secret, + ], + ) + """ + Alternative to `client_secret`: allows the secret to be specified in an + external file. + """ + jwk: Optional["JsonWebKey"] = attr.ib(default=None, converter=_parse_jwks) """ The JWKS to use when calling the introspection endpoint, @@ -133,7 +189,7 @@ class MSC3861: ClientAuthMethod.CLIENT_SECRET_BASIC, ClientAuthMethod.CLIENT_SECRET_JWT, ) - and self.client_secret is None + and self.client_secret() is None ): raise ConfigError( f"A client secret must be provided when using the {value} client auth method", @@ -152,16 +208,51 @@ class MSC3861: ) """The URL of the My Account page on the OIDC Provider as per MSC2965.""" - admin_token: Optional[str] = attr.ib( + _admin_token: Optional[str] = attr.ib( default=None, - validator=attr.validators.optional(attr.validators.instance_of(str)), + validator=[ + attr.validators.optional(attr.validators.instance_of(str)), + _check_admin_token, + ], ) """ A token that should be considered as an admin token. This is used by the OIDC provider, to make admin calls to Synapse. """ - def check_config_conflicts(self, root: RootConfig) -> None: + _admin_token_path: Optional[str] = attr.ib( + default=None, + validator=[ + attr.validators.optional(attr.validators.instance_of(str)), + _check_admin_token, + ], + ) + """ + Alternative to `admin_token`: allows the secret to be specified in an + external file. + """ + + def client_secret(self) -> Optional[str]: + """Returns the secret given via `client_secret` or `client_secret_path`.""" + if self._client_secret_path: + return read_secret_from_file_once( + self._client_secret_path, + ("experimental_features", "msc3861", "client_secret_path"), + ) + return self._client_secret + + def admin_token(self) -> Optional[str]: + """Returns the admin token given via `admin_token` or `admin_token_path`.""" + if self._admin_token_path: + return read_secret_from_file_once( + self._admin_token_path, + ("experimental_features", "msc3861", "admin_token_path"), + ) + return self._admin_token + + def check_config_conflicts( + self, root: RootConfig, allow_secrets_in_config: bool + ) -> None: """Checks for any configuration conflicts with other parts of Synapse. Raises: @@ -171,6 +262,24 @@ class MSC3861: if not self.enabled: return + if self._client_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("experimental", "msc3861", "client_secret"), + ) + + if self.jwk and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("experimental", "msc3861", "jwk"), + ) + + if self._admin_token and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("experimental", "msc3861", "admin_token"), + ) + if ( root.auth.password_enabled_for_reauth or root.auth.password_enabled_for_login @@ -261,7 +370,9 @@ class ExperimentalConfig(Config): section = "experimental" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: experimental = config.get("experimental_features") or {} # MSC3026 (busy presence state) @@ -405,7 +516,9 @@ class ExperimentalConfig(Config): ) from exc # Check that none of the other config options conflict with MSC3861 when enabled - self.msc3861.check_config_conflicts(self.root) + self.msc3861.check_config_conflicts( + self.root, allow_secrets_in_config=allow_secrets_in_config + ) self.msc4028_push_encrypted_events = experimental.get( "msc4028_push_encrypted_events", False @@ -422,11 +535,15 @@ class ExperimentalConfig(Config): "msc4108_delegation_endpoint", None ) + auth_delegated = self.msc3861.enabled or ( + config.get("matrix_authentication_service") or {} + ).get("enabled", False) + if ( self.msc4108_enabled or self.msc4108_delegation_endpoint is not None - ) and not self.msc3861.enabled: + ) and not auth_delegated: raise ConfigError( - "MSC4108 requires MSC3861 to be enabled", + "MSC4108 requires MSC3861 or matrix_authentication_service to be enabled", ("experimental", "msc4108_delegation_endpoint"), ) @@ -436,15 +553,42 @@ class ExperimentalConfig(Config): ("experimental", "msc4108_delegation_endpoint"), ) - self.msc3823_account_suspension = experimental.get( - "msc3823_account_suspension", False - ) - - # MSC4151: Report room API (Client-Server API) - self.msc4151_enabled: bool = experimental.get("msc4151_enabled", False) + # MSC4133: Custom profile fields + self.msc4133_enabled: bool = experimental.get("msc4133_enabled", False) # MSC4210: Remove legacy mentions self.msc4210_enabled: bool = experimental.get("msc4210_enabled", False) # MSC4222: Adding `state_after` to sync v2 self.msc4222_enabled: bool = experimental.get("msc4222_enabled", False) + + # MSC4076: Add `disable_badge_count`` to pusher configuration + self.msc4076_enabled: bool = experimental.get("msc4076_enabled", False) + + # MSC4277: Harmonizing the reporting endpoints + # + # If enabled, ignore the score parameter and respond with HTTP 200 on + # reporting requests regardless of the subject's existence. + self.msc4277_enabled: bool = experimental.get("msc4277_enabled", False) + + # MSC4235: Add `via` param to hierarchy endpoint + self.msc4235_enabled: bool = experimental.get("msc4235_enabled", False) + + # MSC4263: Preventing MXID enumeration via key queries + self.msc4263_limit_key_queries_to_users_who_share_rooms = experimental.get( + "msc4263_limit_key_queries_to_users_who_share_rooms", + False, + ) + + # MSC4267: Automatically forgetting rooms on leave + self.msc4267_enabled: bool = experimental.get("msc4267_enabled", False) + + # MSC4155: Invite filtering + self.msc4155_enabled: bool = experimental.get("msc4155_enabled", False) + + # MSC4293: Redact on Kick/Ban + self.msc4293_enabled: bool = experimental.get("msc4293_enabled", False) + + # MSC4306: Thread Subscriptions + # (and MSC4308: Thread Subscriptions extension to Sliding Sync) + self.msc4306_enabled: bool = experimental.get("msc4306_enabled", False) diff --git a/synapse/config/federation.py b/synapse/config/federation.py index cf29fa2562..31f46e420d 100644 --- a/synapse/config/federation.py +++ b/synapse/config/federation.py @@ -94,5 +94,21 @@ class FederationConfig(Config): 2**62, ) + def is_domain_allowed_according_to_federation_whitelist(self, domain: str) -> bool: + """ + Returns whether a domain is allowed according to the federation whitelist. If a + federation whitelist is not set, all domains are allowed. + + Args: + domain: The domain to test. + + Returns: + True if the domain is allowed or if a whitelist is not set, False otherwise. + """ + if self.federation_domain_whitelist is None: + return True + + return domain in self.federation_domain_whitelist + _METRICS_FOR_DOMAINS_SCHEMA = {"type": "array", "items": {"type": "string"}} diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index e36c0bd6ae..5d7089c2e6 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -36,6 +36,7 @@ from .federation import FederationConfig from .jwt import JWTConfig from .key import KeyConfig from .logger import LoggingConfig +from .mas import MasConfig from .metrics import MetricsConfig from .modules import ModulesConfig from .oembed import OembedConfig @@ -59,6 +60,7 @@ from .third_party_event_rules import ThirdPartyRulesConfig from .tls import TlsConfig from .tracer import TracerConfig from .user_directory import UserDirectoryConfig +from .user_types import UserTypesConfig from .voip import VoipConfig from .workers import WorkerConfig @@ -107,4 +109,7 @@ class HomeServerConfig(RootConfig): ExperimentalConfig, BackgroundUpdateConfig, AutoAcceptInvitesConfig, + UserTypesConfig, + # This must be last, as it checks for conflicts with other config options. + MasConfig, ] diff --git a/synapse/config/key.py b/synapse/config/key.py index bc96888967..f78ff5114f 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -43,7 +43,7 @@ from unpaddedbase64 import decode_base64 from synapse.types import JsonDict from synapse.util.stringutils import random_string, random_string_with_symbols -from ._base import Config, ConfigError +from ._base import Config, ConfigError, read_file if TYPE_CHECKING: from signedjson.key import VerifyKeyWithExpiry @@ -91,6 +91,16 @@ To suppress this warning and continue using 'matrix.org', admins should set 'suppress_key_server_warning' to 'true' in homeserver.yaml. --------------------------------------------------------------------------------""" +CONFLICTING_MACAROON_SECRET_KEY_OPTS_ERROR = """\ +Conflicting options 'macaroon_secret_key' and 'macaroon_secret_key_path' are +both defined in config file. +""" + +CONFLICTING_FORM_SECRET_OPTS_ERROR = """\ +Conflicting options 'form_secret' and 'form_secret_path' are both defined in +config file. +""" + logger = logging.getLogger(__name__) @@ -107,7 +117,11 @@ class KeyConfig(Config): section = "key" def read_config( - self, config: JsonDict, config_dir_path: str, **kwargs: Any + self, + config: JsonDict, + config_dir_path: str, + allow_secrets_in_config: bool, + **kwargs: Any, ) -> None: # the signing key can be specified inline or in a separate file if "signing_key" in config: @@ -166,10 +180,21 @@ class KeyConfig(Config): ) ) - macaroon_secret_key: Optional[str] = config.get( - "macaroon_secret_key", self.root.registration.registration_shared_secret - ) - + macaroon_secret_key = config.get("macaroon_secret_key") + if macaroon_secret_key and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("macaroon_secret_key",), + ) + macaroon_secret_key_path = config.get("macaroon_secret_key_path") + if macaroon_secret_key_path: + if macaroon_secret_key: + raise ConfigError(CONFLICTING_MACAROON_SECRET_KEY_OPTS_ERROR) + macaroon_secret_key = read_file( + macaroon_secret_key_path, ("macaroon_secret_key_path",) + ).strip() + if not macaroon_secret_key: + macaroon_secret_key = self.root.registration.registration_shared_secret if not macaroon_secret_key: # Unfortunately, there are people out there that don't have this # set. Lets just be "nice" and derive one from their secret key. @@ -181,7 +206,24 @@ class KeyConfig(Config): # a secret which is used to calculate HMACs for form values, to stop # falsification of values - self.form_secret = config.get("form_secret", None) + form_secret = config.get("form_secret", None) + if form_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("form_secret",), + ) + if form_secret is not None and not isinstance(form_secret, str): + raise ConfigError("Config option must be a string", ("form_secret",)) + + form_secret_path = config.get("form_secret_path", None) + if form_secret_path: + if form_secret: + raise ConfigError(CONFLICTING_FORM_SECRET_OPTS_ERROR) + self.form_secret: Optional[str] = read_file( + form_secret_path, ("form_secret_path",) + ).strip() + else: + self.form_secret = form_secret def generate_config_section( self, diff --git a/synapse/config/logger.py b/synapse/config/logger.py index cfc1a57107..3f86ec1169 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -51,6 +51,8 @@ if TYPE_CHECKING: from synapse.config.homeserver import HomeServerConfig from synapse.server import HomeServer +logger = logging.getLogger(__name__) + DEFAULT_LOG_CONFIG = Template( """\ # Log configuration for Synapse. @@ -291,7 +293,7 @@ def _load_logging_config(log_config_path: str) -> None: log_config = yaml.safe_load(f.read()) if not log_config: - logging.warning("Loaded a blank logging config?") + logger.warning("Loaded a blank logging config?") # If the old structured logging configuration is being used, raise an error. if "structured" in log_config and log_config.get("structured"): @@ -312,7 +314,7 @@ def _reload_logging_config(log_config_path: Optional[str]) -> None: return _load_logging_config(log_config_path) - logging.info("Reloaded log config from %s due to SIGHUP", log_config_path) + logger.info("Reloaded log config from %s due to SIGHUP", log_config_path) def setup_logging( @@ -349,16 +351,17 @@ def setup_logging( appbase.register_sighup(_reload_logging_config, log_config_path) # Log immediately so we can grep backwards. - logging.warning("***** STARTING SERVER *****") - logging.warning( + logger.warning("***** STARTING SERVER *****") + logger.warning( "Server %s version %s", sys.argv[0], SYNAPSE_VERSION, ) - logging.warning("Copyright (c) 2023 New Vector, Inc") - logging.warning( + logger.warning("Copyright (c) 2023 New Vector, Inc") + logger.warning( "Licensed under the AGPL 3.0 license. Website: https://github.com/element-hq/synapse" ) - logging.info("Server hostname: %s", config.server.server_name) - logging.info("Instance name: %s", hs.get_instance_name()) - logging.info("Twisted reactor: %s", type(reactor).__name__) + logger.info("Server hostname: %s", config.server.server_name) + logger.info("Public Base URL: %s", config.server.public_baseurl) + logger.info("Instance name: %s", hs.get_instance_name()) + logger.info("Twisted reactor: %s", type(reactor).__name__) diff --git a/synapse/config/mas.py b/synapse/config/mas.py new file mode 100644 index 0000000000..fe0d326f7a --- /dev/null +++ b/synapse/config/mas.py @@ -0,0 +1,192 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +from typing import Any, Optional + +from synapse._pydantic_compat import ( + AnyHttpUrl, + Field, + FilePath, + StrictBool, + StrictStr, + ValidationError, + validator, +) +from synapse.config.experimental import read_secret_from_file_once +from synapse.types import JsonDict +from synapse.util.pydantic_models import ParseModel + +from ._base import Config, ConfigError, RootConfig + + +class MasConfigModel(ParseModel): + enabled: StrictBool = False + endpoint: AnyHttpUrl = Field(default="http://localhost:8080") + secret: Optional[StrictStr] = Field(default=None) + secret_path: Optional[FilePath] = Field(default=None) + + @validator("secret") + def validate_secret_is_set_if_enabled(cls, v: Any, values: dict) -> Any: + if values.get("enabled", False) and not values.get("secret_path") and not v: + raise ValueError( + "You must set a `secret` or `secret_path` when enabling Matrix Authentication Service integration." + ) + + return v + + @validator("secret_path") + def validate_secret_path_is_set_if_enabled(cls, v: Any, values: dict) -> Any: + if values.get("secret"): + raise ValueError( + "`secret` and `secret_path` cannot be set at the same time." + ) + + return v + + +class MasConfig(Config): + section = "mas" + + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: + mas_config = config.get("matrix_authentication_service", {}) + if mas_config is None: + mas_config = {} + + try: + parsed = MasConfigModel(**mas_config) + except ValidationError as e: + raise ConfigError( + "Could not validate Matrix Authentication Service configuration", + path=("matrix_authentication_service",), + ) from e + + if parsed.secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("matrix_authentication_service", "secret"), + ) + + self.enabled = parsed.enabled + self.endpoint = parsed.endpoint + self._secret = parsed.secret + self._secret_path = parsed.secret_path + + self.check_config_conflicts(self.root) + + def check_config_conflicts( + self, + root: RootConfig, + ) -> None: + """Checks for any configuration conflicts with other parts of Synapse. + + Raises: + ConfigError: If there are any configuration conflicts. + """ + + if not self.enabled: + return + + if root.experimental.msc3861.enabled: + raise ConfigError( + "Experimental MSC3861 was replaced by Matrix Authentication Service." + "Please disable MSC3861 or disable Matrix Authentication Service.", + ("experimental", "msc3861"), + ) + + if ( + root.auth.password_enabled_for_reauth + or root.auth.password_enabled_for_login + ): + raise ConfigError( + "Password auth cannot be enabled when OAuth delegation is enabled", + ("password_config", "enabled"), + ) + + if root.registration.enable_registration: + raise ConfigError( + "Registration cannot be enabled when OAuth delegation is enabled", + ("enable_registration",), + ) + + # We only need to test the user consent version, as if it must be set if the user_consent section was present in the config + if root.consent.user_consent_version is not None: + raise ConfigError( + "User consent cannot be enabled when OAuth delegation is enabled", + ("user_consent",), + ) + + if ( + root.oidc.oidc_enabled + or root.saml2.saml2_enabled + or root.cas.cas_enabled + or root.jwt.jwt_enabled + ): + raise ConfigError("SSO cannot be enabled when OAuth delegation is enabled") + + if bool(root.authproviders.password_providers): + raise ConfigError( + "Password auth providers cannot be enabled when OAuth delegation is enabled" + ) + + if root.captcha.enable_registration_captcha: + raise ConfigError( + "CAPTCHA cannot be enabled when OAuth delegation is enabled", + ("captcha", "enable_registration_captcha"), + ) + + if root.auth.login_via_existing_enabled: + raise ConfigError( + "Login via existing session cannot be enabled when OAuth delegation is enabled", + ("login_via_existing_session", "enabled"), + ) + + if root.registration.refresh_token_lifetime: + raise ConfigError( + "refresh_token_lifetime cannot be set when OAuth delegation is enabled", + ("refresh_token_lifetime",), + ) + + if root.registration.nonrefreshable_access_token_lifetime: + raise ConfigError( + "nonrefreshable_access_token_lifetime cannot be set when OAuth delegation is enabled", + ("nonrefreshable_access_token_lifetime",), + ) + + if root.registration.session_lifetime: + raise ConfigError( + "session_lifetime cannot be set when OAuth delegation is enabled", + ("session_lifetime",), + ) + + if root.registration.enable_3pid_changes: + raise ConfigError( + "enable_3pid_changes cannot be enabled when OAuth delegation is enabled", + ("enable_3pid_changes",), + ) + + def secret(self) -> str: + if self._secret is not None: + return self._secret + elif self._secret_path is not None: + return read_secret_from_file_once( + str(self._secret_path), + ("matrix_authentication_service", "secret_path"), + ) + else: + raise RuntimeError( + "Neither `secret` nor `secret_path` are set, this is a bug.", + ) diff --git a/synapse/config/oembed.py b/synapse/config/oembed.py index b177a75cf6..1b6c521087 100644 --- a/synapse/config/oembed.py +++ b/synapse/config/oembed.py @@ -18,13 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # +import importlib.resources as importlib_resources import json import re from typing import Any, Dict, Iterable, List, Optional, Pattern from urllib import parse as urlparse import attr -import pkg_resources from synapse.types import JsonDict, StrSequence @@ -64,7 +64,12 @@ class OembedConfig(Config): """ # Whether to use the packaged providers.json file. if not oembed_config.get("disable_default_providers") or False: - with pkg_resources.resource_stream("synapse", "res/providers.json") as s: + path = ( + importlib_resources.files("synapse") + .joinpath("res") + .joinpath("providers.json") + ) + with path.open("r", encoding="utf-8") as s: providers = json.load(s) yield from self._parse_and_validate_provider( diff --git a/synapse/config/oidc.py b/synapse/config/oidc.py index d0a03baf55..3ddf65a3e9 100644 --- a/synapse/config/oidc.py +++ b/synapse/config/oidc.py @@ -125,6 +125,10 @@ OIDC_PROVIDER_CONFIG_SCHEMA = { "enum": ["client_secret_basic", "client_secret_post", "none"], }, "pkce_method": {"type": "string", "enum": ["auto", "always", "never"]}, + "id_token_signing_alg_values_supported": { + "type": "array", + "items": {"type": "string"}, + }, "scopes": {"type": "array", "items": {"type": "string"}}, "authorization_endpoint": {"type": "string"}, "token_endpoint": {"type": "string"}, @@ -137,6 +141,9 @@ OIDC_PROVIDER_CONFIG_SCHEMA = { "type": "string", "enum": ["auto", "userinfo_endpoint"], }, + "redirect_uri": { + "type": ["string", "null"], + }, "allow_existing_users": {"type": "boolean"}, "user_mapping_provider": {"type": ["object", "null"]}, "attribute_requirements": { @@ -326,6 +333,9 @@ def _parse_oidc_config_dict( client_secret_jwt_key=client_secret_jwt_key, client_auth_method=client_auth_method, pkce_method=oidc_config.get("pkce_method", "auto"), + id_token_signing_alg_values_supported=oidc_config.get( + "id_token_signing_alg_values_supported" + ), scopes=oidc_config.get("scopes", ["openid"]), authorization_endpoint=oidc_config.get("authorization_endpoint"), token_endpoint=oidc_config.get("token_endpoint"), @@ -337,6 +347,7 @@ def _parse_oidc_config_dict( ), skip_verification=oidc_config.get("skip_verification", False), user_profile_method=oidc_config.get("user_profile_method", "auto"), + redirect_uri=oidc_config.get("redirect_uri"), allow_existing_users=oidc_config.get("allow_existing_users", False), user_mapping_provider_class=user_mapping_provider_class, user_mapping_provider_config=user_mapping_provider_config, @@ -345,6 +356,9 @@ def _parse_oidc_config_dict( additional_authorization_parameters=oidc_config.get( "additional_authorization_parameters", {} ), + passthrough_authorization_parameters=oidc_config.get( + "passthrough_authorization_parameters", [] + ), ) @@ -402,6 +416,34 @@ class OidcProviderConfig: # Valid values are 'auto', 'always', and 'never'. pkce_method: str + id_token_signing_alg_values_supported: Optional[List[str]] + """ + List of the JWS signing algorithms (`alg` values) that are supported for signing the + `id_token`. + + This is *not* required if `discovery` is disabled. We default to supporting `RS256` + in the downstream usage if no algorithms are configured here or in the discovery + document. + + According to the spec, the algorithm `"RS256"` MUST be included. The absolute rigid + approach would be to reject this provider as non-compliant if it's not included but + we can just allow whatever and see what happens (they're the ones that configured + the value and cooperating with the identity provider). It wouldn't be wise to add it + ourselves because absence of `RS256` might indicate that the provider actually + doesn't support it, despite the spec requirement. Adding it silently could lead to + failed authentication attempts or strange mismatch attacks. + + The `alg` value `"none"` MAY be supported but can only be used if the Authorization + Endpoint does not include `id_token` in the `response_type` (ex. + `/authorize?response_type=code` where `none` can apply, + `/authorize?response_type=code%20id_token` where `none` can't apply) (such as when + using the Authorization Code Flow). + + Spec: + - https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata + - https://openid.net/specs/openid-connect-core-1_0.html#AuthorizationExamples + """ + # list of scopes to request scopes: Collection[str] @@ -432,6 +474,18 @@ class OidcProviderConfig: # values are: "auto" or "userinfo_endpoint". user_profile_method: str + redirect_uri: Optional[str] + """ + An optional replacement for Synapse's hardcoded `redirect_uri` URL + (`/_synapse/client/oidc/callback`). This can be used to send + the client to a different URL after it receives a response from the + `authorization_endpoint`. + + If this is set, the client is expected to call Synapse's OIDC callback URL + reproduced above itself with the necessary parameters and session cookie, in + order to complete OIDC login. + """ + # whether to allow a user logging in via OIDC to match a pre-existing account # instead of failing allow_existing_users: bool @@ -450,3 +504,6 @@ class OidcProviderConfig: # Additional parameters that will be passed to the authorization grant URL additional_authorization_parameters: Mapping[str, str] + + # Allow query parameters to the redirect endpoint that will be passed to the authorization grant URL + passthrough_authorization_parameters: Collection[str] diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 3fa33f5373..b082daa8f7 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -228,3 +228,27 @@ class RatelimitConfig(Config): config.get("remote_media_download_burst_count", "500M") ), ) + + self.rc_presence_per_user = RatelimitSettings.parse( + config, + "rc_presence.per_user", + defaults={"per_second": 0.1, "burst_count": 1}, + ) + + self.rc_delayed_event_mgmt = RatelimitSettings.parse( + config, + "rc_delayed_event_mgmt", + defaults={"per_second": 1, "burst_count": 5}, + ) + + self.rc_room_creation = RatelimitSettings.parse( + config, + "rc_room_creation", + defaults={"per_second": 0.016, "burst_count": 10}, + ) + + self.rc_reports = RatelimitSettings.parse( + config, + "rc_reports", + defaults={"per_second": 1, "burst_count": 5}, + ) diff --git a/synapse/config/redis.py b/synapse/config/redis.py index 3f38fa11b0..948c95eef7 100644 --- a/synapse/config/redis.py +++ b/synapse/config/redis.py @@ -34,7 +34,9 @@ These are mutually incompatible. class RedisConfig(Config): section = "redis" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: redis_config = config.get("redis") or {} self.redis_enabled = redis_config.get("enabled", False) @@ -48,6 +50,11 @@ class RedisConfig(Config): self.redis_path = redis_config.get("path", None) self.redis_dbid = redis_config.get("dbid", None) self.redis_password = redis_config.get("password") + if self.redis_password and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("redis", "password"), + ) redis_password_path = redis_config.get("password_path") if redis_password_path: if self.redis_password: diff --git a/synapse/config/registration.py b/synapse/config/registration.py index c7f3e6d35e..283199aa11 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -43,7 +43,9 @@ You have configured both `registration_shared_secret` and class RegistrationConfig(Config): section = "registration" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: self.enable_registration = strtobool( str(config.get("enable_registration", False)) ) @@ -68,6 +70,11 @@ class RegistrationConfig(Config): # read the shared secret, either inline or from an external file self.registration_shared_secret = config.get("registration_shared_secret") + if self.registration_shared_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("registration_shared_secret",), + ) registration_shared_secret_path = config.get("registration_shared_secret_path") if registration_shared_secret_path: if self.registration_shared_secret: @@ -141,20 +148,23 @@ class RegistrationConfig(Config): self.enable_set_displayname = config.get("enable_set_displayname", True) self.enable_set_avatar_url = config.get("enable_set_avatar_url", True) + auth_delegated = (config.get("experimental_features") or {}).get( + "msc3861", {} + ).get("enabled", False) or ( + config.get("matrix_authentication_service") or {} + ).get("enabled", False) + # The default value of enable_3pid_changes is True, unless msc3861 is enabled. - msc3861_enabled = ( - (config.get("experimental_features") or {}) - .get("msc3861", {}) - .get("enabled", False) - ) - self.enable_3pid_changes = config.get( - "enable_3pid_changes", not msc3861_enabled - ) + self.enable_3pid_changes = config.get("enable_3pid_changes", not auth_delegated) self.disable_msisdn_registration = config.get( "disable_msisdn_registration", False ) + self.allow_underscore_prefixed_localpart = config.get( + "allow_underscore_prefixed_localpart", False + ) + session_lifetime = config.get("session_lifetime") if session_lifetime is not None: session_lifetime = self.parse_duration(session_lifetime) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 27860154e1..e7d23740f9 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -22,11 +22,10 @@ import logging import os from typing import Any, Dict, List, Tuple -from urllib.request import getproxies_environment # type: ignore import attr -from synapse.config.server import generate_ip_set +from synapse.config.server import generate_ip_set, parse_proxy_config from synapse.types import JsonDict from synapse.util.check_dependencies import check_requirements from synapse.util.module_loader import load_module @@ -61,7 +60,7 @@ THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP = { "image/png": "png", } -HTTP_PROXY_SET_WARNING = """\ +URL_PREVIEW_BLACKLIST_IGNORED_BECAUSE_HTTP_PROXY_SET_WARNING = """\ The Synapse config url_preview_ip_range_blacklist will be ignored as an HTTP(s) proxy is configured.""" @@ -119,6 +118,23 @@ def parse_thumbnail_requirements( } +@attr.s(auto_attribs=True, slots=True, frozen=True) +class MediaUploadLimit: + """ + Represents a limit on the amount of data a user can upload in a given time + period. + + These can be configured through the `media_upload_limits` [config option](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#media_upload_limits) + or via the `get_media_upload_limits_for_user` module API [callback](https://element-hq.github.io/synapse/latest/modules/media_repository_callbacks.html#get_media_upload_limits_for_user). + """ + + max_bytes: int + """The maximum number of bytes that can be uploaded in the given time period.""" + + time_period_ms: int + """The time period in milliseconds.""" + + class ContentRepositoryConfig(Config): section = "media" @@ -225,17 +241,25 @@ class ContentRepositoryConfig(Config): if self.url_preview_enabled: check_requirements("url-preview") - proxy_env = getproxies_environment() - if "url_preview_ip_range_blacklist" not in config: - if "http" not in proxy_env or "https" not in proxy_env: + proxy_config = parse_proxy_config(config) + is_proxy_configured = ( + proxy_config.http_proxy is not None + or proxy_config.https_proxy is not None + ) + if "url_preview_ip_range_blacklist" in config: + if is_proxy_configured: + logger.warning( + "".join( + URL_PREVIEW_BLACKLIST_IGNORED_BECAUSE_HTTP_PROXY_SET_WARNING + ) + ) + else: + if not is_proxy_configured: raise ConfigError( "For security, you must specify an explicit target IP address " "blacklist in url_preview_ip_range_blacklist for url previewing " "to work" ) - else: - if "http" in proxy_env or "https" in proxy_env: - logger.warning("".join(HTTP_PROXY_SET_WARNING)) # we always block '0.0.0.0' and '::', which are supposed to be # unroutable addresses. @@ -274,6 +298,13 @@ class ContentRepositoryConfig(Config): self.enable_authenticated_media = config.get("enable_authenticated_media", True) + self.media_upload_limits: List[MediaUploadLimit] = [] + for limit_config in config.get("media_upload_limits", []): + time_period_ms = self.parse_duration(limit_config["time_period"]) + max_bytes = self.parse_size(limit_config["max_size"]) + + self.media_upload_limits.append(MediaUploadLimit(max_bytes, time_period_ms)) + def generate_config_section(self, data_dir_path: str, **kwargs: Any) -> str: assert data_dir_path is not None media_store = os.path.join(data_dir_path, "media_store") diff --git a/synapse/config/room.py b/synapse/config/room.py index ec8cf5be36..e698c7bafd 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -27,7 +27,7 @@ from synapse.types import JsonDict from ._base import Config, ConfigError -logger = logging.Logger(__name__) +logger = logging.getLogger(__name__) class RoomDefaultEncryptionTypes: @@ -85,4 +85,4 @@ class RoomConfig(Config): # When enabled, users will forget rooms when they leave them, either via a # leave, kick or ban. - self.forget_on_leave = config.get("forget_rooms_on_leave", False) + self.forget_on_leave: bool = config.get("forget_rooms_on_leave", False) diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py index 704895cf9a..f0349b68f2 100644 --- a/synapse/config/room_directory.py +++ b/synapse/config/room_directory.py @@ -54,9 +54,7 @@ class RoomDirectoryConfig(Config): for rule in room_list_publication_rules ] else: - self._room_list_publication_rules = [ - _RoomDirectoryRule("room_list_publication_rules", {"action": "allow"}) - ] + self._room_list_publication_rules = [] def is_alias_creation_allowed(self, user_id: str, room_id: str, alias: str) -> bool: """Checks if the given user is allowed to create the given alias diff --git a/synapse/config/server.py b/synapse/config/server.py index ad7331de42..e15bceb296 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -25,11 +25,13 @@ import logging import os.path import urllib.parse from textwrap import indent -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, TypedDict, Union +from urllib.request import getproxies_environment import attr import yaml from netaddr import AddrFormatError, IPNetwork, IPSet +from typing_extensions import TypeGuard from twisted.conch.ssh.keys import Key @@ -41,7 +43,22 @@ from synapse.util.stringutils import parse_and_validate_server_name from ._base import Config, ConfigError from ._util import validate_config -logger = logging.Logger(__name__) +logger = logging.getLogger(__name__) + + +# Directly from the mypy docs: +# https://typing.python.org/en/latest/spec/narrowing.html#typeguard +def is_str_list(val: Any, allow_empty: bool) -> TypeGuard[list[str]]: + """ + Type-narrow a value to a list of strings (compatible with mypy). + """ + if not isinstance(val, list): + return False + + if len(val) == 0: + return allow_empty + return all(isinstance(x, str) for x in val) + DIRECT_TCP_ERROR = """ Using direct TCP replication for workers is no longer supported. @@ -291,6 +308,102 @@ class LimitRemoteRoomsConfig: ) +class ProxyConfigDictionary(TypedDict): + """ + Dictionary of proxy settings suitable for interacting with `urllib.request` API's + """ + + http: Optional[str] + """ + Proxy server to use for HTTP requests. + """ + https: Optional[str] + """ + Proxy server to use for HTTPS requests. + """ + no: str + """ + Comma-separated list of hosts, IP addresses, or IP ranges in CIDR format which + should not use the proxy. + + Empty string means no hosts should be excluded from the proxy. + """ + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ProxyConfig: + """ + Synapse configuration for HTTP proxy settings. + """ + + http_proxy: Optional[str] + """ + Proxy server to use for HTTP requests. + """ + https_proxy: Optional[str] + """ + Proxy server to use for HTTPS requests. + """ + no_proxy_hosts: Optional[List[str]] + """ + List of hosts, IP addresses, or IP ranges in CIDR format which should not use the + proxy. Synapse will directly connect to these hosts. + """ + + def get_proxies_dictionary(self) -> ProxyConfigDictionary: + """ + Returns a dictionary of proxy settings suitable for interacting with + `urllib.request` API's (e.g. `urllib.request.proxy_bypass_environment`) + + The keys are `"http"`, `"https"`, and `"no"`. + """ + return ProxyConfigDictionary( + http=self.http_proxy, + https=self.https_proxy, + no=",".join(self.no_proxy_hosts) if self.no_proxy_hosts else "", + ) + + +def parse_proxy_config(config: JsonDict) -> ProxyConfig: + """ + Figure out forward proxy config for outgoing HTTP requests. + + Prefer values from the given config over the environment variables (`http_proxy`, + `https_proxy`, `no_proxy`, not case-sensitive). + + Args: + config: The top-level homeserver configuration dictionary. + """ + proxies_from_env = getproxies_environment() + http_proxy = config.get("http_proxy", proxies_from_env.get("http")) + if http_proxy is not None and not isinstance(http_proxy, str): + raise ConfigError("'http_proxy' must be a string", ("http_proxy",)) + + https_proxy = config.get("https_proxy", proxies_from_env.get("https")) + if https_proxy is not None and not isinstance(https_proxy, str): + raise ConfigError("'https_proxy' must be a string", ("https_proxy",)) + + # List of hosts which should not use the proxy. Synapse will directly connect to + # these hosts. + no_proxy_hosts = config.get("no_proxy_hosts") + # The `no_proxy` environment variable should be a comma-separated list of hosts, + # IP addresses, or IP ranges in CIDR format + no_proxy_from_env = proxies_from_env.get("no") + if no_proxy_hosts is None and no_proxy_from_env is not None: + no_proxy_hosts = no_proxy_from_env.split(",") + + if no_proxy_hosts is not None and not is_str_list(no_proxy_hosts, allow_empty=True): + raise ConfigError( + "'no_proxy_hosts' must be a list of strings", ("no_proxy_hosts",) + ) + + return ProxyConfig( + http_proxy=http_proxy, + https_proxy=https_proxy, + no_proxy_hosts=no_proxy_hosts, + ) + + class ServerConfig(Config): section = "server" @@ -332,8 +445,14 @@ class ServerConfig(Config): logger.info("Using default public_baseurl %s", public_baseurl) else: self.serve_client_wellknown = True + # Ensure that public_baseurl ends with a trailing slash if public_baseurl[-1] != "/": public_baseurl += "/" + + # Scrutinize user-provided config + if not isinstance(public_baseurl, str): + raise ConfigError("Must be a string", ("public_baseurl",)) + self.public_baseurl = public_baseurl # check that public_baseurl is valid @@ -712,6 +831,17 @@ class ServerConfig(Config): ) ) + # Figure out forward proxy config for outgoing HTTP requests. + # + # Prefer values from the file config over the environment variables + self.proxy_config = parse_proxy_config(config) + logger.debug( + "Using proxy settings: http_proxy=%s, https_proxy=%s, no_proxy=%s", + self.proxy_config.http_proxy, + self.proxy_config.https_proxy, + self.proxy_config.no_proxy_hosts, + ) + self.cleanup_extremities_with_dummy_events = config.get( "cleanup_extremities_with_dummy_events", True ) diff --git a/synapse/config/sso.py b/synapse/config/sso.py index d7a2187e7d..cf27a7ee13 100644 --- a/synapse/config/sso.py +++ b/synapse/config/sso.py @@ -19,7 +19,7 @@ # # import logging -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import attr @@ -43,13 +43,18 @@ class SsoAttributeRequirement: """Object describing a single requirement for SSO attributes.""" attribute: str - # If a value is not given, than the attribute must simply exist. - value: Optional[str] + # If neither `value` nor `one_of` is given, the attribute must simply exist. + value: Optional[str] = None + one_of: Optional[List[str]] = None JSON_SCHEMA = { "type": "object", - "properties": {"attribute": {"type": "string"}, "value": {"type": "string"}}, - "required": ["attribute", "value"], + "properties": { + "attribute": {"type": "string"}, + "value": {"type": "string"}, + "one_of": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["attribute"], } diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 51dc15eb61..a48d81fdc3 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -108,8 +108,7 @@ class TlsConfig(Config): # Raise an error if this option has been specified without any # corresponding certificates. raise ConfigError( - "federation_custom_ca_list specified without " - "any certificate files" + "federation_custom_ca_list specified without any certificate files" ) certs = [] diff --git a/synapse/config/user_directory.py b/synapse/config/user_directory.py index c67796906f..fe4e2dc65c 100644 --- a/synapse/config/user_directory.py +++ b/synapse/config/user_directory.py @@ -38,6 +38,9 @@ class UserDirectoryConfig(Config): self.user_directory_search_all_users = user_directory_config.get( "search_all_users", False ) + self.user_directory_exclude_remote_users = user_directory_config.get( + "exclude_remote_users", False + ) self.user_directory_search_prefer_local_users = user_directory_config.get( "prefer_local_users", False ) diff --git a/synapse/config/user_types.py b/synapse/config/user_types.py new file mode 100644 index 0000000000..2d9c9f7afb --- /dev/null +++ b/synapse/config/user_types.py @@ -0,0 +1,44 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import Any, List, Optional + +from synapse.api.constants import UserTypes +from synapse.types import JsonDict + +from ._base import Config, ConfigError + + +class UserTypesConfig(Config): + section = "user_types" + + def read_config(self, config: JsonDict, **kwargs: Any) -> None: + user_types: JsonDict = config.get("user_types", {}) + + self.default_user_type: Optional[str] = user_types.get( + "default_user_type", None + ) + self.extra_user_types: List[str] = user_types.get("extra_user_types", []) + + all_user_types: List[str] = [] + all_user_types.extend(UserTypes.ALL_BUILTIN_USER_TYPES) + all_user_types.extend(self.extra_user_types) + + self.all_user_types = all_user_types + + if self.default_user_type is not None: + if self.default_user_type not in all_user_types: + raise ConfigError( + f"Default user type {self.default_user_type} is not in the list of all user types: {all_user_types}" + ) diff --git a/synapse/config/voip.py b/synapse/config/voip.py index 8614a41dd4..f33602d975 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -34,9 +34,16 @@ These are mutually incompatible. class VoipConfig(Config): section = "voip" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: self.turn_uris = config.get("turn_uris", []) self.turn_shared_secret = config.get("turn_shared_secret") + if self.turn_shared_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("turn_shared_secret",), + ) turn_shared_secret_path = config.get("turn_shared_secret_path") if turn_shared_secret_path: if self.turn_shared_secret: diff --git a/synapse/config/workers.py b/synapse/config/workers.py index ab896be307..825ba78482 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -27,8 +27,6 @@ from typing import Any, Dict, List, Optional, Union import attr from synapse._pydantic_compat import ( - BaseModel, - Extra, StrictBool, StrictInt, StrictStr, @@ -38,6 +36,7 @@ from synapse.config._base import ( ConfigError, RoutableShardedWorkerHandlingConfig, ShardedWorkerHandlingConfig, + read_file, ) from synapse.config._util import parse_and_validate_mapping from synapse.config.server import ( @@ -46,6 +45,7 @@ from synapse.config.server import ( parse_listener_def, ) from synapse.types import JsonDict +from synapse.util.pydantic_models import ParseModel _DEPRECATED_WORKER_DUTY_OPTION_USED = """ The '%s' configuration option is deprecated and will be removed in a future @@ -65,6 +65,11 @@ configuration under `main` inside the `instance_map`. See workers documentation `https://element-hq.github.io/synapse/latest/workers.html#worker-configuration` """ +CONFLICTING_WORKER_REPLICATION_SECRET_OPTS_ERROR = """\ +Conflicting options 'worker_replication_secret' and +'worker_replication_secret_path' are both defined in config file. +""" + # This allows for a handy knob when it's time to change from 'master' to # something with less 'history' MAIN_PROCESS_INSTANCE_NAME = "master" @@ -84,30 +89,7 @@ def _instance_to_list_converter(obj: Union[str, List[str]]) -> List[str]: return obj -class ConfigModel(BaseModel): - """A custom version of Pydantic's BaseModel which - - - ignores unknown fields and - - does not allow fields to be overwritten after construction, - - but otherwise uses Pydantic's default behaviour. - - For now, ignore unknown fields. In the future, we could change this so that unknown - config values cause a ValidationError, provided the error messages are meaningful to - server operators. - - Subclassing in this way is recommended by - https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally - """ - - class Config: - # By default, ignore fields that we don't recognise. - extra = Extra.ignore - # By default, don't allow fields to be reassigned after parsing. - allow_mutation = False - - -class InstanceTcpLocationConfig(ConfigModel): +class InstanceTcpLocationConfig(ParseModel): """The host and port to talk to an instance via HTTP replication.""" host: StrictStr @@ -123,7 +105,7 @@ class InstanceTcpLocationConfig(ConfigModel): return f"{self.host}:{self.port}" -class InstanceUnixLocationConfig(ConfigModel): +class InstanceUnixLocationConfig(ParseModel): """The socket file to talk to an instance via HTTP replication.""" path: StrictStr @@ -152,39 +134,47 @@ class WriterLocations: can only be a single instance. account_data: The instances that write to the account data streams. Currently can only be a single instance. - receipts: The instances that write to the receipts stream. Currently - can only be a single instance. + receipts: The instances that write to the receipts stream. presence: The instances that write to the presence stream. Currently can only be a single instance. push_rules: The instances that write to the push stream. Currently can only be a single instance. + device_lists: The instances that write to the device list stream. """ events: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) typing: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) to_device: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) account_data: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) receipts: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) presence: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) push_rules: List[str] = attr.ib( + default=[MAIN_PROCESS_INSTANCE_NAME], + converter=_instance_to_list_converter, + ) + device_lists: List[str] = attr.ib( + default=[MAIN_PROCESS_INSTANCE_NAME], + converter=_instance_to_list_converter, + ) + thread_subscriptions: List[str] = attr.ib( default=["master"], converter=_instance_to_list_converter, ) @@ -218,7 +208,9 @@ class WorkerConfig(Config): section = "worker" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: self.worker_app = config.get("worker_app") # Canonicalise worker_app so that master always has None @@ -242,7 +234,29 @@ class WorkerConfig(Config): raise ConfigError(DIRECT_TCP_ERROR, ("worker_replication_port",)) # The shared secret used for authentication when connecting to the main synapse. - self.worker_replication_secret = config.get("worker_replication_secret", None) + worker_replication_secret = config.get("worker_replication_secret", None) + if worker_replication_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("worker_replication_secret",), + ) + worker_replication_secret_path = config.get( + "worker_replication_secret_path", None + ) + if worker_replication_secret_path: + if worker_replication_secret: + raise ConfigError(CONFLICTING_WORKER_REPLICATION_SECRET_OPTS_ERROR) + self.worker_replication_secret: Optional[str] = read_file( + worker_replication_secret_path, ("worker_replication_secret_path",) + ).strip() + else: + if worker_replication_secret is not None and not isinstance( + worker_replication_secret, str + ): + raise ConfigError( + "Config option must be a string", ("worker_replication_secret",) + ) + self.worker_replication_secret = worker_replication_secret self.worker_name = config.get("worker_name", self.worker_app) self.instance_name = self.worker_name or MAIN_PROCESS_INSTANCE_NAME @@ -352,7 +366,10 @@ class WorkerConfig(Config): ): instances = _instance_to_list_converter(getattr(self.writers, stream)) for instance in instances: - if instance != "master" and instance not in self.instance_map: + if ( + instance != MAIN_PROCESS_INSTANCE_NAME + and instance not in self.instance_map + ): raise ConfigError( "Instance %r is configured to write %s but does not appear in `instance_map` config." % (instance, stream) @@ -391,6 +408,11 @@ class WorkerConfig(Config): "Must only specify one instance to handle `push` messages." ) + if len(self.writers.device_lists) == 0: + raise ConfigError( + "Must specify at least one instance to handle `device_lists` messages." + ) + self.events_shard_config = RoutableShardedWorkerHandlingConfig( self.writers.events ) @@ -413,9 +435,12 @@ class WorkerConfig(Config): # # No effort is made to ensure only a single instance of these tasks is # running. - background_tasks_instance = config.get("run_background_tasks_on") or "master" + background_tasks_instance = ( + config.get("run_background_tasks_on") or MAIN_PROCESS_INSTANCE_NAME + ) self.run_background_tasks = ( - self.worker_name is None and background_tasks_instance == "master" + self.worker_name is None + and background_tasks_instance == MAIN_PROCESS_INSTANCE_NAME ) or self.worker_name == background_tasks_instance self.should_notify_appservices = self._should_this_worker_perform_duty( @@ -487,9 +512,10 @@ class WorkerConfig(Config): # 'don't run here'. new_option_should_run_here = None if new_option_name in config: - designated_worker = config[new_option_name] or "master" + designated_worker = config[new_option_name] or MAIN_PROCESS_INSTANCE_NAME new_option_should_run_here = ( - designated_worker == "master" and self.worker_name is None + designated_worker == MAIN_PROCESS_INSTANCE_NAME + and self.worker_name is None ) or designated_worker == self.worker_name legacy_option_should_run_here = None @@ -586,7 +612,7 @@ class WorkerConfig(Config): # If no worker instances are set we check if the legacy option # is set, which means use the main process. if legacy_option: - worker_instances = ["master"] + worker_instances = [MAIN_PROCESS_INSTANCE_NAME] if self.worker_app == legacy_app_name: if legacy_option: diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index b85df1ce42..c36398cec0 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -101,6 +101,9 @@ def compute_content_hash( event_dict.pop("outlier", None) event_dict.pop("destinations", None) + # N.B. no need to pop the room_id from create events in MSC4291 rooms + # as they shouldn't have one. + event_json_bytes = encode_canonical_json(event_dict) hashed = hash_algorithm(event_json_bytes) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 643d2d4e66..8c59772e56 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -152,6 +152,8 @@ class Keyring: def __init__( self, hs: "HomeServer", key_fetchers: "Optional[Iterable[KeyFetcher]]" = None ): + self.server_name = hs.hostname + if key_fetchers is None: # Always fetch keys from the database. mutable_key_fetchers: List[KeyFetcher] = [StoreKeyFetcher(hs)] @@ -169,7 +171,8 @@ class Keyring: self._fetch_keys_queue: BatchingQueue[ _FetchKeyRequest, Dict[str, Dict[str, FetchKeyResult]] ] = BatchingQueue( - "keyring_server", + name="keyring_server", + server_name=self.server_name, clock=hs.get_clock(), # The method called to fetch each key process_batch_callback=self._inner_fetch_key_requests, @@ -473,8 +476,12 @@ class Keyring: class KeyFetcher(metaclass=abc.ABCMeta): def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._queue = BatchingQueue( - self.__class__.__name__, hs.get_clock(), self._fetch_keys + name=self.__class__.__name__, + server_name=self.server_name, + clock=hs.get_clock(), + process_batch_callback=self._fetch_keys, ) async def get_keys( diff --git a/synapse/event_auth.py b/synapse/event_auth.py index c208b900c5..64de3f7ef8 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -32,6 +32,7 @@ from typing import ( Mapping, MutableMapping, Optional, + Protocol, Set, Tuple, Union, @@ -41,10 +42,10 @@ from typing import ( from canonicaljson import encode_canonical_json from signedjson.key import decode_verify_key_bytes from signedjson.sign import SignatureVerifyException, verify_signed_json -from typing_extensions import Protocol from unpaddedbase64 import decode_base64 from synapse.api.constants import ( + CREATOR_POWER_LEVEL, MAX_PDU_SIZE, EventContentFields, EventTypes, @@ -64,6 +65,8 @@ from synapse.api.room_versions import ( RoomVersion, RoomVersions, ) +from synapse.events import is_creator +from synapse.state import CREATE_KEY from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( MutableStateMap, @@ -260,7 +263,8 @@ async def check_state_independent_auth_rules( f"Event {event.event_id} has unexpected auth_event for {k}: {auth_event_id}", ) - # We also need to check that the auth event itself is not rejected. + # 2.3 ... If there are entries which were themselves rejected under the checks performed on receipt + # of a PDU, reject. if auth_event.rejected_reason: raise AuthError( 403, @@ -270,7 +274,7 @@ async def check_state_independent_auth_rules( auth_dict[k] = auth_event_id - # 3. If event does not have a m.room.create in its auth_events, reject. + # 2.4. If event does not have a m.room.create in its auth_events, reject. creation_event = auth_dict.get((EventTypes.Create, ""), None) if not creation_event: raise AuthError(403, "No create event in auth events") @@ -308,8 +312,16 @@ def check_state_dependent_auth_rules( auth_dict = {(e.type, e.state_key): e for e in auth_events} + # Later code relies on there being a create event e.g _can_federate, _is_membership_change_allowed + # so produce a more intelligible error if we don't have one. + create_event = auth_dict.get(CREATE_KEY) + if create_event is None: + raise AuthError( + 403, f"Event {event.event_id} is missing a create event in auth_events." + ) + # additional check for m.federate - creating_domain = get_domain_from_id(event.room_id) + creating_domain = get_domain_from_id(create_event.sender) originating_domain = get_domain_from_id(event.sender) if creating_domain != originating_domain: if not _can_federate(event, auth_dict): @@ -462,12 +474,20 @@ def _check_create(event: "EventBase") -> None: if event.prev_event_ids(): raise AuthError(403, "Create event has prev events") - # 1.2 If the domain of the room_id does not match the domain of the sender, - # reject. - sender_domain = get_domain_from_id(event.sender) - room_id_domain = get_domain_from_id(event.room_id) - if room_id_domain != sender_domain: - raise AuthError(403, "Creation event's room_id domain does not match sender's") + if event.room_version.msc4291_room_ids_as_hashes: + # 1.2 If the create event has a room_id, reject + if "room_id" in event: + raise AuthError(403, "Create event has a room_id") + else: + # 1.2 If the domain of the room_id does not match the domain of the sender, + # reject. + if not event.room_version.msc4291_room_ids_as_hashes: + sender_domain = get_domain_from_id(event.sender) + room_id_domain = get_domain_from_id(event.room_id) + if room_id_domain != sender_domain: + raise AuthError( + 403, "Creation event's room_id domain does not match sender's" + ) # 1.3 If content.room_version is present and is not a recognised version, reject room_version_prop = event.content.get("room_version", "1") @@ -484,6 +504,16 @@ def _check_create(event: "EventBase") -> None: ): raise AuthError(403, "Create event lacks a 'creator' property") + # 1.5 If the additional_creators field is present and is not an array of strings where each + # string is a valid user ID, reject. + if ( + event.room_version.msc4289_creator_power_enabled + and EventContentFields.ADDITIONAL_CREATORS in event.content + ): + check_valid_additional_creators( + event.content[EventContentFields.ADDITIONAL_CREATORS] + ) + def _can_federate(event: "EventBase", auth_events: StateMap["EventBase"]) -> bool: creation_event = auth_events.get((EventTypes.Create, "")) @@ -525,7 +555,13 @@ def _is_membership_change_allowed( target_user_id = event.state_key - creating_domain = get_domain_from_id(event.room_id) + # We need the create event in order to check if we can federate or not. + # If it's missing, yell loudly. Previously we only did this inside the + # _can_federate check. + create_event = auth_events.get((EventTypes.Create, "")) + if not create_event: + raise AuthError(403, "Create event missing from auth_events") + creating_domain = get_domain_from_id(create_event.sender) target_domain = get_domain_from_id(target_user_id) if creating_domain != target_domain: if not _can_federate(event, auth_events): @@ -566,6 +602,7 @@ def _is_membership_change_allowed( logger.debug( "_is_membership_change_allowed: %s", { + "caller_membership": caller.membership if caller else None, "caller_in_room": caller_in_room, "caller_invited": caller_invited, "caller_knocked": caller_knocked, @@ -677,7 +714,8 @@ def _is_membership_change_allowed( and join_rule == JoinRules.KNOCK_RESTRICTED ) ): - if not caller_in_room and not caller_invited: + # You can only join the room if you are invited or are already in the room. + if not (caller_in_room or caller_invited): raise AuthError(403, "You are not invited to this room.") else: # TODO (erikj): may_join list @@ -893,6 +931,32 @@ def _check_power_levels( except Exception: raise SynapseError(400, "Not a valid power level: %s" % (v,)) + if room_version_obj.msc4289_creator_power_enabled: + # Enforce the creator does not appear in the users map + create_event = auth_events.get((EventTypes.Create, "")) + if not create_event: + raise SynapseError( + 400, "Cannot check power levels without a create event in auth_events" + ) + if create_event.sender in user_list: + raise SynapseError( + 400, + "Creator user %s must not appear in content.users" + % (create_event.sender,), + ) + additional_creators = create_event.content.get( + EventContentFields.ADDITIONAL_CREATORS, [] + ) + if additional_creators: + creators_in_user_list = set(additional_creators).intersection( + set(user_list) + ) + if len(creators_in_user_list) > 0: + raise SynapseError( + 400, + "Additional creators users must not appear in content.users", + ) + # Reject events with stringy power levels if required by room version if ( event.type == EventTypes.PowerLevels @@ -984,8 +1048,7 @@ def _check_power_levels( if old_level == user_level: raise AuthError( 403, - "You don't have permission to remove ops level equal " - "to your own", + "You don't have permission to remove ops level equal to your own", ) # Check if the old and new levels are greater than the user level @@ -1009,11 +1072,19 @@ def get_user_power_level(user_id: str, auth_events: StateMap["EventBase"]) -> in user_id: user's id to look up in power_levels auth_events: state in force at this point in the room (or rather, a subset of - it including at least the create event and power levels event. + it including at least the create event, and possibly a power levels event). Returns: the user's power level in this room. """ + create_event = auth_events.get(CREATE_KEY) + assert create_event is not None, ( + "A create event in the auth events chain is required to calculate user power level correctly," + " but was not found. This indicates a bug" + ) + if create_event.room_version.msc4289_creator_power_enabled: + if is_creator(create_event, user_id): + return CREATOR_POWER_LEVEL power_level_event = get_power_level_event(auth_events) if power_level_event: level = power_level_event.content.get("users", {}).get(user_id) @@ -1027,18 +1098,12 @@ def get_user_power_level(user_id: str, auth_events: StateMap["EventBase"]) -> in else: # if there is no power levels event, the creator gets 100 and everyone # else gets 0. - - # some things which call this don't pass the create event: hack around - # that. - key = (EventTypes.Create, "") - create_event = auth_events.get(key) - if create_event is not None: - if create_event.room_version.implicit_room_creator: - creator = create_event.sender - else: - creator = create_event.content[EventContentFields.ROOM_CREATOR] - if creator == user_id: - return 100 + if create_event.room_version.implicit_room_creator: + creator = create_event.sender + else: + creator = create_event.content[EventContentFields.ROOM_CREATOR] + if creator == user_id: + return 100 return 0 @@ -1180,3 +1245,26 @@ def auth_types_for_event( auth_types.add(key) return auth_types + + +def check_valid_additional_creators(additional_creators: Any) -> None: + """Check if the additional_creators provided is valid according to MSC4289. + + The additional_creators can be supplied from an m.room.create event or from an /upgrade request. + + Raises: + AuthError if the additional_creators is invalid for some reason. + """ + if type(additional_creators) is not list: + raise AuthError(400, "additional_creators must be an array") + for entry in additional_creators: + if type(entry) is not str: + raise AuthError(400, "entry in additional_creators is not a string") + if not UserID.is_valid(entry): + raise AuthError(400, "entry in additional_creators is not a valid user ID") + # UserID.is_valid doesn't actually validate everything, so check the rest manually. + if len(entry) > 255 or len(entry.encode("utf-8")) > 255: + raise AuthError( + 400, + "entry in additional_creators too long", + ) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 2e56b671f0..db38754280 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -22,7 +22,6 @@ import abc import collections.abc -import os from typing import ( TYPE_CHECKING, Any, @@ -30,6 +29,7 @@ from typing import ( Generic, Iterable, List, + Literal, Optional, Tuple, Type, @@ -39,30 +39,32 @@ from typing import ( ) import attr -from typing_extensions import Literal from unpaddedbase64 import encode_base64 -from synapse.api.constants import RelationTypes +from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions from synapse.synapse_rust.events import EventInternalMetadata -from synapse.types import JsonDict, StrCollection +from synapse.types import ( + JsonDict, + StrCollection, +) from synapse.util.caches import intern_dict from synapse.util.frozenutils import freeze -from synapse.util.stringutils import strtobool if TYPE_CHECKING: from synapse.events.builder import EventBuilder -# Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents -# bugs where we accidentally share e.g. signature dicts. However, converting a -# dict to frozen_dicts is expensive. -# -# NOTE: This is overridden by the configuration by the Synapse worker apps, but -# for the sake of tests, it is set here while it cannot be configured on the -# homeserver object itself. -USE_FROZEN_DICTS = strtobool(os.environ.get("SYNAPSE_USE_FROZEN_DICTS", "0")) +USE_FROZEN_DICTS = False +""" +Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents +bugs where we accidentally share e.g. signature dicts. However, converting a +dict to frozen_dicts is expensive. +NOTE: This is overridden by the configuration by the Synapse worker apps, but +for the sake of tests, it is set here because it cannot be configured on the +homeserver object itself. +""" T = TypeVar("T") @@ -209,9 +211,7 @@ class EventBase(metaclass=abc.ABCMeta): depth: DictProperty[int] = DictProperty("depth") content: DictProperty[JsonDict] = DictProperty("content") hashes: DictProperty[Dict[str, str]] = DictProperty("hashes") - origin: DictProperty[str] = DictProperty("origin") origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts") - room_id: DictProperty[str] = DictProperty("room_id") sender: DictProperty[str] = DictProperty("sender") # TODO state_key should be Optional[str]. This is generally asserted in Synapse # by calling is_state() first (which ensures it is not None), but it is hard (not possible?) @@ -226,6 +226,10 @@ class EventBase(metaclass=abc.ABCMeta): def event_id(self) -> str: raise NotImplementedError() + @property + def room_id(self) -> str: + raise NotImplementedError() + @property def membership(self) -> str: return self.content["membership"] @@ -325,12 +329,17 @@ class EventBase(metaclass=abc.ABCMeta): def __repr__(self) -> str: rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else "" + conditional_membership_string = "" + if self.get("type") == EventTypes.Member: + conditional_membership_string = f"membership={self.membership}, " + return ( f"<{self.__class__.__name__} " f"{rejection}" f"event_id={self.event_id}, " f"type={self.get('type')}, " f"state_key={self.get('state_key')}, " + f"{conditional_membership_string}" f"outlier={self.internal_metadata.is_outlier()}" ">" ) @@ -383,6 +392,10 @@ class FrozenEvent(EventBase): def event_id(self) -> str: return self._event_id + @property + def room_id(self) -> str: + return self._dict["room_id"] + class FrozenEventV2(EventBase): format_version = EventFormatVersions.ROOM_V3 # All events of this type are V2 @@ -440,6 +453,10 @@ class FrozenEventV2(EventBase): self._event_id = "$" + encode_base64(compute_event_reference_hash(self)[1]) return self._event_id + @property + def room_id(self) -> str: + return self._dict["room_id"] + def prev_event_ids(self) -> List[str]: """Returns the list of prev event IDs. The order matches the order specified in the event, though there is no meaning to it. @@ -478,6 +495,67 @@ class FrozenEventV3(FrozenEventV2): return self._event_id +class FrozenEventV4(FrozenEventV3): + """FrozenEventV4 for MSC4291 room IDs are hashes""" + + format_version = EventFormatVersions.ROOM_V11_HYDRA_PLUS + + """Override the room_id for m.room.create events""" + + def __init__( + self, + event_dict: JsonDict, + room_version: RoomVersion, + internal_metadata_dict: Optional[JsonDict] = None, + rejected_reason: Optional[str] = None, + ): + super().__init__( + event_dict=event_dict, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, + ) + self._room_id: Optional[str] = None + + @property + def room_id(self) -> str: + # if we have calculated the room ID already, don't do it again. + if self._room_id: + return self._room_id + + is_create_event = self.type == EventTypes.Create and self.get_state_key() == "" + + # for non-create events: use the supplied value from the JSON, as per FrozenEventV3 + if not is_create_event: + self._room_id = self._dict["room_id"] + assert self._room_id is not None + return self._room_id + + # for create events: calculate the room ID + from synapse.crypto.event_signing import compute_event_reference_hash + + self._room_id = "!" + encode_base64( + compute_event_reference_hash(self)[1], urlsafe=True + ) + return self._room_id + + def auth_event_ids(self) -> StrCollection: + """Returns the list of auth event IDs. The order matches the order + specified in the event, though there is no meaning to it. + Returns: + The list of event IDs of this event's auth_events + Includes the creation event ID for convenience of all the codepaths + which expects the auth chain to include the creator ID, even though + it's explicitly not included on the wire. Excludes the create event + for the create event itself. + """ + create_event_id = "$" + self.room_id[1:] + assert create_event_id not in self._dict["auth_events"] + if self.type == EventTypes.Create and self.get_state_key() == "": + return self._dict["auth_events"] # should be [] + return self._dict["auth_events"] + [create_event_id] + + def _event_type_from_format_version( format_version: int, ) -> Type[Union[FrozenEvent, FrozenEventV2, FrozenEventV3]]: @@ -497,6 +575,8 @@ def _event_type_from_format_version( return FrozenEventV2 elif format_version == EventFormatVersions.ROOM_V4_PLUS: return FrozenEventV3 + elif format_version == EventFormatVersions.ROOM_V11_HYDRA_PLUS: + return FrozenEventV4 else: raise Exception("No event format %r" % (format_version,)) @@ -556,6 +636,23 @@ def relation_from_event(event: EventBase) -> Optional[_EventRelation]: return _EventRelation(parent_id, rel_type, aggregation_key) +def is_creator(create: EventBase, user_id: str) -> bool: + """ + Return true if the provided user ID is the room creator. + + This includes additional creators in MSC4289. + """ + assert create.type == EventTypes.Create + if create.sender == user_id: + return True + if create.room_version.msc4289_creator_power_enabled: + additional_creators = set( + create.content.get(EventContentFields.ADDITIONAL_CREATORS, []) + ) + return user_id in additional_creators + return False + + @attr.s(slots=True, frozen=True, auto_attribs=True) class StrippedStateEvent: """ diff --git a/synapse/events/auto_accept_invites.py b/synapse/events/auto_accept_invites.py index d88ec51d9d..6873ee9d31 100644 --- a/synapse/events/auto_accept_invites.py +++ b/synapse/events/auto_accept_invites.py @@ -34,6 +34,7 @@ class InviteAutoAccepter: def __init__(self, config: AutoAcceptInvitesConfig, api: ModuleApi): # Keep a reference to the Module API. self._api = api + self.server_name = api.server_name self._config = config if not self._config.enabled: @@ -66,50 +67,67 @@ class InviteAutoAccepter: event: The incoming event. """ # Check if the event is an invite for a local user. - is_invite_for_local_user = ( - event.type == EventTypes.Member - and event.is_state() - and event.membership == Membership.INVITE - and self._api.is_mine(event.state_key) - ) + if ( + event.type != EventTypes.Member + or event.is_state() is False + or event.membership != Membership.INVITE + or self._api.is_mine(event.state_key) is False + ): + return # Only accept invites for direct messages if the configuration mandates it. is_direct_message = event.content.get("is_direct", False) - is_allowed_by_direct_message_rules = ( - not self._config.accept_invites_only_for_direct_messages - or is_direct_message is True - ) + if ( + self._config.accept_invites_only_for_direct_messages + and is_direct_message is False + ): + return # Only accept invites from remote users if the configuration mandates it. is_from_local_user = self._api.is_mine(event.sender) - is_allowed_by_local_user_rules = ( - not self._config.accept_invites_only_from_local_users - or is_from_local_user is True + if ( + self._config.accept_invites_only_from_local_users + and is_from_local_user is False + ): + return + + # Check the user is activated. + recipient = await self._api.get_userinfo_by_id(event.state_key) + + # Ignore if the user doesn't exist. + if recipient is None: + return + + # Never accept invites for deactivated users. + if recipient.is_deactivated: + return + + # Never accept invites for suspended users. + if recipient.suspended: + return + + # Never accept invites for locked users. + if recipient.locked: + return + + # Make the user join the room. We run this as a background process to circumvent a race condition + # that occurs when responding to invites over federation (see https://github.com/matrix-org/synapse-auto-accept-invite/issues/12) + run_as_background_process( + "retry_make_join", + self._retry_make_join, + event.state_key, + event.state_key, + event.room_id, + "join", + bg_start_span=False, ) - if ( - is_invite_for_local_user - and is_allowed_by_direct_message_rules - and is_allowed_by_local_user_rules - ): - # Make the user join the room. We run this as a background process to circumvent a race condition - # that occurs when responding to invites over federation (see https://github.com/matrix-org/synapse-auto-accept-invite/issues/12) - run_as_background_process( - "retry_make_join", - self._retry_make_join, - event.state_key, - event.state_key, - event.room_id, - "join", - bg_start_span=False, + if is_direct_message: + # Mark this room as a direct message! + await self._mark_room_as_direct_message( + event.state_key, event.sender, event.room_id ) - if is_direct_message: - # Mark this room as a direct message! - await self._mark_room_as_direct_message( - event.state_key, event.sender, event.room_id - ) - async def _mark_room_as_direct_message( self, user_id: str, dm_user_id: str, room_id: str ) -> None: @@ -178,15 +196,18 @@ class InviteAutoAccepter: except SynapseError as e: if e.code == HTTPStatus.FORBIDDEN: logger.debug( - f"Update_room_membership was forbidden. This can sometimes be expected for remote invites. Exception: {e}" + "Update_room_membership was forbidden. This can sometimes be expected for remote invites. Exception: %s", + e, ) else: - logger.warn( - f"Update_room_membership raised the following unexpected (SynapseError) exception: {e}" + logger.warning( + "Update_room_membership raised the following unexpected (SynapseError) exception: %s", + e, ) except Exception as e: - logger.warn( - f"Update_room_membership raised the following unexpected exception: {e}" + logger.warning( + "Update_room_membership raised the following unexpected exception: %s", + e, ) sleep = 2**retries diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 10ef01131b..5e1913d389 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import attr from signedjson.types import SigningKey -from synapse.api.constants import MAX_DEPTH +from synapse.api.constants import MAX_DEPTH, EventTypes from synapse.api.room_versions import ( KNOWN_EVENT_FORMAT_VERSIONS, EventFormatVersions, @@ -82,7 +82,8 @@ class EventBuilder: room_version: RoomVersion - room_id: str + # MSC4291 makes the room ID == the create event ID. This means the create event has no room_id. + room_id: Optional[str] type: str sender: str @@ -109,6 +110,19 @@ class EventBuilder: def is_state(self) -> bool: return self._state_key is not None + def is_mine_id(self, user_id: str) -> bool: + """Determines whether a user ID or room alias originates from this homeserver. + + Returns: + `True` if the hostname part of the user ID or room alias matches this + homeserver. + `False` otherwise, or if the user ID or room alias is malformed. + """ + localpart_hostname = user_id.split(":", 1) + if len(localpart_hostname) < 2: + return False + return localpart_hostname[1] == self._hostname + async def build( self, prev_event_ids: List[str], @@ -129,7 +143,14 @@ class EventBuilder: Returns: The signed and hashed event. """ + # Create events always have empty auth_events. + if self.type == EventTypes.Create and self.is_state() and self.state_key == "": + auth_event_ids = [] + + # Calculate auth_events for non-create events if auth_event_ids is None: + # Every non-create event must have a room ID + assert self.room_id is not None state_ids = await self._state.compute_state_after_events( self.room_id, prev_event_ids, @@ -142,6 +163,46 @@ class EventBuilder: self, state_ids ) + # Check for out-of-band membership that may have been exposed on `/sync` but + # the events have not been de-outliered yet so they won't be part of the + # room state yet. + # + # This helps in situations where a remote homeserver invites a local user to + # a room that we're already participating in; and we've persisted the invite + # as an out-of-band membership (outlier), but it hasn't been pushed to us as + # part of a `/send` transaction yet and de-outliered. This also helps for + # any of the other out-of-band membership transitions. + # + # As an optimization, we could check if the room state already includes a + # non-`leave` membership event, then we can assume the membership event has + # been de-outliered and we don't need to check for an out-of-band + # membership. But we don't have the necessary information from a + # `StateMap[str]` and we'll just have to take the hit of this extra lookup + # for any membership event for now. + if self.type == EventTypes.Member and self.is_mine_id(self.state_key): + ( + _membership, + member_event_id, + ) = await self._store.get_local_current_membership_for_user_in_room( + user_id=self.state_key, + room_id=self.room_id, + ) + # There is no need to check if the membership is actually an + # out-of-band membership (`outlier`) as we would end up with the + # same result either way (adding the member event to the + # `auth_event_ids`). + if ( + member_event_id is not None + # We only need to be careful about duplicating the event in the + # `auth_event_ids` list (duplicate `type`/`state_key` is part of the + # authorization rules) + and member_event_id not in auth_event_ids + ): + auth_event_ids.append(member_event_id) + # Also make sure to point to the previous membership event that will + # allow this one to happen so the computed state works out. + prev_event_ids.append(member_event_id) + format_version = self.room_version.event_format # The types of auth/prev events changes between event versions. prev_events: Union[StrCollection, List[Tuple[str, Dict[str, str]]]] @@ -171,12 +232,31 @@ class EventBuilder: "auth_events": auth_events, "prev_events": prev_events, "type": self.type, - "room_id": self.room_id, "sender": self.sender, "content": self.content, "unsigned": self.unsigned, "depth": depth, } + if self.room_id is not None: + event_dict["room_id"] = self.room_id + + if self.room_version.msc4291_room_ids_as_hashes: + # In MSC4291: the create event has no room ID as the create event ID /is/ the room ID. + if ( + self.type == EventTypes.Create + and self.is_state() + and self._state_key == "" + ): + assert self.room_id is None + else: + # All other events do not reference the create event in auth_events, as the room ID + # /is/ the create event. However, the rest of the code (for consistency between room + # versions) assume that the create event remains part of the auth events. c.f. event + # class which automatically adds the create event when `.auth_event_ids()` is called + assert self.room_id is not None + create_event_id = "$" + self.room_id[1:] + auth_event_ids.remove(create_event_id) + event_dict["auth_events"] = auth_event_ids if self.is_state(): event_dict["state_key"] = self._state_key @@ -232,7 +312,7 @@ class EventBuilderFactory: room_version=room_version, type=key_values["type"], state_key=key_values.get("state_key"), - room_id=key_values["room_id"], + room_id=key_values.get("room_id"), sender=key_values["sender"], content=key_values.get("content", {}), unsigned=key_values.get("unsigned", {}), @@ -249,8 +329,8 @@ def create_local_event_from_event_dict( event_dict: JsonDict, internal_metadata_dict: Optional[JsonDict] = None, ) -> EventBase: - """Takes a fully formed event dict, ensuring that fields like `origin` - and `origin_server_ts` have correct values for a locally produced event, + """Takes a fully formed event dict, ensuring that fields like + `origin_server_ts` have correct values for a locally produced event, then signs and hashes it. """ @@ -266,7 +346,6 @@ def create_local_event_from_event_dict( if format_version == EventFormatVersions.ROOM_V1_V2: event_dict["event_id"] = _create_event_id(clock, hostname) - event_dict["origin"] = hostname event_dict.setdefault("origin_server_ts", time_now) event_dict.setdefault("unsigned", {}) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index dd21a6136b..63551143d8 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -248,7 +248,7 @@ class EventContext(UnpersistedEventContextBase): @tag_args async def get_current_state_ids( self, state_filter: Optional["StateFilter"] = None - ) -> Optional[StateMap[str]]: + ) -> StateMap[str]: """ Gets the room state map, including this event - ie, the state in ``state_group`` @@ -256,13 +256,12 @@ class EventContext(UnpersistedEventContextBase): not make it into the room state. This method will raise an exception if ``rejected`` is set. + It is also an error to access this for an outlier event. + Arg: state_filter: specifies the type of state event to fetch from DB, example: EventTypes.JoinRules Returns: - Returns None if state_group is None, which happens when the associated - event is an outlier. - Maps a (type, state_key) to the event ID of the state event matching this tuple. """ @@ -300,12 +299,19 @@ class EventContext(UnpersistedEventContextBase): this tuple. """ - assert self.state_group_before_event is not None + if self.state_group_before_event is None: + return {} return await self._storage.state.get_state_ids_for_group( self.state_group_before_event, state_filter ) +EventPersistencePair = Tuple[EventBase, EventContext] +""" +The combination of an event to be persisted and its context. +""" + + @attr.s(slots=True, auto_attribs=True) class UnpersistedEventContext(UnpersistedEventContextBase): """ @@ -363,7 +369,7 @@ class UnpersistedEventContext(UnpersistedEventContextBase): room_id: str, last_known_state_group: int, datastore: "StateGroupDataStore", - ) -> List[Tuple[EventBase, EventContext]]: + ) -> List[EventPersistencePair]: """ Takes a list of events and their associated unpersisted contexts and persists the unpersisted contexts, returning a list of events and persisted contexts. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 54f94add4d..942072cf84 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -26,8 +26,8 @@ from typing import ( Any, Awaitable, Callable, + Collection, Dict, - Iterable, List, Mapping, Match, @@ -40,6 +40,8 @@ import attr from canonicaljson import encode_canonical_json from synapse.api.constants import ( + CANONICALJSON_MAX_INT, + CANONICALJSON_MIN_INT, MAX_PDU_SIZE, EventContentFields, EventTypes, @@ -47,6 +49,7 @@ from synapse.api.constants import ( ) from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersion +from synapse.logging.opentracing import SynapseTags, set_tag, trace from synapse.types import JsonDict, Requester from . import EventBase, StrippedStateEvent, make_event_from_dict @@ -61,9 +64,6 @@ SPLIT_FIELD_REGEX = re.compile(r"\\*\.") # Find escaped characters, e.g. those with a \ in front of them. ESCAPE_SEQUENCE_PATTERN = re.compile(r"\\(.)") -CANONICALJSON_MAX_INT = (2**53) - 1 -CANONICALJSON_MIN_INT = -CANONICALJSON_MAX_INT - # Module API callback that allows adding fields to the unsigned section of # events that are sent to clients. @@ -177,9 +177,12 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic if room_version.updated_redaction_rules: # MSC2176 rules state that create events cannot have their `content` redacted. new_content = event_dict["content"] - elif not room_version.implicit_room_creator: + if not room_version.implicit_room_creator: # Some room versions give meaning to `creator` add_fields("creator") + if room_version.msc4291_room_ids_as_hashes: + # room_id is not allowed on the create event as it's derived from the event ID + allowed_keys.remove("room_id") elif event_type == EventTypes.JoinRules: add_fields("join_rule") @@ -422,11 +425,21 @@ class SerializeEventConfig: # False, that state will be removed from the event before it is returned. # Otherwise, it will be kept. include_stripped_room_state: bool = False + # When True, sets unsigned fields to help clients identify events which + # only server admins can see through other configuration. For example, + # whether an event was soft failed by the server. + include_admin_metadata: bool = False _DEFAULT_SERIALIZE_EVENT_CONFIG = SerializeEventConfig() +def make_config_for_admin(existing: SerializeEventConfig) -> SerializeEventConfig: + # Set the options which are only available to server admins, + # and copy the rest. + return attr.evolve(existing, include_admin_metadata=True) + + def serialize_event( e: Union[JsonDict, EventBase], time_now_ms: int, @@ -518,6 +531,10 @@ def serialize_event( if config.as_client_event: d = config.event_format(d) + # Ensure the room_id field is set for create events in MSC4291 rooms + if e.type == EventTypes.Create and e.room_version.msc4291_room_ids_as_hashes: + d["room_id"] = e.room_id + # If the event is a redaction, the field with the redacted event ID appears # in a different location depending on the room version. e.redacts handles # fetching from the proper location; copy it to the other location for forwards- @@ -529,6 +546,12 @@ def serialize_event( d["content"] = dict(d["content"]) d["content"]["redacts"] = e.redacts + if config.include_admin_metadata: + if e.internal_metadata.is_soft_failed(): + d["unsigned"]["io.element.synapse.soft_failed"] = True + if e.internal_metadata.policy_server_spammy: + d["unsigned"]["io.element.synapse.policy_server_spammy"] = True + only_event_fields = config.only_event_fields if only_event_fields: if not isinstance(only_event_fields, list) or not all( @@ -549,6 +572,7 @@ class EventClientSerializer: def __init__(self, hs: "HomeServer") -> None: self._store = hs.get_datastores().main + self._auth = hs.get_auth() self._add_extra_fields_to_unsigned_client_event_callbacks: List[ ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK ] = [] @@ -577,6 +601,15 @@ class EventClientSerializer: if not isinstance(event, EventBase): return event + # Force-enable server admin metadata because the only time an event with + # relevant metadata will be when the admin requested it via their admin + # client config account data. Also, it's "just" some `unsigned` fields, so + # shouldn't cause much in terms of problems to downstream consumers. + if config.requester is not None and await self._auth.is_server_admin( + config.requester + ): + config = make_config_for_admin(config) + serialized_event = serialize_event(event, time_now, config=config) new_unsigned = {} @@ -678,9 +711,10 @@ class EventClientSerializer: "m.relations", {} ).update(serialized_aggregations) + @trace async def serialize_events( self, - events: Iterable[Union[JsonDict, EventBase]], + events: Collection[Union[JsonDict, EventBase]], time_now: int, *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, @@ -699,6 +733,11 @@ class EventClientSerializer: Returns: The list of serialized events """ + set_tag( + SynapseTags.FUNC_ARG_PREFIX + "events.length", + str(len(events)), + ) + return [ await self.serialize_event( event, @@ -847,6 +886,14 @@ def strip_event(event: EventBase) -> JsonDict: Stripped state events can only have the `sender`, `type`, `state_key` and `content` properties present. """ + # MSC4311: Ensure the create event is available on invites and knocks. + # TODO: Implement the rest of MSC4311 + if ( + event.room_version.msc4291_room_ids_as_hashes + and event.type == EventTypes.Create + and event.get_state_key() == "" + ): + return event.get_pdu_json() return { "type": event.type, diff --git a/synapse/events/validator.py b/synapse/events/validator.py index 8aa8d7e017..4d9ba15829 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -67,7 +67,6 @@ class EventValidator: "auth_events", "content", "hashes", - "origin", "prev_events", "sender", "type", @@ -77,18 +76,9 @@ class EventValidator: if k not in event: raise SynapseError(400, "Event does not have key %s" % (k,)) - # Check that the following keys have string values - event_strings = ["origin"] - - for s in event_strings: - if not isinstance(getattr(event, s), str): - raise SynapseError(400, "'%s' not a string type" % (s,)) - # Depending on the room version, ensure the data is spec compliant JSON. if event.room_version.strict_canonicaljson: - # Note that only the client controlled portion of the event is - # checked, since we trust the portions of the event we created. - validate_canonicaljson(event.content) + validate_canonicaljson(event.get_pdu_json()) if event.type == EventTypes.Aliases: if "aliases" in event.content: @@ -193,8 +183,18 @@ class EventValidator: fields an event would have """ + create_event_as_room_id = ( + event.room_version.msc4291_room_ids_as_hashes + and event.type == EventTypes.Create + and hasattr(event, "state_key") + and event.state_key == "" + ) + strings = ["room_id", "sender", "type"] + if create_event_as_room_id: + strings.remove("room_id") + if hasattr(event, "state_key"): strings.append("state_key") @@ -202,7 +202,14 @@ class EventValidator: if not isinstance(getattr(event, s), str): raise SynapseError(400, "Not '%s' a string type" % (s,)) - RoomID.from_string(event.room_id) + if not create_event_as_room_id: + assert event.room_id is not None + RoomID.from_string(event.room_id) + if event.room_version.msc4291_room_ids_as_hashes and not RoomID.is_valid( + event.room_id + ): + raise SynapseError(400, f"Invalid room ID '{event.room_id}'") + UserID.from_string(event.sender) if event.type == EventTypes.Message: diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index b101a389ef..a1c9c286ac 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -20,7 +20,7 @@ # # import logging -from typing import TYPE_CHECKING, Awaitable, Callable, Optional +from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional, Sequence from synapse.api.constants import MAX_DEPTH, EventContentFields, EventTypes, Membership from synapse.api.errors import Codes, SynapseError @@ -29,6 +29,8 @@ from synapse.crypto.event_signing import check_event_content_hash from synapse.crypto.keyring import Keyring from synapse.events import EventBase, make_event_from_dict from synapse.events.utils import prune_event, validate_canonicaljson +from synapse.federation.units import filter_pdus_for_valid_depth +from synapse.handlers.room_policy import RoomPolicyHandler from synapse.http.servlet import assert_params_in_dict from synapse.logging.opentracing import log_kv, trace from synapse.types import JsonDict, get_domain_from_id @@ -63,6 +65,24 @@ class FederationBase: self._clock = hs.get_clock() self._storage_controllers = hs.get_storage_controllers() + # We need to define this lazily otherwise we get a cyclic dependency. + # self._policy_handler = hs.get_room_policy_handler() + self._policy_handler: Optional[RoomPolicyHandler] = None + + def _lazily_get_policy_handler(self) -> RoomPolicyHandler: + """Lazily get the room policy handler. + + This is required to avoid an import cycle: RoomPolicyHandler requires a + FederationClient, which requires a FederationBase, which requires a + RoomPolicyHandler. + + Returns: + RoomPolicyHandler: The room policy handler. + """ + if self._policy_handler is None: + self._policy_handler = self.hs.get_room_policy_handler() + return self._policy_handler + @trace async def _check_sigs_and_hash( self, @@ -79,6 +99,10 @@ class FederationBase: Also runs the event through the spam checker; if it fails, redacts the event and flags it as soft-failed. + Also checks that the event is allowed by the policy server, if the room uses + a policy server. If the event is not allowed, the event is flagged as + soft-failed but not redacted. + Args: room_version: The room version of the PDU pdu: the event to be checked @@ -144,6 +168,18 @@ class FederationBase: ) return redacted_event + policy_allowed = await self._lazily_get_policy_handler().is_event_allowed(pdu) + if not policy_allowed: + logger.warning( + "Event not allowed by policy server, soft-failing %s", pdu.event_id + ) + pdu.internal_metadata.soft_failed = True + pdu.internal_metadata.policy_server_spammy = True + # Note: we don't redact the event so admins can inspect the event after the + # fact. Other processes may redact the event, but that won't be applied to + # the database copy of the event until the server's config requires it. + return pdu + spam_check = await self._spam_checker_module_callbacks.check_event_for_spam(pdu) if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: @@ -267,6 +303,15 @@ def _is_invite_via_3pid(event: EventBase) -> bool: ) +def parse_events_from_pdu_json( + pdus_json: Sequence[JsonDict], room_version: RoomVersion +) -> List[EventBase]: + return [ + event_from_pdu_json(pdu_json, room_version) + for pdu_json in filter_pdus_for_valid_depth(pdus_json) + ] + + def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventBase: """Construct an EventBase from an event json received over federation @@ -278,8 +323,7 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB SynapseError: if the pdu is missing required fields or is otherwise not a valid matrix event """ - # we could probably enforce a bunch of other fields here (room_id, sender, - # origin, etc etc) + # we could probably enforce a bunch of other fields here (room_id, sender, etc.) assert_params_in_dict(pdu_json, ("type", "depth")) # Strip any unauthorized values from "unsigned" if they exist @@ -299,6 +343,21 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB if room_version.strict_canonicaljson: validate_canonicaljson(pdu_json) + # enforce that MSC4291 auth events don't include the create event. + # N.B. if they DO include a spurious create event, it'll fail auth checks elsewhere, so we don't + # need to do expensive DB lookups to find which event ID is the create event here. + if room_version.msc4291_room_ids_as_hashes: + room_id = pdu_json.get("room_id") + if room_id: + create_event_id = "$" + room_id[1:] + auth_events = pdu_json.get("auth_events") + if auth_events: + if create_event_id in auth_events: + raise SynapseError( + 400, + "auth_events must not contain the create event", + Codes.BAD_JSON, + ) event = make_event_from_dict(pdu_json, room_version) return event diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 7d80ff6998..542d9650d4 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -68,12 +68,15 @@ from synapse.federation.federation_base import ( FederationBase, InvalidEventSignatureError, event_from_pdu_json, + parse_events_from_pdu_json, ) from synapse.federation.transport.client import SendJoinResponse from synapse.http.client import is_unknown_endpoint from synapse.http.types import QueryParams from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, tag_args, trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.types import JsonDict, StrCollection, UserID, get_domain_from_id +from synapse.types.handlers.policy_server import RECOMMENDATION_OK, RECOMMENDATION_SPAM from synapse.util.async_helpers import concurrently_execute from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.retryutils import NotRetryingDestination @@ -83,7 +86,9 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -sent_queries_counter = Counter("synapse_federation_client_sent_queries", "", ["type"]) +sent_queries_counter = Counter( + "synapse_federation_client_sent_queries", "", labelnames=["type", SERVER_NAME_LABEL] +) PDU_RETRY_TIME_MS = 1 * 60 * 1000 @@ -135,13 +140,14 @@ class FederationClient(FederationBase): self.state = hs.get_state_handler() self.transport_layer = hs.get_federation_transport_client() - self.hostname = hs.hostname + self.server_name = hs.hostname self.signing_key = hs.signing_key # Cache mapping `event_id` to a tuple of the event itself and the `pull_origin` # (which server we pulled the event from) self._get_pdu_cache: ExpiringCache[str, Tuple[EventBase, str]] = ExpiringCache( cache_name="get_pdu_cache", + server_name=self.server_name, clock=self._clock, max_len=1000, expiry_ms=120 * 1000, @@ -160,6 +166,7 @@ class FederationClient(FederationBase): Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]], ] = ExpiringCache( cache_name="get_room_hierarchy_cache", + server_name=self.server_name, clock=self._clock, max_len=1000, expiry_ms=5 * 60 * 1000, @@ -205,7 +212,10 @@ class FederationClient(FederationBase): Returns: The JSON object from the response """ - sent_queries_counter.labels(query_type).inc() + sent_queries_counter.labels( + type=query_type, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() return await self.transport_layer.make_query( destination, @@ -227,7 +237,10 @@ class FederationClient(FederationBase): Returns: The JSON object from the response """ - sent_queries_counter.labels("client_device_keys").inc() + sent_queries_counter.labels( + type="client_device_keys", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() return await self.transport_layer.query_client_keys( destination, content, timeout ) @@ -238,7 +251,10 @@ class FederationClient(FederationBase): """Query the device keys for a list of user ids hosted on a remote server. """ - sent_queries_counter.labels("user_devices").inc() + sent_queries_counter.labels( + type="user_devices", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() return await self.transport_layer.query_user_devices( destination, user_id, timeout ) @@ -260,7 +276,10 @@ class FederationClient(FederationBase): Returns: The JSON object from the response """ - sent_queries_counter.labels("client_one_time_keys").inc() + sent_queries_counter.labels( + type="client_one_time_keys", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() # Convert the query with counts into a stable and unstable query and check # if attempting to claim more than 1 OTK. @@ -349,7 +368,7 @@ class FederationClient(FederationBase): room_version = await self.store.get_room_version(room_id) - pdus = [event_from_pdu_json(p, room_version) for p in transaction_data_pdus] + pdus = parse_events_from_pdu_json(transaction_data_pdus, room_version) # Check signatures and hash of pdus, removing any from the list that fail checks pdus[:] = await self._check_sigs_and_hash_for_pulled_events_and_fetch( @@ -393,9 +412,7 @@ class FederationClient(FederationBase): transaction_data, ) - pdu_list: List[EventBase] = [ - event_from_pdu_json(p, room_version) for p in transaction_data["pdus"] - ] + pdu_list = parse_events_from_pdu_json(transaction_data["pdus"], room_version) if pdu_list and pdu_list[0]: pdu = pdu_list[0] @@ -422,6 +439,62 @@ class FederationClient(FederationBase): return None + @trace + @tag_args + async def get_pdu_policy_recommendation( + self, destination: str, pdu: EventBase, timeout: Optional[int] = None + ) -> str: + """Requests that the destination server (typically a policy server) + check the event and return its recommendation on how to handle the + event. + + If the policy server could not be contacted or the policy server + returned an unknown recommendation, this returns an OK recommendation. + This type fixing behaviour is done because the typical caller will be + in a critical call path and would generally interpret a `None` or similar + response as "weird value; don't care; move on without taking action". We + just frontload that logic here. + + + Args: + destination: The remote homeserver to ask (a policy server) + pdu: The event to check + timeout: How long to try (in ms) the destination for before + giving up. None indicates no timeout. + + Returns: + The policy recommendation, or RECOMMENDATION_OK if the policy server was + uncontactable or returned an unknown recommendation. + """ + + logger.debug( + "get_pdu_policy_recommendation for event_id=%s from %s", + pdu.event_id, + destination, + ) + + try: + res = await self.transport_layer.get_policy_recommendation_for_pdu( + destination, pdu, timeout=timeout + ) + recommendation = res.get("recommendation") + if not isinstance(recommendation, str): + raise InvalidResponseError("recommendation is not a string") + if recommendation not in (RECOMMENDATION_OK, RECOMMENDATION_SPAM): + logger.warning( + "get_pdu_policy_recommendation: unknown recommendation: %s", + recommendation, + ) + return RECOMMENDATION_OK + return recommendation + except Exception as e: + logger.warning( + "get_pdu_policy_recommendation: server %s responded with error, assuming OK recommendation: %s", + destination, + e, + ) + return RECOMMENDATION_OK + @trace @tag_args async def get_pdu( @@ -809,7 +882,7 @@ class FederationClient(FederationBase): room_version = await self.store.get_room_version(room_id) - auth_chain = [event_from_pdu_json(p, room_version) for p in res["auth_chain"]] + auth_chain = parse_events_from_pdu_json(res["auth_chain"], room_version) signed_auth = await self._check_sigs_and_hash_for_pulled_events_and_fetch( destination, auth_chain, room_version=room_version @@ -1012,7 +1085,7 @@ class FederationClient(FederationBase): # there's some we never care about ev = builder.create_local_event_from_event_dict( self._clock, - self.hostname, + self.server_name, self.signing_key, room_version=room_version, event_dict=pdu_dict, @@ -1529,9 +1602,7 @@ class FederationClient(FederationBase): room_version = await self.store.get_room_version(room_id) - events = [ - event_from_pdu_json(e, room_version) for e in content.get("events", []) - ] + events = parse_events_from_pdu_json(content.get("events", []), room_version) signed_events = await self._check_sigs_and_hash_for_pulled_events_and_fetch( destination, events, room_version=room_version @@ -1764,7 +1835,7 @@ class FederationClient(FederationBase): ) return timestamp_to_event_response except SynapseError as e: - logger.warn( + logger.warning( "timestamp_to_event(room_id=%s, timestamp=%s, direction=%s): encountered error when trying to fetch from destinations: %s", room_id, timestamp, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 1932fa82a4..a8d5c3c280 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -59,14 +59,14 @@ from synapse.api.errors import ( from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.crypto.event_signing import compute_event_signature from synapse.events import EventBase -from synapse.events.snapshot import EventContext +from synapse.events.snapshot import EventPersistencePair from synapse.federation.federation_base import ( FederationBase, InvalidEventSignatureError, event_from_pdu_json, ) from synapse.federation.persistence import TransactionActions -from synapse.federation.units import Edu, Transaction +from synapse.federation.units import Edu, Transaction, serialize_and_filter_pdus from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.http.servlet import assert_params_in_dict from synapse.logging.context import ( @@ -82,10 +82,10 @@ from synapse.logging.opentracing import ( tag_args, trace, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.replication.http.federation import ( ReplicationFederationSendEduRestServlet, - ReplicationGetQueryRestServlet, ) from synapse.storage.databases.main.lock import Lock from synapse.storage.databases.main.roommember import extract_heroes_from_room_summary @@ -105,23 +105,30 @@ TRANSACTION_CONCURRENCY_LIMIT = 10 logger = logging.getLogger(__name__) -received_pdus_counter = Counter("synapse_federation_server_received_pdus", "") +received_pdus_counter = Counter( + "synapse_federation_server_received_pdus", "", labelnames=[SERVER_NAME_LABEL] +) -received_edus_counter = Counter("synapse_federation_server_received_edus", "") +received_edus_counter = Counter( + "synapse_federation_server_received_edus", "", labelnames=[SERVER_NAME_LABEL] +) received_queries_counter = Counter( - "synapse_federation_server_received_queries", "", ["type"] + "synapse_federation_server_received_queries", + "", + labelnames=["type", SERVER_NAME_LABEL], ) pdu_process_time = Histogram( "synapse_federation_server_pdu_process_time", "Time taken to process an event", + labelnames=[SERVER_NAME_LABEL], ) last_pdu_ts_metric = Gauge( "synapse_federation_last_received_pdu_time", "The timestamp of the last PDU which was successfully received from the given domain", - labelnames=("server_name",), + labelnames=("origin_server_name", SERVER_NAME_LABEL), ) @@ -160,7 +167,10 @@ class FederationServer(FederationBase): # We cache results for transaction with the same ID self._transaction_resp_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "fed_txn_handler", timeout_ms=30000 + clock=hs.get_clock(), + name="fed_txn_handler", + server_name=self.server_name, + timeout_ms=30000, ) self.transaction_actions = TransactionActions(self.store) @@ -170,10 +180,18 @@ class FederationServer(FederationBase): # We cache responses to state queries, as they take a while and often # come in waves. self._state_resp_cache: ResponseCache[Tuple[str, Optional[str]]] = ( - ResponseCache(hs.get_clock(), "state_resp", timeout_ms=30000) + ResponseCache( + clock=hs.get_clock(), + name="state_resp", + server_name=self.server_name, + timeout_ms=30000, + ) ) self._state_ids_resp_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "state_ids_resp", timeout_ms=30000 + clock=hs.get_clock(), + name="state_ids_resp", + server_name=self.server_name, + timeout_ms=30000, ) self._federation_metrics_domains = ( @@ -424,7 +442,9 @@ class FederationServer(FederationBase): report back to the sending server. """ - received_pdus_counter.inc(len(transaction.pdus)) + received_pdus_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc( + len(transaction.pdus) + ) origin_host, _ = parse_server_name(origin) @@ -469,7 +489,12 @@ class FederationServer(FederationBase): logger.info("Ignoring PDU: %s", e) continue - event = event_from_pdu_json(p, room_version) + try: + event = event_from_pdu_json(p, room_version) + except SynapseError as e: + logger.info("Ignoring PDU for failing to deserialize: %s", e) + continue + pdus_by_room.setdefault(room_id, []).append(event) if event.origin_server_ts > newest_pdu_ts: @@ -530,7 +555,9 @@ class FederationServer(FederationBase): ) if newest_pdu_ts and origin in self._federation_metrics_domains: - last_pdu_ts_metric.labels(server_name=origin).set(newest_pdu_ts / 1000) + last_pdu_ts_metric.labels( + origin_server_name=origin, **{SERVER_NAME_LABEL: self.server_name} + ).set(newest_pdu_ts / 1000) return pdu_results @@ -538,7 +565,7 @@ class FederationServer(FederationBase): """Process the EDUs in a received transaction.""" async def _process_edu(edu_dict: JsonDict) -> None: - received_edus_counter.inc() + received_edus_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc() edu = Edu( origin=origin, @@ -636,8 +663,8 @@ class FederationServer(FederationBase): ) return { - "pdus": [pdu.get_pdu_json() for pdu in pdus], - "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], + "pdus": serialize_and_filter_pdus(pdus), + "auth_chain": serialize_and_filter_pdus(auth_chain), } async def on_pdu_request( @@ -653,7 +680,10 @@ class FederationServer(FederationBase): async def on_query_request( self, query_type: str, args: Dict[str, str] ) -> Tuple[int, Dict[str, Any]]: - received_queries_counter.labels(query_type).inc() + received_queries_counter.labels( + type=query_type, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() resp = await self.registry.on_query(query_type, args) return 200, resp @@ -696,6 +726,12 @@ class FederationServer(FederationBase): pdu = event_from_pdu_json(content, room_version) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, pdu.room_id) + if await self._spam_checker_module_callbacks.should_drop_federated_event(pdu): + logger.info( + "Federated event contains spam, dropping %s", + pdu.event_id, + ) + raise SynapseError(403, Codes.FORBIDDEN) try: pdu = await self._check_sigs_and_hash(room_version, pdu) except InvalidEventSignatureError as e: @@ -761,8 +797,8 @@ class FederationServer(FederationBase): event_json = event.get_pdu_json(time_now) resp = { "event": event_json, - "state": [p.get_pdu_json(time_now) for p in state_events], - "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain_events], + "state": serialize_and_filter_pdus(state_events, time_now), + "auth_chain": serialize_and_filter_pdus(auth_chain_events, time_now), "members_omitted": caller_supports_partial_state, } @@ -878,7 +914,7 @@ class FederationServer(FederationBase): async def _on_send_membership_event( self, origin: str, content: JsonDict, membership_type: str, room_id: str - ) -> Tuple[EventBase, EventContext]: + ) -> EventPersistencePair: """Handle an on_send_{join,leave,knock} request Does some preliminary validation before passing the request on to the @@ -917,7 +953,8 @@ class FederationServer(FederationBase): # joins) or the full state (for full joins). # Return a 404 as we would if we weren't in the room at all. logger.info( - f"Rejecting /send_{membership_type} to %s because it's a partial state room", + "Rejecting /send_%s to %s because it's a partial state room", + membership_type, room_id, ) raise SynapseError( @@ -1005,7 +1042,7 @@ class FederationServer(FederationBase): time_now = self._clock.time_msec() auth_pdus = await self.handler.on_event_auth(event_id) - res = {"auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus]} + res = {"auth_chain": serialize_and_filter_pdus(auth_pdus, time_now)} return 200, res async def on_query_client_keys( @@ -1090,7 +1127,7 @@ class FederationServer(FederationBase): time_now = self._clock.time_msec() - return {"events": [ev.get_pdu_json(time_now) for ev in missing_events]} + return {"events": serialize_and_filter_pdus(missing_events, time_now)} async def on_openid_userinfo(self, token: str) -> Optional[str]: ts_now_ms = self._clock.time_msec() @@ -1288,9 +1325,9 @@ class FederationServer(FederationBase): origin, event.event_id ) if received_ts is not None: - pdu_process_time.observe( - (self._clock.time_msec() - received_ts) / 1000 - ) + pdu_process_time.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).observe((self._clock.time_msec() - received_ts) / 1000) next = await self._get_next_nonspam_staged_event_for_room( room_id, room_version @@ -1368,7 +1405,6 @@ class FederationHandlerRegistry: # and use them. However we have guards before we use them to ensure that # we don't route to ourselves, and in monolith mode that will always be # the case. - self._get_query_client = ReplicationGetQueryRestServlet.make_client(hs) self._send_edu = ReplicationFederationSendEduRestServlet.make_client(hs) self.edu_handlers: Dict[str, Callable[[str, dict], Awaitable[None]]] = {} @@ -1457,10 +1493,6 @@ class FederationHandlerRegistry: if handler: return await handler(args) - # Check if we can route it somewhere else that isn't us - if self._instance_name == "master": - return await self._get_query_client(query_type=query_type, args=args) - # Uh oh, no handler! Let's raise an exception so the request returns an # error. logger.warning("No handler registered for query type %s", query_type) diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index b5c9fcff7c..2fdee9ac54 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -37,6 +37,7 @@ Events are replicated via a separate events stream. """ import logging +from enum import Enum from typing import ( TYPE_CHECKING, Dict, @@ -54,7 +55,7 @@ from sortedcontainers import SortedDict from synapse.api.presence import UserPresenceState from synapse.federation.sender import AbstractFederationSender, FederationSender -from synapse.metrics import LaterGauge +from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.replication.tcp.streams.federation import FederationStream from synapse.types import JsonDict, ReadReceipt, RoomStreamToken, StrCollection from synapse.util.metrics import Measure @@ -67,6 +68,25 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +class QueueNames(str, Enum): + PRESENCE_MAP = "presence_map" + KEYED_EDU = "keyed_edu" + KEYED_EDU_CHANGED = "keyed_edu_changed" + EDUS = "edus" + POS_TIME = "pos_time" + PRESENCE_DESTINATIONS = "presence_destinations" + + +queue_name_to_gauge_map: Dict[QueueNames, LaterGauge] = {} + +for queue_name in QueueNames: + queue_name_to_gauge_map[queue_name] = LaterGauge( + name=f"synapse_federation_send_queue_{queue_name.value}_size", + desc="", + labelnames=[SERVER_NAME_LABEL], + ) + + class FederationRemoteSendQueue(AbstractFederationSender): """A drop in replacement for FederationSender""" @@ -111,23 +131,16 @@ class FederationRemoteSendQueue(AbstractFederationSender): # we make a new function, so we need to make a new function so the inner # lambda binds to the queue rather than to the name of the queue which # changes. ARGH. - def register(name: str, queue: Sized) -> None: - LaterGauge( - "synapse_federation_send_queue_%s_size" % (queue_name,), - "", - [], - lambda: len(queue), + def register(queue_name: QueueNames, queue: Sized) -> None: + queue_name_to_gauge_map[queue_name].register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: {(self.server_name,): len(queue)}, ) - for queue_name in [ - "presence_map", - "keyed_edu", - "keyed_edu_changed", - "edus", - "pos_time", - "presence_destinations", - ]: - register(queue_name, getattr(self, queue_name)) + for queue_name in QueueNames: + queue = getattr(self, queue_name.value) + assert isinstance(queue, Sized) + register(queue_name, queue=queue) self.clock.looping_call(self._clear_queue, 30 * 1000) @@ -156,7 +169,9 @@ class FederationRemoteSendQueue(AbstractFederationSender): def _clear_queue_before_pos(self, position_to_delete: int) -> None: """Clear all the queues from before a given position""" - with Measure(self.clock, "send_queue._clear"): + with Measure( + self.clock, name="send_queue._clear", server_name=self.server_name + ): # Delete things out of presence maps keys = self.presence_destinations.keys() i = self.presence_destinations.bisect_left(position_to_delete) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 1888480881..6baa233143 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -139,18 +139,18 @@ from typing import ( Hashable, Iterable, List, + Literal, Optional, - Set, Tuple, ) import attr from prometheus_client import Counter -from typing_extensions import Literal from twisted.internet import defer import synapse.metrics +from synapse.api.constants import EventTypes, Membership from synapse.api.presence import UserPresenceState from synapse.events import EventBase from synapse.federation.sender.per_destination_queue import ( @@ -161,6 +161,7 @@ from synapse.federation.sender.transaction_manager import TransactionManager from synapse.federation.units import Edu from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.metrics import ( + SERVER_NAME_LABEL, LaterGauge, event_processing_loop_counter, event_processing_loop_room_count, @@ -170,7 +171,13 @@ from synapse.metrics.background_process_metrics import ( run_as_background_process, wrap_as_background_process, ) -from synapse.types import JsonDict, ReadReceipt, RoomStreamToken, StrCollection +from synapse.types import ( + JsonDict, + ReadReceipt, + RoomStreamToken, + StrCollection, + get_domain_from_id, +) from synapse.util import Clock from synapse.util.metrics import Measure from synapse.util.retryutils import filter_destinations_by_retry_limiter @@ -184,11 +191,31 @@ logger = logging.getLogger(__name__) sent_pdus_destination_dist_count = Counter( "synapse_federation_client_sent_pdu_destinations_count", "Number of PDUs queued for sending to one or more destinations", + labelnames=[SERVER_NAME_LABEL], ) sent_pdus_destination_dist_total = Counter( "synapse_federation_client_sent_pdu_destinations", "Total number of PDUs queued for sending across all destinations", + labelnames=[SERVER_NAME_LABEL], +) + +transaction_queue_pending_destinations_gauge = LaterGauge( + name="synapse_federation_transaction_queue_pending_destinations", + desc="", + labelnames=[SERVER_NAME_LABEL], +) + +transaction_queue_pending_pdus_gauge = LaterGauge( + name="synapse_federation_transaction_queue_pending_pdus", + desc="", + labelnames=[SERVER_NAME_LABEL], +) + +transaction_queue_pending_edus_gauge = LaterGauge( + name="synapse_federation_transaction_queue_pending_edus", + desc="", + labelnames=[SERVER_NAME_LABEL], ) # Time (in s) to wait before trying to wake up destinations that have @@ -291,18 +318,21 @@ class _DestinationWakeupQueue: Staggers waking up of per destination queues to ensure that we don't attempt to start TLS connections with many hosts all at once, leading to pinned CPU. + """ # The maximum duration in seconds between queuing up a destination and it # being woken up. _MAX_TIME_IN_QUEUE = 30.0 - # The maximum duration in seconds between waking up consecutive destination - # queues. - _MAX_DELAY = 0.1 - sender: "FederationSender" = attr.ib() + server_name: str = attr.ib() + """ + Our homeserver name (used to label metrics) (`hs.hostname`). + """ clock: Clock = attr.ib() + max_delay_s: int = attr.ib() + queue: "OrderedDict[str, Literal[None]]" = attr.ib(factory=OrderedDict) processing: bool = attr.ib(default=False) @@ -332,13 +362,15 @@ class _DestinationWakeupQueue: # We also add an upper bound to the delay, to gracefully handle the # case where the queue only has a few entries in it. current_sleep_seconds = min( - self._MAX_DELAY, self._MAX_TIME_IN_QUEUE / len(self.queue) + self.max_delay_s, self._MAX_TIME_IN_QUEUE / len(self.queue) ) while self.queue: destination, _ = self.queue.popitem(last=False) queue = self.sender._get_per_destination_queue(destination) + if queue is None: + continue if not queue._new_data_to_send: # The per destination queue has already been woken up. @@ -385,65 +417,71 @@ class FederationSender(AbstractFederationSender): # map from destination to PerDestinationQueue self._per_destination_queues: Dict[str, PerDestinationQueue] = {} - LaterGauge( - "synapse_federation_transaction_queue_pending_destinations", - "", - [], - lambda: sum( - 1 - for d in self._per_destination_queues.values() - if d.transmission_loop_running - ), + transaction_queue_pending_destinations_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: { + (self.server_name,): sum( + 1 + for d in self._per_destination_queues.values() + if d.transmission_loop_running + ) + }, ) - - LaterGauge( - "synapse_federation_transaction_queue_pending_pdus", - "", - [], - lambda: sum( - d.pending_pdu_count() for d in self._per_destination_queues.values() - ), + transaction_queue_pending_pdus_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: { + (self.server_name,): sum( + d.pending_pdu_count() for d in self._per_destination_queues.values() + ) + }, ) - LaterGauge( - "synapse_federation_transaction_queue_pending_edus", - "", - [], - lambda: sum( - d.pending_edu_count() for d in self._per_destination_queues.values() - ), + transaction_queue_pending_edus_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: { + (self.server_name,): sum( + d.pending_edu_count() for d in self._per_destination_queues.values() + ) + }, ) self._is_processing = False self._last_poked_id = -1 - # map from room_id to a set of PerDestinationQueues which we believe are - # awaiting a call to flush_read_receipts_for_room. The presence of an entry - # here for a given room means that we are rate-limiting RR flushes to that room, - # and that there is a pending call to _flush_rrs_for_room in the system. - self._queues_awaiting_rr_flush_by_room: Dict[str, Set[PerDestinationQueue]] = {} - - self._rr_txn_interval_per_room_ms = ( - 1000.0 - / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second - ) - self._external_cache = hs.get_external_cache() - self._destination_wakeup_queue = _DestinationWakeupQueue(self, self.clock) + + rr_txn_interval_per_room_s = ( + 1.0 / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second + ) + self._destination_wakeup_queue = _DestinationWakeupQueue( + self, self.server_name, self.clock, max_delay_s=rr_txn_interval_per_room_s + ) # Regularly wake up destinations that have outstanding PDUs to be caught up self.clock.looping_call_now( run_as_background_process, WAKEUP_RETRY_PERIOD_SEC * 1000.0, "wake_destinations_needing_catchup", + self.server_name, self._wake_destinations_needing_catchup, ) - def _get_per_destination_queue(self, destination: str) -> PerDestinationQueue: + def _get_per_destination_queue( + self, destination: str + ) -> Optional[PerDestinationQueue]: """Get or create a PerDestinationQueue for the given destination Args: destination: server_name of remote server + + Returns: + None if the destination is not allowed by the federation whitelist. + Otherwise a PerDestinationQueue for this destination. """ + if not self.hs.config.federation.is_domain_allowed_according_to_federation_whitelist( + destination + ): + return None + queue = self._per_destination_queues.get(destination) if not queue: queue = PerDestinationQueue(self.hs, self._transaction_manager, destination) @@ -466,7 +504,9 @@ class FederationSender(AbstractFederationSender): # fire off a processing loop in the background run_as_background_process( - "process_event_queue_for_federation", self._process_event_queue_loop + "process_event_queue_for_federation", + self.server_name, + self._process_event_queue_loop, ) async def _process_event_queue_loop(self) -> None: @@ -616,6 +656,31 @@ class FederationSender(AbstractFederationSender): ) return + # If we've rescinded an invite then we want to tell the + # other server. + if ( + event.type == EventTypes.Member + and event.membership == Membership.LEAVE + and event.sender != event.state_key + ): + # We check if this leave event is rescinding an invite + # by looking if there is an invite event for the user in + # the auth events. It could otherwise be a kick or + # unban, which we don't want to send (if the user wasn't + # already in the room). + auth_events = await self.store.get_events_as_list( + event.auth_event_ids() + ) + for auth_event in auth_events: + if ( + auth_event.type == EventTypes.Member + and auth_event.state_key == event.state_key + and auth_event.membership == Membership.INVITE + ): + destinations = set(destinations) + destinations.add(get_domain_from_id(event.state_key)) + break + sharded_destinations = { d for d in destinations @@ -639,14 +704,19 @@ class FederationSender(AbstractFederationSender): ts = event_to_received_ts[event.event_id] assert ts is not None synapse.metrics.event_processing_lag_by_event.labels( - "federation_sender" + name="federation_sender", + **{SERVER_NAME_LABEL: self.server_name}, ).observe((now - ts) / 1000) async def handle_room_events(events: List[EventBase]) -> None: logger.debug( "Handling %i events in room %s", len(events), events[0].room_id ) - with Measure(self.clock, "handle_room_events"): + with Measure( + self.clock, + name="handle_room_events", + server_name=self.server_name, + ): for event in events: await handle_event(event) @@ -679,22 +749,30 @@ class FederationSender(AbstractFederationSender): assert ts is not None synapse.metrics.event_processing_lag.labels( - "federation_sender" + name="federation_sender", + **{SERVER_NAME_LABEL: self.server_name}, ).set(now - ts) synapse.metrics.event_processing_last_ts.labels( - "federation_sender" + name="federation_sender", + **{SERVER_NAME_LABEL: self.server_name}, ).set(ts) - events_processed_counter.inc(len(event_entries)) + events_processed_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(len(event_entries)) - event_processing_loop_room_count.labels("federation_sender").inc( - len(events_by_room) - ) + event_processing_loop_room_count.labels( + name="federation_sender", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(len(events_by_room)) - event_processing_loop_counter.labels("federation_sender").inc() + event_processing_loop_counter.labels( + name="federation_sender", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() synapse.metrics.event_processing_positions.labels( - "federation_sender" + name="federation_sender", **{SERVER_NAME_LABEL: self.server_name} ).set(next_token) finally: @@ -712,14 +790,28 @@ class FederationSender(AbstractFederationSender): if not destinations: return - sent_pdus_destination_dist_total.inc(len(destinations)) - sent_pdus_destination_dist_count.inc() + sent_pdus_destination_dist_total.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(len(destinations)) + sent_pdus_destination_dist_count.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() assert pdu.internal_metadata.stream_ordering # track the fact that we have a PDU for these destinations, # to allow us to perform catch-up later on if the remote is unreachable # for a while. + # Filter out any destinations not present in the federation_domain_whitelist, if + # the whitelist exists. These destinations should not be sent to so let's not + # waste time or space keeping track of events destined for them. + destinations = [ + d + for d in destinations + if self.hs.config.federation.is_domain_allowed_according_to_federation_whitelist( + d + ) + ] await self.store.store_destination_rooms_entries( destinations, pdu.room_id, @@ -734,7 +826,12 @@ class FederationSender(AbstractFederationSender): ) for destination in destinations: - self._get_per_destination_queue(destination).send_pdu(pdu) + queue = self._get_per_destination_queue(destination) + # We expect `queue` to not be None as we already filtered out + # non-whitelisted destinations above. + assert queue is not None + + queue.send_pdu(pdu) async def send_read_receipt(self, receipt: ReadReceipt) -> None: """Send a RR to any other servers in the room @@ -745,37 +842,48 @@ class FederationSender(AbstractFederationSender): # Some background on the rate-limiting going on here. # - # It turns out that if we attempt to send out RRs as soon as we get them from - # a client, then we end up trying to do several hundred Hz of federation - # transactions. (The number of transactions scales as O(N^2) on the size of a - # room, since in a large room we have both more RRs coming in, and more servers - # to send them to.) + # It turns out that if we attempt to send out RRs as soon as we get them + # from a client, then we end up trying to do several hundred Hz of + # federation transactions. (The number of transactions scales as O(N^2) + # on the size of a room, since in a large room we have both more RRs + # coming in, and more servers to send them to.) # - # This leads to a lot of CPU load, and we end up getting behind. The solution - # currently adopted is as follows: + # This leads to a lot of CPU load, and we end up getting behind. The + # solution currently adopted is to differentiate between receipts and + # destinations we should immediately send to, and those we can trickle + # the receipts to. # - # The first receipt in a given room is sent out immediately, at time T0. Any - # further receipts are, in theory, batched up for N seconds, where N is calculated - # based on the number of servers in the room to achieve a transaction frequency - # of around 50Hz. So, for example, if there were 100 servers in the room, then - # N would be 100 / 50Hz = 2 seconds. + # The current logic is to send receipts out immediately if: + # - the room is "small", i.e. there's only N servers to send receipts + # to, and so sending out the receipts immediately doesn't cause too + # much load; or + # - the receipt is for an event that happened recently, as users + # notice if receipts are delayed when they know other users are + # currently reading the room; or + # - the receipt is being sent to the server that sent the event, so + # that users see receipts for their own receipts quickly. # - # Then, after T+N, we flush out any receipts that have accumulated, and restart - # the timer to flush out more receipts at T+2N, etc. If no receipts accumulate, - # we stop the cycle and go back to the start. + # For destinations that we should delay sending the receipt to, we queue + # the receipts up to be sent in the next transaction, but don't trigger + # a new transaction to be sent. We then add the destination to the + # `DestinationWakeupQueue`, which will slowly iterate over each + # destination and trigger a new transaction to be sent. # - # However, in practice, it is often possible to flush out receipts earlier: in - # particular, if we are sending a transaction to a given server anyway (for - # example, because we have a PDU or a RR in another room to send), then we may - # as well send out all of the pending RRs for that server. So it may be that - # by the time we get to T+N, we don't actually have any RRs left to send out. - # Nevertheless we continue to buffer up RRs for the room in question until we - # reach the point that no RRs arrive between timer ticks. + # However, in practice, it is often possible to send out delayed + # receipts earlier: in particular, if we are sending a transaction to a + # given server anyway (for example, because we have a PDU or a RR in + # another room to send), then we may as well send out all of the pending + # RRs for that server. So it may be that by the time we get to waking up + # the destination, we don't actually have any RRs left to send out. # - # For even more background, see https://github.com/matrix-org/synapse/issues/4730. + # For even more background, see + # https://github.com/matrix-org/synapse/issues/4730. room_id = receipt.room_id + # Local read receipts always have 1 event ID. + event_id = receipt.event_ids[0] + # Work out which remote servers should be poked and poke them. domains_set = await self._storage_controllers.state.get_current_hosts_in_room_or_partial_state_approximation( room_id @@ -797,49 +905,55 @@ class FederationSender(AbstractFederationSender): if not domains: return - queues_pending_flush = self._queues_awaiting_rr_flush_by_room.get(room_id) + # We now split which domains we want to wake up immediately vs which we + # want to delay waking up. + immediate_domains: StrCollection + delay_domains: StrCollection - # if there is no flush yet scheduled, we will send out these receipts with - # immediate flushes, and schedule the next flush for this room. - if queues_pending_flush is not None: - logger.debug("Queuing receipt for: %r", domains) + if len(domains) < 10: + # For "small" rooms send to all domains immediately + immediate_domains = domains + delay_domains = () else: - logger.debug("Sending receipt to: %r", domains) - self._schedule_rr_flush_for_room(room_id, len(domains)) + metadata = await self.store.get_metadata_for_event( + receipt.room_id, event_id + ) + assert metadata is not None - for domain in domains: + sender_domain = get_domain_from_id(metadata.sender) + + if self.clock.time_msec() - metadata.received_ts < 60_000: + # We always send receipts for recent messages immediately + immediate_domains = domains + delay_domains = () + else: + # Otherwise, we delay waking up all destinations except for the + # sender's domain. + immediate_domains = [] + delay_domains = [] + for domain in domains: + if domain == sender_domain: + immediate_domains.append(domain) + else: + delay_domains.append(domain) + + for domain in immediate_domains: + # Add to destination queue and wake the destination up queue = self._get_per_destination_queue(domain) + if queue is None: + continue + queue.queue_read_receipt(receipt) + queue.attempt_new_transaction() + + for domain in delay_domains: + # Add to destination queue... + queue = self._get_per_destination_queue(domain) + if queue is None: + continue queue.queue_read_receipt(receipt) - # if there is already a RR flush pending for this room, then make sure this - # destination is registered for the flush - if queues_pending_flush is not None: - queues_pending_flush.add(queue) - else: - queue.flush_read_receipts_for_room(room_id) - - def _schedule_rr_flush_for_room(self, room_id: str, n_domains: int) -> None: - # that is going to cause approximately len(domains) transactions, so now back - # off for that multiplied by RR_TXN_INTERVAL_PER_ROOM - backoff_ms = self._rr_txn_interval_per_room_ms * n_domains - - logger.debug("Scheduling RR flush in %s in %d ms", room_id, backoff_ms) - self.clock.call_later(backoff_ms, self._flush_rrs_for_room, room_id) - self._queues_awaiting_rr_flush_by_room[room_id] = set() - - def _flush_rrs_for_room(self, room_id: str) -> None: - queues = self._queues_awaiting_rr_flush_by_room.pop(room_id) - logger.debug("Flushing RRs in %s to %s", room_id, queues) - - if not queues: - # no more RRs arrived for this room; we are done. - return - - # schedule the next flush - self._schedule_rr_flush_for_room(room_id, len(queues)) - - for queue in queues: - queue.flush_read_receipts_for_room(room_id) + # ... and schedule the destination to be woken up. + self._destination_wakeup_queue.add_to_queue(domain) async def send_presence_to_destinations( self, states: Iterable[UserPresenceState], destinations: Iterable[str] @@ -871,9 +985,10 @@ class FederationSender(AbstractFederationSender): if self.is_mine_server_name(destination): continue - self._get_per_destination_queue(destination).send_presence( - states, start_loop=False - ) + queue = self._get_per_destination_queue(destination) + if queue is None: + continue + queue.send_presence(states, start_loop=False) self._destination_wakeup_queue.add_to_queue(destination) @@ -923,6 +1038,8 @@ class FederationSender(AbstractFederationSender): return queue = self._get_per_destination_queue(edu.destination) + if queue is None: + return if key: queue.send_keyed_edu(edu, key) else: @@ -947,9 +1064,15 @@ class FederationSender(AbstractFederationSender): for destination in destinations: if immediate: - self._get_per_destination_queue(destination).attempt_new_transaction() + queue = self._get_per_destination_queue(destination) + if queue is None: + continue + queue.attempt_new_transaction() else: - self._get_per_destination_queue(destination).mark_new_data() + queue = self._get_per_destination_queue(destination) + if queue is None: + continue + queue.mark_new_data() self._destination_wakeup_queue.add_to_queue(destination) def wake_destination(self, destination: str) -> None: @@ -968,7 +1091,9 @@ class FederationSender(AbstractFederationSender): ): return - self._get_per_destination_queue(destination).attempt_new_transaction() + queue = self._get_per_destination_queue(destination) + if queue is not None: + queue.attempt_new_transaction() @staticmethod def get_current_token() -> int: @@ -1013,6 +1138,9 @@ class FederationSender(AbstractFederationSender): d for d in destinations_to_wake if self._federation_shard_config.should_handle(self._instance_name, d) + and self.hs.config.federation.is_domain_allowed_according_to_federation_whitelist( + d + ) ] for destination in destinations_to_wake: diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index d097e65ea7..4c844d403a 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -40,7 +40,7 @@ from synapse.federation.units import Edu from synapse.handlers.presence import format_user_presence_state from synapse.logging import issue9533_logger from synapse.logging.opentracing import SynapseTags, set_tag -from synapse.metrics import sent_transactions_counter +from synapse.metrics import SERVER_NAME_LABEL, sent_transactions_counter from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import JsonDict, ReadReceipt from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter @@ -56,13 +56,15 @@ logger = logging.getLogger(__name__) sent_edus_counter = Counter( - "synapse_federation_client_sent_edus", "Total number of EDUs successfully sent" + "synapse_federation_client_sent_edus", + "Total number of EDUs successfully sent", + labelnames=[SERVER_NAME_LABEL], ) sent_edus_by_type = Counter( "synapse_federation_client_sent_edus_by_type", "Number of sent EDUs successfully sent, by event type", - ["type"], + labelnames=["type", SERVER_NAME_LABEL], ) @@ -91,7 +93,7 @@ class PerDestinationQueue: transaction_manager: "synapse.federation.sender.TransactionManager", destination: str, ): - self._server_name = hs.hostname + self.server_name = hs.hostname self._clock = hs.get_clock() self._storage_controllers = hs.get_storage_controllers() self._store = hs.get_datastores().main @@ -129,6 +131,8 @@ class PerDestinationQueue: # The stream_ordering of the most recent PDU that was discarded due to # being in catch-up mode. + # Can be set to zero if no PDU has been discarded since the last time + # we queried for new PDUs during catch-up. self._catchup_last_skipped: int = 0 # Cache of the last successfully-transmitted stream ordering for this @@ -156,7 +160,6 @@ class PerDestinationQueue: # Each receipt can only have a single receipt per # (room ID, receipt type, user ID, thread ID) tuple. self._pending_receipt_edus: List[Dict[str, Dict[str, Dict[str, dict]]]] = [] - self._rrs_pending_flush = False # stream_id of last successfully sent to-device message. # NB: may be a long or an int. @@ -258,15 +261,7 @@ class PerDestinationQueue: } ) - def flush_read_receipts_for_room(self, room_id: str) -> None: - # If there are any pending receipts for this room then force-flush them - # in a new transaction. - for edu in self._pending_receipt_edus: - if room_id in edu: - self._rrs_pending_flush = True - self.attempt_new_transaction() - # No use in checking remaining EDUs if the room was found. - break + self.mark_new_data() def send_keyed_edu(self, edu: Edu, key: Hashable) -> None: self._pending_edus_keyed[(edu.edu_type, key)] = edu @@ -318,6 +313,7 @@ class PerDestinationQueue: run_as_background_process( "federation_transaction_transmission_loop", + self.server_name, self._transaction_transmission_loop, ) @@ -329,7 +325,12 @@ class PerDestinationQueue: # This will throw if we wouldn't retry. We do this here so we fail # quickly, but we will later check this again in the http client, # hence why we throw the result away. - await get_retry_limiter(self._destination, self._clock, self._store) + await get_retry_limiter( + destination=self._destination, + our_server_name=self.server_name, + clock=self._clock, + store=self._store, + ) if self._catching_up: # we potentially need to catch-up first @@ -369,10 +370,17 @@ class PerDestinationQueue: self._destination, pending_pdus, pending_edus ) - sent_transactions_counter.inc() - sent_edus_counter.inc(len(pending_edus)) + sent_transactions_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() + sent_edus_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(len(pending_edus)) for edu in pending_edus: - sent_edus_by_type.labels(edu.edu_type).inc() + sent_edus_by_type.labels( + type=edu.edu_type, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() except NotRetryingDestination as e: logger.debug( @@ -471,8 +479,18 @@ class PerDestinationQueue: # of a race condition, so we check that no new events have been # skipped due to us being in catch-up mode - if self._catchup_last_skipped > last_successful_stream_ordering: + if ( + self._catchup_last_skipped != 0 + and self._catchup_last_skipped > last_successful_stream_ordering + ): # another event has been skipped because we were in catch-up mode + # As an exception to this case: we can hit this branch if the + # room has been purged whilst we have been looping. + # In that case we avoid hot-looping by resetting the 'catch-up skipped + # PDU' flag. + # Then if there is still no progress to be made at the next iteration, + # we can exit catch-up mode. + self._catchup_last_skipped = 0 continue # we are done catching up! @@ -563,7 +581,7 @@ class PerDestinationQueue: new_pdus = await filter_events_for_server( self._storage_controllers, self._destination, - self._server_name, + self.server_name, new_pdus, redact=False, filter_out_erased_senders=True, @@ -587,7 +605,9 @@ class PerDestinationQueue: self._destination, room_catchup_pdus, [] ) - sent_transactions_counter.inc() + sent_transactions_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() # We pulled this from the DB, so it'll be non-null assert pdu.internal_metadata.stream_ordering @@ -603,17 +623,14 @@ class PerDestinationQueue: self._destination, last_successful_stream_ordering ) - def _get_receipt_edus(self, force_flush: bool, limit: int) -> Iterable[Edu]: + def _get_receipt_edus(self, limit: int) -> Iterable[Edu]: if not self._pending_receipt_edus: return - if not force_flush and not self._rrs_pending_flush: - # not yet time for this lot - return # Send at most limit EDUs for receipts. for content in self._pending_receipt_edus[:limit]: yield Edu( - origin=self._server_name, + origin=self.server_name, destination=self._destination, edu_type=EduTypes.RECEIPT, content=content, @@ -639,7 +656,7 @@ class PerDestinationQueue: ) edus = [ Edu( - origin=self._server_name, + origin=self.server_name, destination=self._destination, edu_type=edu_type, content=content, @@ -666,7 +683,7 @@ class PerDestinationQueue: edus = [ Edu( - origin=self._server_name, + origin=self.server_name, destination=self._destination, edu_type=EduTypes.DIRECT_TO_DEVICE, content=content, @@ -739,7 +756,7 @@ class _TransactionQueueManager: pending_edus.append( Edu( - origin=self.queue._server_name, + origin=self.queue.server_name, destination=self.queue._destination, edu_type=EduTypes.PRESENCE, content={"push": presence_to_add}, @@ -747,7 +764,7 @@ class _TransactionQueueManager: ) # Add read receipt EDUs. - pending_edus.extend(self.queue._get_receipt_edus(force_flush=False, limit=5)) + pending_edus.extend(self.queue._get_receipt_edus(limit=5)) edu_limit = MAX_EDUS_PER_TRANSACTION - len(pending_edus) # Next, prioritize to-device messages so that existing encryption channels @@ -795,13 +812,6 @@ class _TransactionQueueManager: if not self._pdus and not pending_edus: return [], [] - # if we've decided to send a transaction anyway, and we have room, we - # may as well send any pending RRs - if edu_limit: - pending_edus.extend( - self.queue._get_receipt_edus(force_flush=True, limit=edu_limit) - ) - if self._pdus: self._last_stream_ordering = self._pdus[ -1 diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index d8a3eaa525..050982c499 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -26,7 +26,7 @@ from synapse.api.constants import EduTypes from synapse.api.errors import HttpResponseException from synapse.events import EventBase from synapse.federation.persistence import TransactionActions -from synapse.federation.units import Edu, Transaction +from synapse.federation.units import Edu, Transaction, serialize_and_filter_pdus from synapse.logging.opentracing import ( extract_text_map, set_tag, @@ -34,6 +34,7 @@ from synapse.logging.opentracing import ( tags, whitelisted_homeserver, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.types import JsonDict from synapse.util import json_decoder from synapse.util.metrics import measure_func @@ -47,7 +48,7 @@ issue_8631_logger = logging.getLogger("synapse.8631_debug") last_pdu_ts_metric = Gauge( "synapse_federation_last_sent_pdu_time", "The timestamp of the last PDU which was successfully sent to the given domain", - labelnames=("server_name",), + labelnames=("destination_server_name", SERVER_NAME_LABEL), ) @@ -58,7 +59,7 @@ class TransactionManager: """ def __init__(self, hs: "synapse.server.HomeServer"): - self._server_name = hs.hostname + self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func self._store = hs.get_datastores().main self._transaction_actions = TransactionActions(self._store) @@ -116,9 +117,9 @@ class TransactionManager: transaction = Transaction( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, - origin=self._server_name, + origin=self.server_name, destination=destination, - pdus=[p.get_pdu_json() for p in pdus], + pdus=serialize_and_filter_pdus(pdus), edus=[edu.get_dict() for edu in edus], ) @@ -191,6 +192,7 @@ class TransactionManager: if pdus and destination in self._federation_metrics_domains: last_pdu = pdus[-1] - last_pdu_ts_metric.labels(server_name=destination).set( - last_pdu.origin_server_ts / 1000 - ) + last_pdu_ts_metric.labels( + destination_server_name=destination, + **{SERVER_NAME_LABEL: self.server_name}, + ).set(last_pdu.origin_server_ts / 1000) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 206e91ed14..62bf96ce91 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -143,6 +143,33 @@ class TransportLayerClient: destination, path=path, timeout=timeout, try_trailing_slash_on_400=True ) + async def get_policy_recommendation_for_pdu( + self, destination: str, event: EventBase, timeout: Optional[int] = None + ) -> JsonDict: + """Requests the policy recommendation for the given pdu from the given policy server. + + Args: + destination: The host name of the remote homeserver checking the event. + event: The event to check. + timeout: How long to try (in ms) the destination for before giving up. + None indicates no timeout. + + Returns: + The full recommendation object from the remote server. + """ + logger.debug( + "get_policy_recommendation_for_pdu dest=%s, event_id=%s", + destination, + event.event_id, + ) + return await self.client.post_json( + destination=destination, + path=f"/_matrix/policy/unstable/org.matrix.msc4284/event/{event.event_id}/check", + data=event.get_pdu_json(), + ignore_backoff=True, + timeout=timeout, + ) + async def backfill( self, destination: str, room_id: str, event_tuples: Collection[str], limit: int ) -> Optional[Union[JsonDict, list]]: diff --git a/synapse/federation/transport/server/__init__.py b/synapse/federation/transport/server/__init__.py index 43102567db..c4905e63dd 100644 --- a/synapse/federation/transport/server/__init__.py +++ b/synapse/federation/transport/server/__init__.py @@ -20,9 +20,7 @@ # # import logging -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Type - -from typing_extensions import Literal +from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Tuple, Type from synapse.api.errors import FederationDeniedError, SynapseError from synapse.federation.transport.server._base import ( @@ -137,7 +135,7 @@ class PublicRoomList(BaseFederationServlet): if not self.allow_access: raise FederationDeniedError(origin) - limit = parse_integer_from_args(query, "limit", 0) + limit: Optional[int] = parse_integer_from_args(query, "limit", 0) since_token = parse_string_from_args(query, "since", None) include_all_networks = parse_boolean_from_args( query, "include_all_networks", default=False diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index a05e5d5319..eb96ff27f9 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -24,6 +24,7 @@ from typing import ( TYPE_CHECKING, Dict, List, + Literal, Mapping, Optional, Sequence, @@ -32,8 +33,6 @@ from typing import ( Union, ) -from typing_extensions import Literal - from synapse.api.constants import Direction, EduTypes from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersions @@ -509,6 +508,9 @@ class FederationV2InviteServlet(BaseFederationServerServlet): event = content["event"] invite_room_state = content.get("invite_room_state", []) + if not isinstance(invite_room_state, list): + invite_room_state = [] + # Synapse expects invite_room_state to be in unsigned, as it is in v1 # API diff --git a/synapse/federation/units.py b/synapse/federation/units.py index d8b67a6a5b..3bb5f824b7 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -24,10 +24,12 @@ server protocol. """ import logging -from typing import List, Optional +from typing import List, Optional, Sequence import attr +from synapse.api.constants import CANONICALJSON_MAX_INT, CANONICALJSON_MIN_INT +from synapse.events import EventBase from synapse.types import JsonDict logger = logging.getLogger(__name__) @@ -104,8 +106,28 @@ class Transaction: result = { "origin": self.origin, "origin_server_ts": self.origin_server_ts, - "pdus": self.pdus, + "pdus": filter_pdus_for_valid_depth(self.pdus), } if self.edus: result["edus"] = self.edus return result + + +def filter_pdus_for_valid_depth(pdus: Sequence[JsonDict]) -> List[JsonDict]: + filtered_pdus = [] + for pdu in pdus: + # Drop PDUs that have a depth that is outside of the range allowed + # by canonical json. + if ( + "depth" in pdu + and CANONICALJSON_MIN_INT <= pdu["depth"] <= CANONICALJSON_MAX_INT + ): + filtered_pdus.append(pdu) + + return filtered_pdus + + +def serialize_and_filter_pdus( + pdus: Sequence[EventBase], time_now: Optional[int] = None +) -> List[JsonDict]: + return filter_pdus_for_valid_depth([pdu.get_pdu_json(time_now) for pdu in pdus]) diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 7004d95a0f..39a22b8cbb 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -38,6 +38,9 @@ logger = logging.getLogger(__name__) class AccountValidityHandler: def __init__(self, hs: "HomeServer"): self.hs = hs + self.server_name = ( + hs.hostname + ) # nb must be called this for @wrap_as_background_process self.config = hs.config self.store = hs.get_datastores().main self.send_email_handler = hs.get_send_email_handler() diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index d1989e9d2c..e90d675b59 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -124,6 +124,7 @@ class AdminHandler: "consent_ts": user_info.consent_ts, "user_type": user_info.user_type, "is_guest": user_info.is_guest, + "suspended": user_info.suspended, } if self._msc3866_enabled: @@ -357,6 +358,7 @@ class AdminHandler: user_id: str, rooms: list, requester: JsonMapping, + use_admin: bool, reason: Optional[str], limit: Optional[int], ) -> str: @@ -367,6 +369,7 @@ class AdminHandler: user_id: the user ID of the user whose events should be redacted rooms: the rooms in which to redact the user's events requester: the user requesting the events + use_admin: whether to use the admin account to issue the redactions reason: reason for requesting the redaction, ie spam, etc limit: limit on the number of events in each room to redact @@ -394,6 +397,7 @@ class AdminHandler: "rooms": rooms, "requester": requester, "user_id": user_id, + "use_admin": use_admin, "reason": reason, "limit": limit, }, @@ -425,9 +429,17 @@ class AdminHandler: user_id = task.params.get("user_id") assert user_id is not None - # puppet the user if they're ours, otherwise use admin to redact + use_admin = task.params.get("use_admin", False) + + # default to puppeting the user unless they are not local or it's been requested to + # use the admin user to issue the redactions + requester_id = ( + admin.user.to_string() + if use_admin or not self.hs.is_mine_id(user_id) + else user_id + ) requester = create_requester( - user_id if self.hs.is_mine_id(user_id) else admin.user.to_string(), + requester_id, authenticated_entity=admin.user.to_string(), ) @@ -444,7 +456,7 @@ class AdminHandler: user_id, room, limit, - ["m.room.member", "m.room.message"], + ["m.room.member", "m.room.message", "m.room.encrypted"], ) if not event_ids: # nothing to redact in this room @@ -472,7 +484,7 @@ class AdminHandler: "type": EventTypes.Redaction, "content": {"reason": reason} if reason else {}, "room_id": room, - "sender": user_id, + "sender": requester.user.to_string(), } if room_version.updated_redaction_rules: event_dict["content"]["redacts"] = event.event_id @@ -494,7 +506,7 @@ class AdminHandler: ) except Exception as ex: logger.info( - f"Redaction of event {event.event_id} failed due to: {ex}" + "Redaction of event %s failed due to: %s", event.event_id, ex ) result["failed_redactions"][event.event_id] = str(ex) await self._task_scheduler.update_task(task.id, result=result) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 4b33e1330d..5bd239e5fe 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -42,6 +42,7 @@ from synapse.events import EventBase from synapse.handlers.presence import format_user_presence_state from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.metrics import ( + SERVER_NAME_LABEL, event_processing_loop_counter, event_processing_loop_room_count, ) @@ -68,11 +69,16 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -events_processed_counter = Counter("synapse_handlers_appservice_events_processed", "") +events_processed_counter = Counter( + "synapse_handlers_appservice_events_processed", "", labelnames=[SERVER_NAME_LABEL] +) class ApplicationServicesHandler: def __init__(self, hs: "HomeServer"): + self.server_name = ( + hs.hostname + ) # nb must be called this for @wrap_as_background_process self.store = hs.get_datastores().main self.is_mine_id = hs.is_mine_id self.appservice_api = hs.get_application_service_api() @@ -120,7 +126,9 @@ class ApplicationServicesHandler: @wrap_as_background_process("notify_interested_services") async def _notify_interested_services(self, max_token: RoomStreamToken) -> None: - with Measure(self.clock, "notify_interested_services"): + with Measure( + self.clock, name="notify_interested_services", server_name=self.server_name + ): self.is_processing = True try: upper_bound = -1 @@ -163,7 +171,9 @@ class ApplicationServicesHandler: except Exception: logger.error("Application Services Failure") - run_as_background_process("as_scheduler", start_scheduler) + run_as_background_process( + "as_scheduler", self.server_name, start_scheduler + ) self.started_scheduler = True # Fork off pushes to these services @@ -177,7 +187,8 @@ class ApplicationServicesHandler: assert ts is not None synapse.metrics.event_processing_lag_by_event.labels( - "appservice_sender" + name="appservice_sender", + **{SERVER_NAME_LABEL: self.server_name}, ).observe((now - ts) / 1000) async def handle_room_events(events: Iterable[EventBase]) -> None: @@ -197,16 +208,23 @@ class ApplicationServicesHandler: await self.store.set_appservice_last_pos(upper_bound) synapse.metrics.event_processing_positions.labels( - "appservice_sender" + name="appservice_sender", + **{SERVER_NAME_LABEL: self.server_name}, ).set(upper_bound) - events_processed_counter.inc(len(events)) + events_processed_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(len(events)) - event_processing_loop_room_count.labels("appservice_sender").inc( - len(events_by_room) - ) + event_processing_loop_room_count.labels( + name="appservice_sender", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(len(events_by_room)) - event_processing_loop_counter.labels("appservice_sender").inc() + event_processing_loop_counter.labels( + name="appservice_sender", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() if events: now = self.clock.time_msec() @@ -214,10 +232,12 @@ class ApplicationServicesHandler: assert ts is not None synapse.metrics.event_processing_lag.labels( - "appservice_sender" + name="appservice_sender", + **{SERVER_NAME_LABEL: self.server_name}, ).set(now - ts) synapse.metrics.event_processing_last_ts.labels( - "appservice_sender" + name="appservice_sender", + **{SERVER_NAME_LABEL: self.server_name}, ).set(ts) finally: self.is_processing = False @@ -329,7 +349,11 @@ class ApplicationServicesHandler: users: Collection[Union[str, UserID]], ) -> None: logger.debug("Checking interested services for %s", stream_key) - with Measure(self.clock, "notify_interested_services_ephemeral"): + with Measure( + self.clock, + name="notify_interested_services_ephemeral", + server_name=self.server_name, + ): for service in services: if stream_key == StreamKeyType.TYPING: # Note that we don't persist the token (via set_appservice_stream_type_pos) @@ -465,9 +489,7 @@ class ApplicationServicesHandler: service, "read_receipt" ) if new_token is not None and new_token.stream <= from_key: - logger.debug( - "Rejecting token lower than or equal to stored: %s" % (new_token,) - ) + logger.debug("Rejecting token lower than or equal to stored: %s", new_token) return [] from_token = MultiWriterStreamToken(stream=from_key) @@ -509,9 +531,7 @@ class ApplicationServicesHandler: service, "presence" ) if new_token is not None and new_token <= from_key: - logger.debug( - "Rejecting token lower than or equal to stored: %s" % (new_token,) - ) + logger.debug("Rejecting token lower than or equal to stored: %s", new_token) return [] for user in users: @@ -635,7 +655,8 @@ class ApplicationServicesHandler: # Fetch the users who have modified their device list since then. users_with_changed_device_lists = await self.store.get_all_devices_changed( - from_key, to_key=new_key + MultiWriterStreamToken(stream=from_key), + to_key=MultiWriterStreamToken(stream=new_key), ) # Filter out any users the application service is not interested in @@ -843,7 +864,7 @@ class ApplicationServicesHandler: # user not found; could be the AS though, so check. services = self.store.get_app_services() - service_list = [s for s in services if s.sender == user_id] + service_list = [s for s in services if s.sender.to_string() == user_id] return len(service_list) == 0 async def _check_user_exists(self, user_id: str) -> bool: @@ -896,10 +917,10 @@ class ApplicationServicesHandler: results = await make_deferred_yieldable( defer.DeferredList( [ - run_in_background( + run_in_background( # type: ignore[call-overload] self.appservice_api.claim_client_keys, # We know this must be an app service. - self.store.get_app_service_by_id(service_id), # type: ignore[arg-type] + self.store.get_app_service_by_id(service_id), service_query, ) for service_id, service_query in query_by_appservice.items() @@ -952,10 +973,10 @@ class ApplicationServicesHandler: results = await make_deferred_yieldable( defer.DeferredList( [ - run_in_background( + run_in_background( # type: ignore[call-overload] self.appservice_api.query_keys, # We know this must be an app service. - self.store.get_app_service_by_id(service_id), # type: ignore[arg-type] + self.store.get_app_service_by_id(service_id), service_query, ) for service_id, service_query in query_by_appservice.items() diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1f4264ad7e..2d1990cce5 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -70,13 +70,14 @@ from synapse.http import get_request_user_agent from synapse.http.server import finish_request, respond_with_html from synapse.http.site import SynapseRequest from synapse.logging.context import defer_to_thread +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main.registration import ( LoginTokenExpired, LoginTokenLookupResult, LoginTokenReused, ) -from synapse.types import JsonDict, Requester, UserID +from synapse.types import JsonDict, Requester, StrCollection, UserID from synapse.util import stringutils as stringutils from synapse.util.async_helpers import delay_cancellation, maybe_awaitable from synapse.util.msisdn import phone_number_to_msisdn @@ -95,7 +96,7 @@ INVALID_USERNAME_OR_PASSWORD = "Invalid username or password" invalid_login_token_counter = Counter( "synapse_user_login_invalid_login_tokens", "Counts the number of rejected m.login.token on /login", - ["reason"], + labelnames=["reason", SERVER_NAME_LABEL], ) @@ -174,6 +175,7 @@ def login_id_phone_to_thirdparty(identifier: JsonDict) -> Dict[str, str]: # Accept both "phone" and "number" as valid keys in m.id.phone phone_number = identifier.get("phone", identifier["number"]) + assert isinstance(phone_number, str) # Convert user-provided phone number to a consistent representation msisdn = phone_number_to_msisdn(identifier["country"], phone_number) @@ -198,6 +200,7 @@ class AuthHandler: SESSION_EXPIRE_MS = 48 * 60 * 60 * 1000 def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.auth = hs.get_auth() self.auth_blocking = hs.get_auth_blocking() @@ -219,6 +222,7 @@ class AuthHandler: self._password_localdb_enabled = hs.config.auth.password_localdb_enabled self._third_party_rules = hs.get_module_api_callbacks().third_party_event_rules self._account_validity_handler = hs.get_account_validity_handler() + self._pusher_pool = hs.get_pusherpool() # Ratelimiter for failed auth during UIA. Uses same ratelimit config # as per `rc_login.failed_attempts`. @@ -246,6 +250,7 @@ class AuthHandler: run_as_background_process, 5 * 60 * 1000, "expire_old_sessions", + self.server_name, self._expire_old_sessions, ) @@ -270,8 +275,6 @@ class AuthHandler: hs.config.sso.sso_account_deactivated_template ) - self._server_name = hs.config.server.server_name - # cast to tuple for use with str.startswith self._whitelisted_sso_clients = tuple(hs.config.sso.sso_client_whitelist) @@ -279,7 +282,9 @@ class AuthHandler: # response. self._extra_attributes: Dict[str, SsoLoginExtraAttributes] = {} - self.msc3861_oauth_delegation_enabled = hs.config.experimental.msc3861.enabled + self._auth_delegation_enabled = ( + hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + ) async def validate_user_via_ui_auth( self, @@ -330,7 +335,7 @@ class AuthHandler: LimitExceededError if the ratelimiter's failed request count for this user is too high to proceed """ - if self.msc3861_oauth_delegation_enabled: + if self._auth_delegation_enabled: raise SynapseError( HTTPStatus.INTERNAL_SERVER_ERROR, "UIA shouldn't be used with MSC3861" ) @@ -1477,11 +1482,20 @@ class AuthHandler: try: return await self.store.consume_login_token(login_token) except LoginTokenExpired: - invalid_login_token_counter.labels("expired").inc() + invalid_login_token_counter.labels( + reason="expired", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() except LoginTokenReused: - invalid_login_token_counter.labels("reused").inc() + invalid_login_token_counter.labels( + reason="reused", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() except NotFoundError: - invalid_login_token_counter.labels("not found").inc() + invalid_login_token_counter.labels( + reason="not found", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() raise AuthError(403, "Invalid login token", errcode=Codes.FORBIDDEN) @@ -1547,6 +1561,31 @@ class AuthHandler: user_id, (token_id for _, token_id, _ in tokens_and_devices) ) + async def delete_access_tokens_for_devices( + self, + user_id: str, + device_ids: StrCollection, + ) -> None: + """Invalidate access tokens for the devices + + Args: + user_id: ID of user the tokens belong to + device_ids: ID of device the tokens are associated with. + If None, tokens associated with any device (or no device) will + be deleted + """ + tokens_and_devices = await self.store.user_delete_access_tokens_for_devices( + user_id, + device_ids, + ) + + # see if any modules want to know about this + if self.password_auth_provider.on_logged_out_callbacks: + for token, _, device_id in tokens_and_devices: + await self.password_auth_provider.on_logged_out( + user_id=user_id, device_id=device_id, access_token=token + ) + async def add_threepid( self, user_id: str, medium: str, address: str, validated_at: int ) -> None: @@ -1579,7 +1618,10 @@ class AuthHandler: # for the presence of an email address during password reset was # case sensitive). if medium == "email": - address = canonicalise_email(address) + try: + address = canonicalise_email(address) + except ValueError as e: + raise SynapseError(400, str(e)) await self.store.user_add_threepid( user_id, medium, address, validated_at, self.hs.get_clock().time_msec() @@ -1610,7 +1652,10 @@ class AuthHandler: """ # 'Canonicalise' email addresses as per above if medium == "email": - address = canonicalise_email(address) + try: + address = canonicalise_email(address) + except ValueError as e: + raise SynapseError(400, str(e)) await self.store.user_delete_threepid(user_id, medium, address) @@ -1620,7 +1665,7 @@ class AuthHandler: ) if medium == "email": - await self.store.delete_pusher_by_app_id_pushkey_user_id( + await self._pusher_pool.remove_pusher( app_id="m.email", pushkey=address, user_id=user_id ) @@ -1825,7 +1870,7 @@ class AuthHandler: html = self._sso_redirect_confirm_template.render( display_url=display_url, redirect_url=redirect_url, - server_name=self._server_name, + server_name=self.server_name, new_user=new_user, user_id=registered_user_id, user_profile=user_profile_data, @@ -1889,7 +1934,7 @@ def load_single_legacy_password_auth_provider( try: provider = module(config=config, account_handler=api) except Exception as e: - logger.error("Error while initializing %r: %s", module, e) + logger.exception("Error while initializing %r: %s", module, e) raise # All methods that the module provides should be async, but this wasn't enforced @@ -2422,7 +2467,7 @@ class PasswordAuthProvider: except CancelledError: raise except Exception as e: - logger.error("Module raised an exception in is_3pid_allowed: %s", e) + logger.exception("Module raised an exception in is_3pid_allowed: %s", e) raise SynapseError(code=500, msg="Internal Server Error") return True diff --git a/synapse/handlers/cas.py b/synapse/handlers/cas.py index cc3d641b7d..fbe79c2e4c 100644 --- a/synapse/handlers/cas.py +++ b/synapse/handlers/cas.py @@ -378,7 +378,8 @@ class CasHandler: # Arbitrarily use the first attribute found. display_name = cas_response.attributes.get( - self._cas_displayname_attribute, [None] + self._cas_displayname_attribute, # type: ignore[arg-type] + [None], )[0] return UserAttributes(localpart=localpart, display_name=display_name) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 12a7cace55..e4169321cc 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -24,8 +24,10 @@ from typing import TYPE_CHECKING, Optional from synapse.api.constants import Membership from synapse.api.errors import SynapseError -from synapse.handlers.device import DeviceHandler from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.replication.http.deactivate_account import ( + ReplicationNotifyAccountDeactivatedServlet, +) from synapse.types import Codes, Requester, UserID, create_requester if TYPE_CHECKING: @@ -40,11 +42,13 @@ class DeactivateAccountHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.hs = hs + self.server_name = hs.hostname self._auth_handler = hs.get_auth_handler() self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() self._identity_handler = hs.get_identity_handler() self._profile_handler = hs.get_profile_handler() + self._pusher_pool = hs.get_pusherpool() self.user_directory_handler = hs.get_user_directory_handler() self._server_name = hs.hostname self._third_party_rules = hs.get_module_api_callbacks().third_party_event_rules @@ -53,10 +57,16 @@ class DeactivateAccountHandler: self._user_parter_running = False self._third_party_rules = hs.get_module_api_callbacks().third_party_event_rules + self._notify_account_deactivated_client = None + # Start the user parter loop so it can resume parting users from rooms where # it left off (if it has work left to do). - if hs.config.worker.run_background_tasks: + if hs.config.worker.worker_app is None: hs.get_reactor().callWhenRunning(self._start_user_parting) + else: + self._notify_account_deactivated_client = ( + ReplicationNotifyAccountDeactivatedServlet.make_client(hs) + ) self._account_validity_enabled = ( hs.config.account_validity.account_validity_enabled @@ -84,10 +94,6 @@ class DeactivateAccountHandler: Returns: True if identity server supports removing threepids, otherwise False. """ - - # This can only be called on the main process. - assert isinstance(self._device_handler, DeviceHandler) - # Check if this user can be deactivated if not await self._third_party_rules.check_can_deactivate_user( user_id, by_admin @@ -96,6 +102,14 @@ class DeactivateAccountHandler: 403, "Deactivation of this user is forbidden", Codes.FORBIDDEN ) + logger.info( + "%s requested deactivation of %s erase_data=%s id_server=%s", + requester.user, + user_id, + erase_data, + id_server, + ) + # FIXME: Theoretically there is a race here wherein user resets # password using threepid. @@ -142,7 +156,7 @@ class DeactivateAccountHandler: # Most of the pushers will have been deleted when we logged out the # associated devices above, but we still need to delete pushers not # associated with devices, e.g. email pushers. - await self.store.delete_all_pushers_for_user(user_id) + await self._pusher_pool.delete_all_pushers_for_user(user_id) # Add the user to a table of users pending deactivation (ie. # removal from all the rooms they're a member of) @@ -166,10 +180,6 @@ class DeactivateAccountHandler: logger.info("Marking %s as erased", user_id) await self.store.mark_user_erased(user_id) - # Now start the process that goes through that list and - # parts users from rooms (if it isn't already running) - self._start_user_parting() - # Reject all pending invites and knocks for the user, so that the # user doesn't show up in the "invited" section of rooms' members list. await self._reject_pending_invites_and_knocks_for_user(user_id) @@ -184,18 +194,43 @@ class DeactivateAccountHandler: # Remove account data (including ignored users and push rules). await self.store.purge_account_data_for_user(user_id) + # Remove thread subscriptions for the user + await self.store.purge_thread_subscription_settings_for_user(user_id) + # Delete any server-side backup keys await self.store.bulk_delete_backup_keys_and_versions_for_user(user_id) + # Notify modules and start the room parting process. + await self.notify_account_deactivated(user_id, by_admin=by_admin) + + return identity_server_supports_unbinding + + async def notify_account_deactivated( + self, + user_id: str, + by_admin: bool = False, + ) -> None: + """Notify modules and start the room parting process. + Goes through replication if this is not the main process. + """ + if self._notify_account_deactivated_client is not None: + await self._notify_account_deactivated_client( + user_id=user_id, + by_admin=by_admin, + ) + return + + # Now start the process that goes through that list and + # parts users from rooms (if it isn't already running) + self._start_user_parting() + # Let modules know the user has been deactivated. await self._third_party_rules.on_user_deactivation_status_changed( user_id, True, - by_admin, + by_admin=by_admin, ) - return identity_server_supports_unbinding - async def _reject_pending_invites_and_knocks_for_user(self, user_id: str) -> None: """Reject pending invites and knocks addressed to a given user ID. @@ -237,7 +272,9 @@ class DeactivateAccountHandler: pending deactivation, if it isn't already running. """ if not self._user_parter_running: - run_as_background_process("user_parter_loop", self._user_parter_loop) + run_as_background_process( + "user_parter_loop", self.server_name, self._user_parter_loop + ) async def _user_parter_loop(self) -> None: """Loop that parts deactivated users from rooms""" diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 3c88a96fd3..a6749801a5 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -19,9 +19,10 @@ from twisted.internet.interfaces import IDelayedCall from synapse.api.constants import EventTypes from synapse.api.errors import ShadowBanError +from synapse.api.ratelimiting import Ratelimiter from synapse.config.workers import MAIN_PROCESS_INSTANCE_NAME from synapse.logging.opentracing import set_tag -from synapse.metrics import event_processing_positions +from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.delayed_events import ( ReplicationAddedDelayedEventRestServlet, @@ -53,14 +54,24 @@ logger = logging.getLogger(__name__) class DelayedEventsHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self._config = hs.config self._clock = hs.get_clock() - self._request_ratelimiter = hs.get_request_ratelimiter() self._event_creation_handler = hs.get_event_creation_handler() self._room_member_handler = hs.get_room_member_handler() + self._request_ratelimiter = hs.get_request_ratelimiter() + + # Ratelimiter for management of existing delayed events, + # keyed by the sending user ID & device ID. + self._delayed_event_mgmt_ratelimiter = Ratelimiter( + store=self._store, + clock=self._clock, + cfg=self._config.ratelimiting.rc_delayed_event_mgmt, + ) + self._next_delayed_event_call: Optional[IDelayedCall] = None # The current position in the current_state_delta stream @@ -99,12 +110,13 @@ class DelayedEventsHandler: # Can send the events in background after having awaited on marking them as processed run_as_background_process( "_send_events", + self.server_name, self._send_events, events, ) self._initialized_from_db = run_as_background_process( - "_schedule_db_events", _schedule_db_events + "_schedule_db_events", self.server_name, _schedule_db_events ) else: self._repl_client = ReplicationAddedDelayedEventRestServlet.make_client(hs) @@ -129,7 +141,9 @@ class DelayedEventsHandler: finally: self._event_processing = False - run_as_background_process("delayed_events.notify_new_event", process) + run_as_background_process( + "delayed_events.notify_new_event", self.server_name, process + ) async def _unsafe_process_new_event(self) -> None: # If self._event_pos is None then means we haven't fetched it from the DB yet @@ -149,7 +163,9 @@ class DelayedEventsHandler: # Loop round handling deltas until we're up to date while True: - with Measure(self._clock, "delayed_events_delta"): + with Measure( + self._clock, name="delayed_events_delta", server_name=self.server_name + ): room_max_stream_ordering = self._store.get_room_max_stream_ordering() if self._event_pos == room_max_stream_ordering: return @@ -175,24 +191,44 @@ class DelayedEventsHandler: self._event_pos = max_pos # Expose current event processing position to prometheus - event_processing_positions.labels("delayed_events").set(max_pos) + event_processing_positions.labels( + name="delayed_events", **{SERVER_NAME_LABEL: self.server_name} + ).set(max_pos) await self._store.update_delayed_events_stream_pos(max_pos) async def _handle_state_deltas(self, deltas: List[StateDelta]) -> None: """ - Process current state deltas to cancel pending delayed events + Process current state deltas to cancel other users' pending delayed events that target the same state. """ for delta in deltas: + if delta.event_id is None: + logger.debug( + "Not handling delta for deleted state: %r %r", + delta.event_type, + delta.state_key, + ) + continue + logger.debug( "Handling: %r %r, %s", delta.event_type, delta.state_key, delta.event_id ) + event = await self._store.get_event(delta.event_id, allow_none=True) + if not event: + continue + sender = UserID.from_string(event.sender) + next_send_ts = await self._store.cancel_delayed_state_events( room_id=delta.room_id, event_type=delta.event_type, state_key=delta.state_key, + not_from_localpart=( + sender.localpart + if sender.domain == self._config.server.server_name + else "" + ), ) if self._next_send_ts_changed(next_send_ts): @@ -227,6 +263,9 @@ class DelayedEventsHandler: Raises: SynapseError: if the delayed event fails validation checks. """ + # Use standard request limiter for scheduling new delayed events. + # TODO: Instead apply ratelimiting based on the scheduled send time. + # See https://github.com/element-hq/synapse/issues/18021 await self._request_ratelimiter.ratelimit(requester) self._event_creation_handler.validator.validate_builder( @@ -285,7 +324,10 @@ class DelayedEventsHandler: NotFoundError: if no matching delayed event could be found. """ assert self._is_master - await self._request_ratelimiter.ratelimit(requester) + await self._delayed_event_mgmt_ratelimiter.ratelimit( + requester, + (requester.user.to_string(), requester.device_id), + ) await self._initialized_from_db next_send_ts = await self._store.cancel_delayed_event( @@ -308,7 +350,10 @@ class DelayedEventsHandler: NotFoundError: if no matching delayed event could be found. """ assert self._is_master - await self._request_ratelimiter.ratelimit(requester) + await self._delayed_event_mgmt_ratelimiter.ratelimit( + requester, + (requester.user.to_string(), requester.device_id), + ) await self._initialized_from_db next_send_ts = await self._store.restart_delayed_event( @@ -332,6 +377,8 @@ class DelayedEventsHandler: NotFoundError: if no matching delayed event could be found. """ assert self._is_master + # Use standard request limiter for sending delayed events on-demand, + # as an on-demand send is similar to sending a regular event. await self._request_ratelimiter.ratelimit(requester) await self._initialized_from_db @@ -408,6 +455,7 @@ class DelayedEventsHandler: delay_sec, run_as_background_process, "_send_on_timeout", + self.server_name, self._send_on_timeout, ) else: @@ -415,7 +463,10 @@ class DelayedEventsHandler: async def get_all_for_user(self, requester: Requester) -> List[JsonDict]: """Return all pending delayed events requested by the given user.""" - await self._request_ratelimiter.ratelimit(requester) + await self._delayed_event_mgmt_ratelimiter.ratelimit( + requester, + (requester.user.to_string(), requester.device_id), + ) return await self._store.get_all_delayed_events_for_user( requester.user.localpart ) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index d88660e273..acae34e71f 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -20,6 +20,8 @@ # # import logging +import random +from threading import Lock from typing import ( TYPE_CHECKING, AbstractSet, @@ -30,6 +32,7 @@ from typing import ( Optional, Set, Tuple, + cast, ) from synapse.api import errors @@ -47,6 +50,13 @@ from synapse.metrics.background_process_metrics import ( run_as_background_process, wrap_as_background_process, ) +from synapse.replication.http.devices import ( + ReplicationDeviceHandleRoomUnPartialStated, + ReplicationHandleNewDeviceUpdateRestServlet, + ReplicationMultiUserDevicesResyncRestServlet, + ReplicationNotifyDeviceUpdateRestServlet, + ReplicationNotifyUserSignatureUpdateRestServlet, +) from synapse.storage.databases.main.client_ips import DeviceLastConnectionInfo from synapse.storage.databases.main.roommember import EventIdMembership from synapse.storage.databases.main.state_deltas import StateDelta @@ -74,6 +84,7 @@ from synapse.util.retryutils import ( ) if TYPE_CHECKING: + from synapse.app.generic_worker import GenericWorkerStore from synapse.server import HomeServer logger = logging.getLogger(__name__) @@ -83,32 +94,320 @@ MAX_DEVICE_DISPLAY_NAME_LEN = 100 DELETE_STALE_DEVICES_INTERVAL_MS = 24 * 60 * 60 * 1000 -class DeviceWorkerHandler: +def _check_device_name_length(name: Optional[str]) -> None: + """ + Checks whether a device name is longer than the maximum allowed length. + + Args: + name: The name of the device. + + Raises: + SynapseError: if the device name is too long. + """ + if name and len(name) > MAX_DEVICE_DISPLAY_NAME_LEN: + raise SynapseError( + 400, + "Device display name is too long (max %i)" % (MAX_DEVICE_DISPLAY_NAME_LEN,), + errcode=Codes.TOO_LARGE, + ) + + +class DeviceHandler: + """ + Handles most things related to devices. This doesn't do any writing to the + device list stream on its own, and will call to device list writers through + replication when necessary (see DeviceWriterHandler). + """ + device_list_updater: "DeviceListWorkerUpdater" + store: "GenericWorkerStore" def __init__(self, hs: "HomeServer"): - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.hs = hs - self.store = hs.get_datastores().main + self.store = cast("GenericWorkerStore", hs.get_datastores().main) self.notifier = hs.get_notifier() self.state = hs.get_state_handler() self._appservice_handler = hs.get_application_service_handler() self._state_storage = hs.get_storage_controllers().state self._auth_handler = hs.get_auth_handler() + self._account_data_handler = hs.get_account_data_handler() self._event_sources = hs.get_event_sources() - self.server_name = hs.hostname self._msc3852_enabled = hs.config.experimental.msc3852_enabled self._query_appservices_for_keys = ( hs.config.experimental.msc3984_appservice_key_query ) self._task_scheduler = hs.get_task_scheduler() + self._dont_notify_new_devices_for = ( + hs.config.registration.dont_notify_new_devices_for + ) + self.device_list_updater = DeviceListWorkerUpdater(hs) self._task_scheduler.register_action( self._delete_device_messages, DELETE_DEVICE_MSGS_TASK_NAME ) + self._device_list_writers = hs.config.worker.writers.device_lists + + # Ensure a few operations are only running on the first device list writer + # + # This is needed because of a few linearizers in the DeviceListUpdater, + # and avoid using cross-worker locks. + # + # The main logic update is that the DeviceListUpdater is now only + # instantiated on the first device list writer, and a few methods that + # were safe to move to any worker were moved to the DeviceListWorkerUpdater + # This must be kept in sync with DeviceListWorkerUpdater + self._main_device_list_writer = hs.config.worker.writers.device_lists[0] + + self._notify_device_update_client = ( + ReplicationNotifyDeviceUpdateRestServlet.make_client(hs) + ) + self._notify_user_signature_update_client = ( + ReplicationNotifyUserSignatureUpdateRestServlet.make_client(hs) + ) + self._handle_new_device_update_client = ( + ReplicationHandleNewDeviceUpdateRestServlet.make_client(hs) + ) + self._handle_room_un_partial_stated_client = ( + ReplicationDeviceHandleRoomUnPartialStated.make_client(hs) + ) + + # The EDUs are handled on a single writer, as it needs to acquire a + # per-user lock, for which it is cheaper to use in-memory linearizers + # than cross-worker locks. + hs.get_federation_registry().register_instances_for_edu( + EduTypes.DEVICE_LIST_UPDATE, + [self._main_device_list_writer], + ) + + self._delete_stale_devices_after = hs.config.server.delete_stale_devices_after + + if ( + hs.config.worker.run_background_tasks + and self._delete_stale_devices_after is not None + ): + self.clock.looping_call( + run_as_background_process, + DELETE_STALE_DEVICES_INTERVAL_MS, + desc="delete_stale_devices", + server_name=self.server_name, + func=self._delete_stale_devices, + ) + + async def _delete_stale_devices(self) -> None: + """Background task that deletes devices which haven't been accessed for more than + a configured time period. + """ + # We should only be running this job if the config option is defined. + assert self._delete_stale_devices_after is not None + now_ms = self.clock.time_msec() + since_ms = now_ms - self._delete_stale_devices_after + devices = await self.store.get_local_devices_not_accessed_since(since_ms) + + for user_id, user_devices in devices.items(): + await self.delete_devices(user_id, user_devices) + + async def check_device_registered( + self, + user_id: str, + device_id: Optional[str], + initial_device_display_name: Optional[str] = None, + auth_provider_id: Optional[str] = None, + auth_provider_session_id: Optional[str] = None, + ) -> str: + """ + If the given device has not been registered, register it with the + supplied display name. + + If no device_id is supplied, we make one up. + + Args: + user_id: @user:id + device_id: device id supplied by client + initial_device_display_name: device display name from client + auth_provider_id: The SSO IdP the user used, if any. + auth_provider_session_id: The session ID (sid) got from the SSO IdP. + Returns: + device id (generated if none was supplied) + """ + + _check_device_name_length(initial_device_display_name) + + # Check if we should send out device lists updates for this new device. + notify = user_id not in self._dont_notify_new_devices_for + + if device_id is not None: + new_device = await self.store.store_device( + user_id=user_id, + device_id=device_id, + initial_device_display_name=initial_device_display_name, + auth_provider_id=auth_provider_id, + auth_provider_session_id=auth_provider_session_id, + ) + if new_device: + if notify: + await self.notify_device_update(user_id, [device_id]) + return device_id + + # if the device id is not specified, we'll autogen one, but loop a few + # times in case of a clash. + attempts = 0 + while attempts < 5: + new_device_id = stringutils.random_string(10).upper() + new_device = await self.store.store_device( + user_id=user_id, + device_id=new_device_id, + initial_device_display_name=initial_device_display_name, + auth_provider_id=auth_provider_id, + auth_provider_session_id=auth_provider_session_id, + ) + if new_device: + if notify: + await self.notify_device_update(user_id, [new_device_id]) + return new_device_id + attempts += 1 + + raise errors.StoreError(500, "Couldn't generate a device ID.") + + @trace + async def delete_all_devices_for_user( + self, user_id: str, except_device_id: Optional[str] = None + ) -> None: + """Delete all of the user's devices + + Args: + user_id: The user to remove all devices from + except_device_id: optional device id which should not be deleted + """ + device_map = await self.store.get_devices_by_user(user_id) + if except_device_id is not None: + device_map.pop(except_device_id, None) + user_device_ids = device_map.keys() + await self.delete_devices(user_id, user_device_ids) + + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: + """Delete several devices + + Args: + user_id: The user to delete devices from. + device_ids: The list of device IDs to delete + """ + to_device_stream_id = self._event_sources.get_current_token().to_device_key + + try: + await self.store.delete_devices(user_id, device_ids) + except errors.StoreError as e: + if e.code == 404: + # no match + set_tag("error", True) + set_tag("reason", "User doesn't have that device id.") + else: + raise + + # Delete data specific to each device. Not optimised as its an + # experimental MSC. + if self.hs.config.experimental.msc3890_enabled: + for device_id in device_ids: + # Remove any local notification settings for this device in accordance + # with MSC3890. + await self._account_data_handler.remove_account_data_for_user( + user_id, + f"org.matrix.msc3890.local_notification_settings.{device_id}", + ) + + # If we're deleting a lot of devices, a bunch of them may not have any + # to-device messages queued up. We filter those out to avoid scheduling + # unnecessary tasks. + devices_with_messages = await self.store.get_devices_with_messages( + user_id, device_ids + ) + for device_id in devices_with_messages: + # Delete device messages asynchronously and in batches using the task scheduler + # We specify an upper stream id to avoid deleting non delivered messages + # if an user re-uses a device ID. + await self._task_scheduler.schedule_task( + DELETE_DEVICE_MSGS_TASK_NAME, + resource_id=device_id, + params={ + "user_id": user_id, + "device_id": device_id, + "up_to_stream_id": to_device_stream_id, + }, + ) + + await self._auth_handler.delete_access_tokens_for_devices( + user_id, device_ids=device_ids + ) + + # Pushers are deleted after `delete_access_tokens_for_user` is called so that + # modules using `on_logged_out` hook can use them if needed. + await self.hs.get_pusherpool().remove_pushers_by_devices(user_id, device_ids) + + await self.notify_device_update(user_id, device_ids) + + async def upsert_device( + self, user_id: str, device_id: str, display_name: Optional[str] = None + ) -> bool: + """Create or update a device + + Args: + user_id: The user to update devices of. + device_id: The device to update. + display_name: The new display name for this device. + + Returns: + True if the device was created, False if it was updated. + + """ + + # Reject a new displayname which is too long. + _check_device_name_length(display_name) + + created = await self.store.store_device( + user_id, + device_id, + initial_device_display_name=display_name, + ) + + if not created: + await self.store.update_device( + user_id, + device_id, + new_display_name=display_name, + ) + + await self.notify_device_update(user_id, [device_id]) + return created + + async def update_device(self, user_id: str, device_id: str, content: dict) -> None: + """Update the given device + + Args: + user_id: The user to update devices of. + device_id: The device to update. + content: body of update request + """ + + # Reject a new displayname which is too long. + new_display_name = content.get("display_name") + + _check_device_name_length(new_display_name) + + try: + await self.store.update_device( + user_id, device_id, new_display_name=new_display_name + ) + await self.notify_device_update(user_id, [device_id]) + except errors.StoreError as e: + if e.code == 404: + raise errors.NotFoundError() + else: + raise + @trace async def get_devices_by_user(self, user_id: str) -> List[JsonDict]: """ @@ -145,6 +444,98 @@ class DeviceWorkerHandler: """ return await self.store.get_dehydrated_device(user_id) + async def store_dehydrated_device( + self, + user_id: str, + device_id: Optional[str], + device_data: JsonDict, + initial_device_display_name: Optional[str] = None, + keys_for_device: Optional[JsonDict] = None, + ) -> str: + """Store a dehydrated device for a user, optionally storing the keys associated with + it as well. If the user had a previous dehydrated device, it is removed. + + Args: + user_id: the user that we are storing the device for + device_id: device id supplied by client + device_data: the dehydrated device information + initial_device_display_name: The display name to use for the device + keys_for_device: keys for the dehydrated device + Returns: + device id of the dehydrated device + """ + device_id = await self.check_device_registered( + user_id, + device_id, + initial_device_display_name, + ) + + time_now = self.clock.time_msec() + + old_device_id = await self.store.store_dehydrated_device( + user_id, device_id, device_data, time_now, keys_for_device + ) + + if old_device_id is not None: + await self.delete_devices(user_id, [old_device_id]) + + return device_id + + async def rehydrate_device( + self, user_id: str, access_token: str, device_id: str + ) -> dict: + """Process a rehydration request from the user. + + Args: + user_id: the user who is rehydrating the device + access_token: the access token used for the request + device_id: the ID of the device that will be rehydrated + Returns: + a dict containing {"success": True} + """ + success = await self.store.remove_dehydrated_device(user_id, device_id) + + if not success: + raise errors.NotFoundError() + + # If the dehydrated device was successfully deleted (the device ID + # matched the stored dehydrated device), then modify the access + # token and refresh token to use the dehydrated device's ID and + # copy the old device display name to the dehydrated device, + # and destroy the old device ID + old_device_id = await self.store.set_device_for_access_token( + access_token, device_id + ) + await self.store.set_device_for_refresh_token(user_id, old_device_id, device_id) + old_device = await self.store.get_device(user_id, old_device_id) + if old_device is None: + raise errors.NotFoundError() + await self.store.update_device(user_id, device_id, old_device["display_name"]) + # can't call self.delete_device because that will clobber the + # access token so call the storage layer directly + await self.store.delete_devices(user_id, [old_device_id]) + + # tell everyone that the old device is gone and that the dehydrated + # device has a new display name + await self.notify_device_update(user_id, [old_device_id, device_id]) + + return {"success": True} + + async def delete_dehydrated_device(self, user_id: str, device_id: str) -> None: + """ + Delete a stored dehydrated device. + + Args: + user_id: the user_id to delete the device from + device_id: id of the dehydrated device to delete + """ + success = await self.store.remove_dehydrated_device(user_id, device_id) + + if not success: + raise errors.NotFoundError() + + await self.delete_devices(user_id, [device_id]) + @trace async def get_device(self, user_id: str, device_id: str) -> JsonDict: """Retrieve the given device @@ -163,6 +554,8 @@ class DeviceWorkerHandler: raise errors.NotFoundError() ips = await self.store.get_last_client_ip_by_device(user_id, device_id) + + device = dict(device) _update_device_from_client_ips(device, ips) set_tag("device", str(device)) @@ -481,10 +874,53 @@ class DeviceWorkerHandler: gone from partial to full state. """ - # TODO(faster_joins): worker mode support - # https://github.com/matrix-org/synapse/issues/12994 - logger.error( - "Trying handling device list state for partial join: not supported on workers." + await self._handle_room_un_partial_stated_client( + instance_name=random.choice(self._device_list_writers), + room_id=room_id, + ) + + @trace + @measure_func("notify_device_update") + async def notify_device_update( + self, user_id: str, device_ids: StrCollection + ) -> None: + """Notify that a user's device(s) has changed. Pokes the notifier, and + remote servers if the user is local. + + Args: + user_id: The Matrix ID of the user who's device list has been updated. + device_ids: The device IDs that have changed. + """ + await self._notify_device_update_client( + instance_name=random.choice(self._device_list_writers), + user_id=user_id, + device_ids=list(device_ids), + ) + + async def notify_user_signature_update( + self, + from_user_id: str, + user_ids: List[str], + ) -> None: + """Notify a device writer that a user have made new signatures of other users. + + Args: + from_user_id: The Matrix ID of the user who's signatures have been updated. + user_ids: The Matrix IDs of the users that have changed. + """ + await self._notify_user_signature_update_client( + instance_name=random.choice(self._device_list_writers), + from_user_id=from_user_id, + user_ids=user_ids, + ) + + async def handle_new_device_update(self) -> None: + """Wake up a device writer to send local device list changes as federation outbound pokes.""" + # This is only sent to the first device writer to avoid cross-worker + # locks in _handle_new_device_update_async, as it makes assumptions + # about being the only instance running. + await self._handle_new_device_update_client( + instance_name=self._device_list_writers[0], ) DEVICE_MSGS_DELETE_BATCH_LIMIT = 1000 @@ -508,37 +944,46 @@ class DeviceWorkerHandler: device_id=device_id, from_stream_id=from_stream_id, to_stream_id=up_to_stream_id, - limit=DeviceHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT, + limit=DeviceWriterHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT, ) if from_stream_id is None: return TaskStatus.COMPLETE, None, None - await self.clock.sleep(DeviceHandler.DEVICE_MSGS_DELETE_SLEEP_MS / 1000.0) + await self.clock.sleep( + DeviceWriterHandler.DEVICE_MSGS_DELETE_SLEEP_MS / 1000.0 + ) -class DeviceHandler(DeviceWorkerHandler): - device_list_updater: "DeviceListUpdater" +class DeviceWriterHandler(DeviceHandler): + """ + Superclass of the DeviceHandler which gets instantiated on workers that can + write to the device list stream. + """ def __init__(self, hs: "HomeServer"): super().__init__(hs) - self.federation_sender = hs.get_federation_sender() - self._account_data_handler = hs.get_account_data_handler() + self.server_name = ( + hs.hostname + ) # nb must be called this for @measure_func and @wrap_as_background_process + # We only need to poke the federation sender explicitly if its on the + # same instance. Other federation sender instances will get notified by + # `synapse.app.generic_worker.FederationSenderHandler` when it sees it + # in the device lists stream. + self.federation_sender = None + if hs.should_send_federation(): + self.federation_sender = hs.get_federation_sender() + self._storage_controllers = hs.get_storage_controllers() - self.db_pool = hs.get_datastores().main.db_pool - self._dont_notify_new_devices_for = ( - hs.config.registration.dont_notify_new_devices_for - ) - - self.device_list_updater = DeviceListUpdater(hs, self) - - federation_registry = hs.get_federation_registry() - - federation_registry.register_edu_handler( - EduTypes.DEVICE_LIST_UPDATE, - self.device_list_updater.incoming_device_list_update, + # There are a few things that are only handled on the main device list + # writer to avoid cross-worker locks + # + # This mainly concerns the `DeviceListUpdater` class, which is only + # instantiated on the first device list writer. + self._is_main_device_list_writer = ( + hs.get_instance_name() == self._main_device_list_writer ) # Whether `_handle_new_device_update_async` is currently processing. @@ -548,212 +993,22 @@ class DeviceHandler(DeviceWorkerHandler): # processing. self._handle_new_device_update_new_data = False - # On start up check if there are any updates pending. - hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) - - self._delete_stale_devices_after = hs.config.server.delete_stale_devices_after - - # Ideally we would run this on a worker and condition this on the - # "run_background_tasks_on" setting, but this would mean making the notification - # of device list changes over federation work on workers, which is nontrivial. - if self._delete_stale_devices_after is not None: - self.clock.looping_call( - run_as_background_process, - DELETE_STALE_DEVICES_INTERVAL_MS, - "delete_stale_devices", - self._delete_stale_devices, + # Only the main device list writer handles device list EDUs and converts + # device list updates to outbound federation pokes. This allows us to + # use in-memory per-user locks instead of cross-worker locks, and + # simplifies the logic for converting outbound pokes. This makes the + # device_list writers a little bit unbalanced in terms of load, but + # still unlocks local device changes (and therefore login/logouts) when + # rolling-restarting Synapse. + if self._is_main_device_list_writer: + # On start up check if there are any updates pending. + hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) + self.device_list_updater = DeviceListUpdater(hs, self) + hs.get_federation_registry().register_edu_handler( + EduTypes.DEVICE_LIST_UPDATE, + self.device_list_updater.incoming_device_list_update, ) - def _check_device_name_length(self, name: Optional[str]) -> None: - """ - Checks whether a device name is longer than the maximum allowed length. - - Args: - name: The name of the device. - - Raises: - SynapseError: if the device name is too long. - """ - if name and len(name) > MAX_DEVICE_DISPLAY_NAME_LEN: - raise SynapseError( - 400, - "Device display name is too long (max %i)" - % (MAX_DEVICE_DISPLAY_NAME_LEN,), - errcode=Codes.TOO_LARGE, - ) - - async def check_device_registered( - self, - user_id: str, - device_id: Optional[str], - initial_device_display_name: Optional[str] = None, - auth_provider_id: Optional[str] = None, - auth_provider_session_id: Optional[str] = None, - ) -> str: - """ - If the given device has not been registered, register it with the - supplied display name. - - If no device_id is supplied, we make one up. - - Args: - user_id: @user:id - device_id: device id supplied by client - initial_device_display_name: device display name from client - auth_provider_id: The SSO IdP the user used, if any. - auth_provider_session_id: The session ID (sid) got from the SSO IdP. - Returns: - device id (generated if none was supplied) - """ - - self._check_device_name_length(initial_device_display_name) - - # Check if we should send out device lists updates for this new device. - notify = user_id not in self._dont_notify_new_devices_for - - if device_id is not None: - new_device = await self.store.store_device( - user_id=user_id, - device_id=device_id, - initial_device_display_name=initial_device_display_name, - auth_provider_id=auth_provider_id, - auth_provider_session_id=auth_provider_session_id, - ) - if new_device: - if notify: - await self.notify_device_update(user_id, [device_id]) - return device_id - - # if the device id is not specified, we'll autogen one, but loop a few - # times in case of a clash. - attempts = 0 - while attempts < 5: - new_device_id = stringutils.random_string(10).upper() - new_device = await self.store.store_device( - user_id=user_id, - device_id=new_device_id, - initial_device_display_name=initial_device_display_name, - auth_provider_id=auth_provider_id, - auth_provider_session_id=auth_provider_session_id, - ) - if new_device: - if notify: - await self.notify_device_update(user_id, [new_device_id]) - return new_device_id - attempts += 1 - - raise errors.StoreError(500, "Couldn't generate a device ID.") - - async def _delete_stale_devices(self) -> None: - """Background task that deletes devices which haven't been accessed for more than - a configured time period. - """ - # We should only be running this job if the config option is defined. - assert self._delete_stale_devices_after is not None - now_ms = self.clock.time_msec() - since_ms = now_ms - self._delete_stale_devices_after - devices = await self.store.get_local_devices_not_accessed_since(since_ms) - - for user_id, user_devices in devices.items(): - await self.delete_devices(user_id, user_devices) - - @trace - async def delete_all_devices_for_user( - self, user_id: str, except_device_id: Optional[str] = None - ) -> None: - """Delete all of the user's devices - - Args: - user_id: The user to remove all devices from - except_device_id: optional device id which should not be deleted - """ - device_map = await self.store.get_devices_by_user(user_id) - device_ids = list(device_map) - if except_device_id is not None: - device_ids = [d for d in device_ids if d != except_device_id] - await self.delete_devices(user_id, device_ids) - - async def delete_devices(self, user_id: str, device_ids: List[str]) -> None: - """Delete several devices - - Args: - user_id: The user to delete devices from. - device_ids: The list of device IDs to delete - """ - to_device_stream_id = self._event_sources.get_current_token().to_device_key - - try: - await self.store.delete_devices(user_id, device_ids) - except errors.StoreError as e: - if e.code == 404: - # no match - set_tag("error", True) - set_tag("reason", "User doesn't have that device id.") - else: - raise - - # Delete data specific to each device. Not optimised as it is not - # considered as part of a critical path. - for device_id in device_ids: - await self._auth_handler.delete_access_tokens_for_user( - user_id, device_id=device_id - ) - await self.store.delete_e2e_keys_by_device( - user_id=user_id, device_id=device_id - ) - - if self.hs.config.experimental.msc3890_enabled: - # Remove any local notification settings for this device in accordance - # with MSC3890. - await self._account_data_handler.remove_account_data_for_user( - user_id, - f"org.matrix.msc3890.local_notification_settings.{device_id}", - ) - - # Delete device messages asynchronously and in batches using the task scheduler - # We specify an upper stream id to avoid deleting non delivered messages - # if an user re-uses a device ID. - await self._task_scheduler.schedule_task( - DELETE_DEVICE_MSGS_TASK_NAME, - resource_id=device_id, - params={ - "user_id": user_id, - "device_id": device_id, - "up_to_stream_id": to_device_stream_id, - }, - ) - - # Pushers are deleted after `delete_access_tokens_for_user` is called so that - # modules using `on_logged_out` hook can use them if needed. - await self.hs.get_pusherpool().remove_pushers_by_devices(user_id, device_ids) - - await self.notify_device_update(user_id, device_ids) - - async def update_device(self, user_id: str, device_id: str, content: dict) -> None: - """Update the given device - - Args: - user_id: The user to update devices of. - device_id: The device to update. - content: body of update request - """ - - # Reject a new displayname which is too long. - new_display_name = content.get("display_name") - - self._check_device_name_length(new_display_name) - - try: - await self.store.update_device( - user_id, device_id, new_display_name=new_display_name - ) - await self.notify_device_update(user_id, [device_id]) - except errors.StoreError as e: - if e.code == 404: - raise errors.NotFoundError() - else: - raise - @trace @measure_func("notify_device_update") async def notify_device_update( @@ -782,10 +1037,11 @@ class DeviceHandler(DeviceWorkerHandler): # This should only happen if there are no updates, so we bail. return - for device_id in device_ids: - logger.debug( - "Notifying about update %r/%r, ID: %r", user_id, device_id, position - ) + if logger.isEnabledFor(logging.DEBUG): + for device_id in device_ids: + logger.debug( + "Notifying about update %r/%r, ID: %r", user_id, device_id, position + ) # specify the user ID too since the user should always get their own device list # updates, even if they aren't in any rooms. @@ -795,7 +1051,7 @@ class DeviceHandler(DeviceWorkerHandler): # We may need to do some processing asynchronously for local user IDs. if self.hs.is_mine_id(user_id): - self._handle_new_device_update_async() + await self.handle_new_device_update() async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] @@ -815,101 +1071,16 @@ class DeviceHandler(DeviceWorkerHandler): StreamKeyType.DEVICE_LIST, position, users=[from_user_id] ) - async def store_dehydrated_device( - self, - user_id: str, - device_id: Optional[str], - device_data: JsonDict, - initial_device_display_name: Optional[str] = None, - keys_for_device: Optional[JsonDict] = None, - ) -> str: - """Store a dehydrated device for a user, optionally storing the keys associated with - it as well. If the user had a previous dehydrated device, it is removed. + async def handle_new_device_update(self) -> None: + # _handle_new_device_update_async is only called on the first device + # writer, as it makes assumptions about only having one instance running + # at a time. If this is not the first device writer, we defer to the + # superclass, which will make the call go through replication. + if not self._is_main_device_list_writer: + return await super().handle_new_device_update() - Args: - user_id: the user that we are storing the device for - device_id: device id supplied by client - device_data: the dehydrated device information - initial_device_display_name: The display name to use for the device - keys_for_device: keys for the dehydrated device - Returns: - device id of the dehydrated device - """ - device_id = await self.check_device_registered( - user_id, - device_id, - initial_device_display_name, - ) - - time_now = self.clock.time_msec() - - old_device_id = await self.store.store_dehydrated_device( - user_id, device_id, device_data, time_now, keys_for_device - ) - - if old_device_id is not None: - await self.delete_devices(user_id, [old_device_id]) - - return device_id - - async def rehydrate_device( - self, user_id: str, access_token: str, device_id: str - ) -> dict: - """Process a rehydration request from the user. - - Args: - user_id: the user who is rehydrating the device - access_token: the access token used for the request - device_id: the ID of the device that will be rehydrated - Returns: - a dict containing {"success": True} - """ - success = await self.store.remove_dehydrated_device(user_id, device_id) - - if not success: - raise errors.NotFoundError() - - # If the dehydrated device was successfully deleted (the device ID - # matched the stored dehydrated device), then modify the access - # token and refresh token to use the dehydrated device's ID and - # copy the old device display name to the dehydrated device, - # and destroy the old device ID - old_device_id = await self.store.set_device_for_access_token( - access_token, device_id - ) - await self.store.set_device_for_refresh_token(user_id, old_device_id, device_id) - old_device = await self.store.get_device(user_id, old_device_id) - if old_device is None: - raise errors.NotFoundError() - await self.store.update_device(user_id, device_id, old_device["display_name"]) - # can't call self.delete_device because that will clobber the - # access token so call the storage layer directly - await self.store.delete_devices(user_id, [old_device_id]) - await self.store.delete_e2e_keys_by_device( - user_id=user_id, device_id=old_device_id - ) - - # tell everyone that the old device is gone and that the dehydrated - # device has a new display name - await self.notify_device_update(user_id, [old_device_id, device_id]) - - return {"success": True} - - async def delete_dehydrated_device(self, user_id: str, device_id: str) -> None: - """ - Delete a stored dehydrated device. - - Args: - user_id: the user_id to delete the device from - device_id: id of the dehydrated device to delete - """ - success = await self.store.remove_dehydrated_device(user_id, device_id) - - if not success: - raise errors.NotFoundError() - - await self.delete_devices(user_id, [device_id]) - await self.store.delete_e2e_keys_by_device(user_id=user_id, device_id=device_id) + self._handle_new_device_update_async() + return @wrap_as_background_process("_handle_new_device_update_async") async def _handle_new_device_update_async(self) -> None: @@ -919,12 +1090,27 @@ class DeviceHandler(DeviceWorkerHandler): This happens in the background so as not to block the original request that generated the device update. """ + # This should only ever be called on the main device list writer, as it + # expects to only have a single instance of this loop running at a time. + # See `handle_new_device_update`. + assert self._is_main_device_list_writer + if self._handle_new_device_update_is_processing: self._handle_new_device_update_new_data = True return self._handle_new_device_update_is_processing = True + # Note that this logic only deals with the minimum stream ID, and not + # the full stream token. This means that oubound pokes are only sent + # once every writer on the device_lists stream has caught up. This is + # fine, it may only introduces a bit of lag on the outbound pokes. + # To fix this, 'device_lists_changes_converted_stream_position' would + # need to include the full stream token instead of just a stream ID. + # We could also consider have each writer converting their own device + # list updates, but that can quickly become complex to handle changes in + # the list of device writers. + # The stream ID we processed previous iteration (if any), and the set of # hosts we've already poked about for this update. This is so that we # don't poke the same remote server about the same update repeatedly. @@ -936,7 +1122,7 @@ class DeviceHandler(DeviceWorkerHandler): while True: self._handle_new_device_update_new_data = False - max_stream_id = self.store.get_device_stream_token() + max_stream_id = self.store.get_device_stream_token().stream rows = await self.store.get_uncoverted_outbound_room_pokes( stream_id, room_id ) @@ -1001,7 +1187,7 @@ class DeviceHandler(DeviceWorkerHandler): # Notify replication that we've updated the device list stream. self.notifier.notify_replication() - if hosts: + if hosts and self.federation_sender: logger.info( "Sending device list update notif for %r to: %r", user_id, @@ -1121,9 +1307,10 @@ class DeviceHandler(DeviceWorkerHandler): # Notify things that device lists need to be sent out. self.notifier.notify_replication() - await self.federation_sender.send_device_messages( - potentially_changed_hosts, immediate=False - ) + if self.federation_sender: + await self.federation_sender.send_device_messages( + potentially_changed_hosts, immediate=False + ) def _update_device_from_client_ips( @@ -1140,19 +1327,21 @@ def _update_device_from_client_ips( class DeviceListWorkerUpdater: - "Handles incoming device list updates from federation and contacts the main process over replication" + "Handles incoming device list updates from federation and contacts the main device list writer over replication" def __init__(self, hs: "HomeServer"): - from synapse.replication.http.devices import ( - ReplicationMultiUserDevicesResyncRestServlet, - ) - + self.store = hs.get_datastores().main + self._notifier = hs.get_notifier() + # On which instance the DeviceListUpdater is running + # Must be kept in sync with DeviceHandler + self._main_device_list_writer = hs.config.worker.writers.device_lists[0] self._multi_user_device_resync_client = ( ReplicationMultiUserDevicesResyncRestServlet.make_client(hs) ) async def multi_user_device_resync( - self, user_ids: List[str], mark_failed_as_stale: bool = True + self, + user_ids: List[str], ) -> Dict[str, Optional[JsonMapping]]: """ Like `user_device_resync` but operates on multiple users **from the same origin** @@ -1161,25 +1350,105 @@ class DeviceListWorkerUpdater: Returns: Dict from User ID to the same Dict as `user_device_resync`. """ - # mark_failed_as_stale is not sent. Ensure this doesn't break expectations. - assert mark_failed_as_stale if not user_ids: # Shortcut empty requests return {} - return await self._multi_user_device_resync_client(user_ids=user_ids) + # This uses a per-user-id lock; to avoid using cross-worker locks, we + # forward the request to the main device list writer. + # See DeviceListUpdater + return await self._multi_user_device_resync_client( + instance_name=self._main_device_list_writer, + user_ids=user_ids, + ) + + async def process_cross_signing_key_update( + self, + user_id: str, + master_key: Optional[JsonDict], + self_signing_key: Optional[JsonDict], + ) -> List[str]: + """Process the given new master and self-signing key for the given remote user. + + Args: + user_id: The ID of the user these keys are for. + master_key: The dict of the cross-signing master key as returned by the + remote server. + self_signing_key: The dict of the cross-signing self-signing key as returned + by the remote server. + + Return: + The device IDs for the given keys. + """ + device_ids = [] + + current_keys_map = await self.store.get_e2e_cross_signing_keys_bulk([user_id]) + current_keys = current_keys_map.get(user_id) or {} + + if master_key and master_key != current_keys.get("master"): + await self.store.set_e2e_cross_signing_key(user_id, "master", master_key) + _, verify_key = get_verify_key_from_cross_signing_key(master_key) + # verify_key is a VerifyKey from signedjson, which uses + # .version to denote the portion of the key ID after the + # algorithm and colon, which is the device ID + device_ids.append(verify_key.version) + if self_signing_key and self_signing_key != current_keys.get("self_signing"): + await self.store.set_e2e_cross_signing_key( + user_id, "self_signing", self_signing_key + ) + _, verify_key = get_verify_key_from_cross_signing_key(self_signing_key) + device_ids.append(verify_key.version) + + return device_ids + + async def handle_room_un_partial_stated(self, room_id: str) -> None: + """Handles sending appropriate device list updates in a room that has + gone from partial to full state. + """ + + pending_updates = ( + await self.store.get_pending_remote_device_list_updates_for_room(room_id) + ) + + for user_id, device_id in pending_updates: + logger.info( + "Got pending device list update in room %s: %s / %s", + room_id, + user_id, + device_id, + ) + position = await self.store.add_device_change_to_streams( + user_id, + [device_id], + room_ids=[room_id], + ) + + if not position: + # This should only happen if there are no updates, which + # shouldn't happen when we've passed in a non-empty set of + # device IDs. + continue + + self._notifier.on_new_event( + StreamKeyType.DEVICE_LIST, position, rooms=[room_id] + ) class DeviceListUpdater(DeviceListWorkerUpdater): - "Handles incoming device list updates from federation and updates the DB" + """Handles incoming device list updates from federation and updates the DB. - def __init__(self, hs: "HomeServer", device_handler: DeviceHandler): - self.store = hs.get_datastores().main + This is only instanciated on the first device list writer, as it uses + in-process linearizers for some operations.""" + + def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): + super().__init__(hs) + + self.server_name = hs.hostname self.federation = hs.get_federation_client() - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.device_handler = device_handler - self._notifier = hs.get_notifier() self._remote_edu_linearizer = Linearizer(name="remote_device_list") self._resync_linearizer = Linearizer(name="remote_device_resync") @@ -1194,6 +1463,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): # resyncs. self._seen_updates: ExpiringCache[str, Set[str]] = ExpiringCache( cache_name="device_update_edu", + server_name=self.server_name, clock=self.clock, max_len=10000, expiry_ms=30 * 60 * 1000, @@ -1201,10 +1471,11 @@ class DeviceListUpdater(DeviceListWorkerUpdater): ) # Attempt to resync out of sync device lists every 30s. - self._resync_retry_in_progress = False + self._resync_retry_lock = Lock() self.clock.looping_call( run_as_background_process, 30 * 1000, + server_name=self.server_name, func=self._maybe_retry_device_resync, desc="_maybe_retry_device_resync", ) @@ -1326,6 +1597,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): await self.store.mark_remote_users_device_caches_as_stale([user_id]) run_as_background_process( "_maybe_retry_device_resync", + self.server_name, self.multi_user_device_resync, [user_id], False, @@ -1383,13 +1655,10 @@ class DeviceListUpdater(DeviceListWorkerUpdater): """Retry to resync device lists that are out of sync, except if another retry is in progress. """ - if self._resync_retry_in_progress: + # If the lock can not be acquired we want to always return immediately instead of blocking here + if not self._resync_retry_lock.acquire(blocking=False): return - try: - # Prevent another call of this function to retry resyncing device lists so - # we don't send too many requests. - self._resync_retry_in_progress = True # Get all of the users that need resyncing. need_resync = await self.store.get_user_ids_requiring_device_list_resync() @@ -1430,8 +1699,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): e, ) finally: - # Allow future calls to retry resyncinc out of sync device lists. - self._resync_retry_in_progress = False + self._resync_retry_lock.release() async def multi_user_device_resync( self, user_ids: List[str], mark_failed_as_stale: bool = True @@ -1567,7 +1835,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): if prev_stream_id is not None and cached_devices == { d["device_id"]: d for d in devices }: - logging.info( + logger.info( "Skipping device list resync for %s, as our cache matches already", user_id, ) @@ -1607,74 +1875,3 @@ class DeviceListUpdater(DeviceListWorkerUpdater): self._seen_updates[user_id] = {stream_id} return result, False - - async def process_cross_signing_key_update( - self, - user_id: str, - master_key: Optional[JsonDict], - self_signing_key: Optional[JsonDict], - ) -> List[str]: - """Process the given new master and self-signing key for the given remote user. - - Args: - user_id: The ID of the user these keys are for. - master_key: The dict of the cross-signing master key as returned by the - remote server. - self_signing_key: The dict of the cross-signing self-signing key as returned - by the remote server. - - Return: - The device IDs for the given keys. - """ - device_ids = [] - - current_keys_map = await self.store.get_e2e_cross_signing_keys_bulk([user_id]) - current_keys = current_keys_map.get(user_id) or {} - - if master_key and master_key != current_keys.get("master"): - await self.store.set_e2e_cross_signing_key(user_id, "master", master_key) - _, verify_key = get_verify_key_from_cross_signing_key(master_key) - # verify_key is a VerifyKey from signedjson, which uses - # .version to denote the portion of the key ID after the - # algorithm and colon, which is the device ID - device_ids.append(verify_key.version) - if self_signing_key and self_signing_key != current_keys.get("self_signing"): - await self.store.set_e2e_cross_signing_key( - user_id, "self_signing", self_signing_key - ) - _, verify_key = get_verify_key_from_cross_signing_key(self_signing_key) - device_ids.append(verify_key.version) - - return device_ids - - async def handle_room_un_partial_stated(self, room_id: str) -> None: - """Handles sending appropriate device list updates in a room that has - gone from partial to full state. - """ - - pending_updates = ( - await self.store.get_pending_remote_device_list_updates_for_room(room_id) - ) - - for user_id, device_id in pending_updates: - logger.info( - "Got pending device list update in room %s: %s / %s", - room_id, - user_id, - device_id, - ) - position = await self.store.add_device_change_to_streams( - user_id, - [device_id], - room_ids=[room_id], - ) - - if not position: - # This should only happen if there are no updates, which - # shouldn't happen when we've passed in a non-empty set of - # device IDs. - continue - - self.device_handler.notifier.on_new_event( - StreamKeyType.DEVICE_LIST, position, rooms=[room_id] - ) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index e56bdb4072..b43cbd9c15 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -33,9 +33,6 @@ from synapse.logging.opentracing import ( log_kv, set_tag, ) -from synapse.replication.http.devices import ( - ReplicationMultiUserDevicesResyncRestServlet, -) from synapse.types import JsonDict, Requester, StreamKeyType, UserID, get_domain_from_id from synapse.util import json_encoder from synapse.util.stringutils import random_string @@ -56,9 +53,9 @@ class DeviceMessageHandler: self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self.is_mine = hs.is_mine + self.device_handler = hs.get_device_handler() if hs.config.experimental.msc3814_enabled: self.event_sources = hs.get_event_sources() - self.device_handler = hs.get_device_handler() # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by @@ -80,18 +77,6 @@ class DeviceMessageHandler: hs.config.worker.writers.to_device, ) - # The handler to call when we think a user's device list might be out of - # sync. We do all device list resyncing on the master instance, so if - # we're on a worker we hit the device resync replication API. - if hs.config.worker.worker_app is None: - self._multi_user_device_resync = ( - hs.get_device_handler().device_list_updater.multi_user_device_resync - ) - else: - self._multi_user_device_resync = ( - ReplicationMultiUserDevicesResyncRestServlet.make_client(hs) - ) - # a rate limiter for room key requests. The keys are # (sending_user_id, sending_device_id). self._ratelimiter = Ratelimiter( @@ -213,7 +198,10 @@ class DeviceMessageHandler: await self.store.mark_remote_users_device_caches_as_stale((sender_user_id,)) # Immediately attempt a resync in the background - run_in_background(self._multi_user_device_resync, user_ids=[sender_user_id]) + run_in_background( + self.device_handler.device_list_updater.multi_user_device_resync, + user_ids=[sender_user_id], + ) async def send_device_message( self, diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 62ce16794f..11284ccd0b 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -21,9 +21,7 @@ import logging import string -from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence - -from typing_extensions import Literal +from typing import TYPE_CHECKING, Iterable, List, Literal, Optional, Sequence from synapse.api.constants import MAX_ALIAS_LENGTH, EventTypes from synapse.api.errors import ( @@ -284,7 +282,7 @@ class DirectoryHandler: except RequestSendFailed: raise SynapseError(502, "Failed to fetch alias") except CodeMessageException as e: - logging.warning( + logger.warning( "Error retrieving alias %s -> %s %s", room_alias, e.code, e.msg ) if e.code == 404: @@ -408,7 +406,7 @@ class DirectoryHandler: ] for service in interested_services: - if user_id == service.sender: + if user_id == service.sender.to_string(): # this user IS the app service so they can do whatever they like return True elif service.is_exclusive_alias(alias.to_string()): diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index c13de7c067..63a9f75f13 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -32,10 +32,9 @@ from twisted.internet import defer from synapse.api.constants import EduTypes from synapse.api.errors import CodeMessageException, Codes, NotFoundError, SynapseError -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import log_kv, set_tag, tag_args, trace -from synapse.replication.http.devices import ReplicationUploadKeysForUserRestServlet from synapse.types import ( JsonDict, JsonMapping, @@ -75,8 +74,10 @@ class E2eKeysHandler: federation_registry = hs.get_federation_registry() - is_master = hs.config.worker.worker_app is None - if is_master: + # Only the first writer in the list should handle EDUs for signing key + # updates, so that we can use an in-memory linearizer instead of worker locks. + edu_writer = hs.config.worker.writers.device_lists[0] + if hs.get_instance_name() == edu_writer: edu_updater = SigningKeyEduUpdater(hs) # Only register this edu handler on master as it requires writing @@ -91,11 +92,14 @@ class E2eKeysHandler: EduTypes.UNSTABLE_SIGNING_KEY_UPDATE, edu_updater.incoming_signing_key_update, ) - - self.device_key_uploader = self.upload_device_keys_for_user else: - self.device_key_uploader = ( - ReplicationUploadKeysForUserRestServlet.make_client(hs) + federation_registry.register_instances_for_edu( + EduTypes.SIGNING_KEY_UPDATE, + [edu_writer], + ) + federation_registry.register_instances_for_edu( + EduTypes.UNSTABLE_SIGNING_KEY_UPDATE, + [edu_writer], ) # doesn't really work as part of the generic query API, because the @@ -157,7 +161,37 @@ class E2eKeysHandler: the number of in-flight queries at a time. """ async with self._query_devices_linearizer.queue((from_user_id, from_device_id)): - device_keys_query: Dict[str, List[str]] = query_body.get("device_keys", {}) + + async def filter_device_key_query( + query: Dict[str, List[str]], + ) -> Dict[str, List[str]]: + if not self.config.experimental.msc4263_limit_key_queries_to_users_who_share_rooms: + # Only ignore invalid user IDs, which is the same behaviour as if + # the user existed but had no keys. + return { + user_id: v + for user_id, v in query.items() + if UserID.is_valid(user_id) + } + + # Strip invalid user IDs and user IDs the requesting user does not share rooms with. + valid_user_ids = [ + user_id for user_id in query.keys() if UserID.is_valid(user_id) + ] + allowed_user_ids = set( + await self.store.do_users_share_a_room_joined_or_invited( + from_user_id, valid_user_ids + ) + ) + return { + user_id: v + for user_id, v in query.items() + if user_id in allowed_user_ids + } + + device_keys_query: Dict[str, List[str]] = await filter_device_key_query( + query_body.get("device_keys", {}) + ) # separate users by domain. # make a map from domain to user_id to device_ids @@ -165,11 +199,6 @@ class E2eKeysHandler: remote_queries = {} for user_id, device_ids in device_keys_query.items(): - if not UserID.is_valid(user_id): - # Ignore invalid user IDs, which is the same behaviour as if - # the user existed but had no keys. - continue - # we use UserID.from_string to catch invalid user ids if self.is_mine(UserID.from_string(user_id)): local_query[user_id] = device_ids @@ -826,7 +855,7 @@ class E2eKeysHandler: device_keys["user_id"] == user_id and device_keys["device_id"] == device_id ): - await self.device_key_uploader( + await self.upload_device_keys_for_user( user_id=user_id, device_id=device_id, keys={"device_keys": device_keys}, @@ -894,9 +923,6 @@ class E2eKeysHandler: device_keys: the `device_keys` of an /keys/upload request. """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - time_now = self.clock.time_msec() device_keys = keys["device_keys"] @@ -988,9 +1014,6 @@ class E2eKeysHandler: user_id: the user uploading the keys keys: the signing keys """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - # if a master key is uploaded, then check it. Otherwise, load the # stored master key, to check signatures on other keys if "master_key" in keys: @@ -1081,9 +1104,6 @@ class E2eKeysHandler: Raises: SynapseError: if the signatures dict is not valid. """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - failures = {} # signatures to be stored. Each item will be a SignatureListItem @@ -1178,7 +1198,7 @@ class E2eKeysHandler: devices = devices[user_id] except SynapseError as e: failure = _exception_to_failure(e) - failures[user_id] = {device: failure for device in signatures.keys()} + failures[user_id] = dict.fromkeys(signatures.keys(), failure) return signature_list, failures for device_id, device in signatures.items(): @@ -1318,7 +1338,7 @@ class E2eKeysHandler: except SynapseError as e: failure = _exception_to_failure(e) for user, devicemap in signatures.items(): - failures[user] = {device_id: failure for device_id in devicemap.keys()} + failures[user] = dict.fromkeys(devicemap.keys(), failure) return signature_list, failures for target_user, devicemap in signatures.items(): @@ -1359,9 +1379,7 @@ class E2eKeysHandler: # other devices were signed -- mark those as failures logger.debug("upload signature: too many devices specified") failure = _exception_to_failure(NotFoundError("Unknown device")) - failures[target_user] = { - device: failure for device in other_devices - } + failures[target_user] = dict.fromkeys(other_devices, failure) if user_signing_key_id in master_key.get("signatures", {}).get( user_id, {} @@ -1382,9 +1400,7 @@ class E2eKeysHandler: except SynapseError as e: failure = _exception_to_failure(e) if device_id is None: - failures[target_user] = { - device_id: failure for device_id in devicemap.keys() - } + failures[target_user] = dict.fromkeys(devicemap.keys(), failure) else: failures.setdefault(target_user, {})[device_id] = failure @@ -1461,9 +1477,6 @@ class E2eKeysHandler: A tuple of the retrieved key content, the key's ID and the matching VerifyKey. If the key cannot be retrieved, all values in the tuple will instead be None. """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - try: remote_result = await self.federation.query_user_devices( user.domain, user.to_string() @@ -1764,7 +1777,7 @@ class SigningKeyEduUpdater: self.clock = hs.get_clock() device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) self._device_handler = device_handler self._remote_edu_linearizer = Linearizer(name="remote_signing_key") diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index f397911f28..623fd33f13 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -20,9 +20,7 @@ # import logging -from typing import TYPE_CHECKING, Dict, Optional, cast - -from typing_extensions import Literal +from typing import TYPE_CHECKING, Dict, Literal, Optional, cast from synapse.api.errors import ( Codes, diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index c4dbf22408..1f1f67dc0d 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -23,6 +23,8 @@ from typing import TYPE_CHECKING, List, Mapping, Optional, Union from synapse import event_auth from synapse.api.constants import ( + CREATOR_POWER_LEVEL, + EventContentFields, EventTypes, JoinRules, Membership, @@ -141,6 +143,8 @@ class EventAuthHandler: Raises: SynapseError if no appropriate user is found. """ + create_event_id = current_state_ids[(EventTypes.Create, "")] + create_event = await self._store.get_event(create_event_id) power_level_event_id = current_state_ids.get((EventTypes.PowerLevels, "")) invite_level = 0 users_default_level = 0 @@ -156,15 +160,28 @@ class EventAuthHandler: # Find the user with the highest power level (only interested in local # users). + user_power_level = 0 + chosen_user = None local_users_in_room = await self._store.get_local_users_in_room(room_id) - chosen_user = max( - local_users_in_room, - key=lambda user: users.get(user, users_default_level), - default=None, - ) + if create_event.room_version.msc4289_creator_power_enabled: + creators = set( + create_event.content.get(EventContentFields.ADDITIONAL_CREATORS, []) + ) + creators.add(create_event.sender) + local_creators = creators.intersection(set(local_users_in_room)) + if len(local_creators) > 0: + chosen_user = local_creators.pop() # random creator + user_power_level = CREATOR_POWER_LEVEL + else: + chosen_user = max( + local_users_in_room, + key=lambda user: users.get(user, users_default_level), + default=None, + ) + # Return the chosen if they can issue invites. + if chosen_user: + user_power_level = users.get(chosen_user, users_default_level) - # Return the chosen if they can issue invites. - user_power_level = users.get(chosen_user, users_default_level) if chosen_user and user_power_level >= invite_level: logger.debug( "Found a user who can issue invites %s with power level %d >= invite level %d", diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2b7aad5b58..34aae7ef3c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -71,13 +71,11 @@ from synapse.handlers.pagination import PURGE_PAGINATION_LOCK_NAME from synapse.http.servlet import assert_params_in_dict from synapse.logging.context import nested_logging_context from synapse.logging.opentracing import SynapseTags, set_tag, tag_args, trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.module_api import NOT_SPAM -from synapse.replication.http.federation import ( - ReplicationCleanRoomRestServlet, - ReplicationStoreRoomOnOutlierMembershipRestServlet, -) from synapse.storage.databases.main.events_worker import EventRedactBehaviour +from synapse.storage.invite_rule import InviteRule from synapse.types import JsonDict, StrCollection, get_domain_from_id from synapse.types.state import StateFilter from synapse.util.async_helpers import Linearizer @@ -93,7 +91,7 @@ logger = logging.getLogger(__name__) backfill_processing_before_timer = Histogram( "synapse_federation_backfill_processing_before_time_seconds", "sec", - [], + labelnames=[SERVER_NAME_LABEL], buckets=( 0.1, 0.5, @@ -162,19 +160,6 @@ class FederationHandler: self._notifier = hs.get_notifier() self._worker_locks = hs.get_worker_locks_handler() - self._clean_room_for_join_client = ReplicationCleanRoomRestServlet.make_client( - hs - ) - - if hs.config.worker.worker_app: - self._maybe_store_room_on_outlier_membership = ( - ReplicationStoreRoomOnOutlierMembershipRestServlet.make_client(hs) - ) - else: - self._maybe_store_room_on_outlier_membership = ( - self.store.maybe_store_room_on_outlier_membership - ) - self._room_backfill = Linearizer("room_backfill") self._third_party_event_rules = ( @@ -203,7 +188,9 @@ class FederationHandler: # were shut down. if not hs.config.worker.worker_app: run_as_background_process( - "resume_sync_partial_state_room", self._resume_partial_state_room_sync + "resume_sync_partial_state_room", + self.server_name, + self._resume_partial_state_room_sync, ) @trace @@ -332,6 +319,7 @@ class FederationHandler: ) run_as_background_process( "_maybe_backfill_inner_anyway_with_max_depth", + self.server_name, self.maybe_backfill, room_id=room_id, # We use `MAX_DEPTH` so that we find all backfill points next @@ -546,9 +534,9 @@ class FederationHandler: # backfill points regardless of `current_depth`. if processing_start_time is not None: processing_end_time = self.clock.time_msec() - backfill_processing_before_timer.observe( - (processing_end_time - processing_start_time) / 1000 - ) + backfill_processing_before_timer.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).observe((processing_end_time - processing_start_time) / 1000) success = await try_backfill(likely_domains) if success: @@ -646,7 +634,7 @@ class FederationHandler: # room. # In short, the races either have an acceptable outcome or should be # impossible. - await self._clean_room_for_join(room_id) + await self.store.clean_room_for_join(room_id) try: # Try the host we successfully got a response to /make_join/ @@ -714,10 +702,19 @@ class FederationHandler: # We may want to reset the partial state info if it's from an # old, failed partial state join. # https://github.com/matrix-org/synapse/issues/13000 + + # FIXME: Ideally, we would store the full stream token here + # not just the minimum stream ID, so that we can compute an + # accurate list of device changes when un-partial-ing the + # room. The only side effect of this is that we may send + # extra unecessary device list outbound pokes through + # federation, which is harmless. + device_lists_stream_id = self.store.get_device_stream_token().stream + await self.store.store_partial_state_room( room_id=room_id, servers=ret.servers_in_room, - device_lists_stream_id=self.store.get_device_stream_token(), + device_lists_stream_id=device_lists_stream_id, joined_via=origin, ) @@ -805,7 +802,10 @@ class FederationHandler: # have. Hence we fire off the background task, but don't wait for it. run_as_background_process( - "handle_queued_pdus", self._handle_queued_pdus, room_queue + "handle_queued_pdus", + self.server_name, + self._handle_queued_pdus, + room_queue, ) async def do_knock( @@ -856,7 +856,7 @@ class FederationHandler: event.internal_metadata.out_of_band_membership = True # Record the room ID and its version so that we have a record of the room - await self._maybe_store_room_on_outlier_membership( + await self.store.maybe_store_room_on_outlier_membership( room_id=event.room_id, room_version=event_format_version ) @@ -880,6 +880,9 @@ class FederationHandler: if stripped_room_state is None: raise KeyError("Missing 'knock_room_state' field in send_knock response") + if not isinstance(stripped_room_state, list): + raise TypeError("'knock_room_state' has wrong type") + event.unsigned["knock_room_state"] = stripped_room_state context = EventContext.for_outlier(self._storage_controllers) @@ -1058,8 +1061,8 @@ class FederationHandler: if self.hs.config.server.block_non_admin_invites: raise SynapseError(403, "This server does not accept room invites") - spam_check = await self._spam_checker_module_callbacks.user_may_invite( - event.sender, event.state_key, event.room_id + spam_check = ( + await self._spam_checker_module_callbacks.federated_user_may_invite(event) ) if spam_check != NOT_SPAM: raise SynapseError( @@ -1086,6 +1089,22 @@ class FederationHandler: if event.state_key == self._server_notices_mxid: raise SynapseError(HTTPStatus.FORBIDDEN, "Cannot invite this user") + # check the invitee's configuration and apply rules + invite_config = await self.store.get_invite_config_for_user(event.state_key) + rule = invite_config.get_invite_rule(event.sender) + if rule == InviteRule.BLOCK: + logger.info( + "Automatically rejecting invite from %s due to the invite filtering rules of %s", + event.sender, + event.state_key, + ) + raise SynapseError( + 403, + "You are not permitted to invite this user.", + errcode=Codes.INVITE_BLOCKED, + ) + # InviteRule.IGNORE is handled at the sync layer + # We retrieve the room member handler here as to not cause a cyclic dependency member_handler = self.hs.get_room_member_handler() # We don't rate limit based on room ID, as that should be done by @@ -1095,7 +1114,7 @@ class FederationHandler: # keep a record of the room version, if we don't yet know it. # (this may get overwritten if we later get a different room version in a # join dance). - await self._maybe_store_room_on_outlier_membership( + await self.store.maybe_store_room_on_outlier_membership( room_id=event.room_id, room_version=room_version ) @@ -1309,9 +1328,9 @@ class FederationHandler: if state_key is not None: # the event was not rejected (get_event raises a NotFoundError for rejected # events) so the state at the event should include the event itself. - assert ( - state_map.get((event.type, state_key)) == event.event_id - ), "State at event did not include event itself" + assert state_map.get((event.type, state_key)) == event.event_id, ( + "State at event did not include event itself" + ) # ... but we need the state *before* that event if "replaces_state" in event.unsigned: @@ -1741,18 +1760,6 @@ class FederationHandler: if "valid" not in response or not response["valid"]: raise AuthError(403, "Third party certificate was invalid") - async def _clean_room_for_join(self, room_id: str) -> None: - """Called to clean up any data in DB for a given room, ready for the - server to join the room. - - Args: - room_id - """ - if self.config.worker.worker_app: - await self._clean_room_for_join_client(room_id) - else: - await self.store.clean_room_for_join(room_id) - async def get_room_complexity( self, remote_room_hosts: List[str], room_id: str ) -> Optional[dict]: @@ -1870,7 +1877,9 @@ class FederationHandler: ) run_as_background_process( - desc="sync_partial_state_room", func=_sync_partial_state_room_wrapper + desc="sync_partial_state_room", + server_name=self.server_name, + func=_sync_partial_state_room_wrapper, ) async def _sync_partial_state_room( diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index c85deaed56..1e47b4ef4f 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -66,7 +66,11 @@ from synapse.event_auth import ( validate_event_for_room_version, ) from synapse.events import EventBase -from synapse.events.snapshot import EventContext, UnpersistedEventContextBase +from synapse.events.snapshot import ( + EventContext, + EventPersistencePair, + UnpersistedEventContextBase, +) from synapse.federation.federation_client import InvalidResponseError, PulledPduInfo from synapse.logging.context import nested_logging_context from synapse.logging.opentracing import ( @@ -76,10 +80,8 @@ from synapse.logging.opentracing import ( tag_args, trace, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.replication.http.devices import ( - ReplicationMultiUserDevicesResyncRestServlet, -) from synapse.replication.http.federation import ( ReplicationFederationSendEventsRestServlet, ) @@ -108,13 +110,14 @@ logger = logging.getLogger(__name__) soft_failed_event_counter = Counter( "synapse_federation_soft_failed_events_total", "Events received over federation that we marked as soft_failed", + labelnames=[SERVER_NAME_LABEL], ) # Added to debug performance and track progress on optimizations backfill_processing_after_timer = Histogram( "synapse_federation_backfill_processing_after_time_seconds", "sec", - [], + labelnames=[SERVER_NAME_LABEL], buckets=( 0.1, 0.25, @@ -149,8 +152,11 @@ class FederationEventHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._clock = hs.get_clock() self._store = hs.get_datastores().main + self._state_store = hs.get_datastores().state + self._state_deletion_store = hs.get_datastores().state_deletion self._storage_controllers = hs.get_storage_controllers() self._state_storage_controller = self._storage_controllers.state @@ -171,19 +177,13 @@ class FederationEventHandler: self._is_mine_id = hs.is_mine_id self._is_mine_server_name = hs.is_mine_server_name - self._server_name = hs.hostname self._instance_name = hs.get_instance_name() self._config = hs.config self._ephemeral_messages_enabled = hs.config.server.enable_ephemeral_messages self._send_events = ReplicationFederationSendEventsRestServlet.make_client(hs) - if hs.config.worker.worker_app: - self._multi_user_device_resync = ( - ReplicationMultiUserDevicesResyncRestServlet.make_client(hs) - ) - else: - self._device_list_updater = hs.get_device_handler().device_list_updater + self._device_list_updater = hs.get_device_handler().device_list_updater # When joining a room we need to queue any events for that room up. # For each room, a list of (pdu, origin) tuples. @@ -248,16 +248,54 @@ class FederationEventHandler: self.room_queues[room_id].append((pdu, origin)) return - # If we're not in the room just ditch the event entirely. This is - # probably an old server that has come back and thinks we're still in - # the room (or we've been rejoined to the room by a state reset). + # If we're not in the room just ditch the event entirely (and not + # invited). This is probably an old server that has come back and thinks + # we're still in the room (or we've been rejoined to the room by a state + # reset). # # Note that if we were never in the room then we would have already # dropped the event, since we wouldn't know the room version. is_in_room = await self._event_auth_handler.is_host_in_room( - room_id, self._server_name + room_id, self.server_name ) if not is_in_room: + # Check if this is a leave event rescinding an invite + if ( + pdu.type == EventTypes.Member + and pdu.membership == Membership.LEAVE + and pdu.state_key != pdu.sender + and self._is_mine_id(pdu.state_key) + ): + ( + membership, + membership_event_id, + ) = await self._store.get_local_current_membership_for_user_in_room( + pdu.state_key, pdu.room_id + ) + if ( + membership == Membership.INVITE + and membership_event_id + and membership_event_id + in pdu.auth_event_ids() # The invite should be in the auth events of the rescission. + ): + invite_event = await self._store.get_event( + membership_event_id, allow_none=True + ) + + # We cannot fully auth the rescission event, but we can + # check if the sender of the leave event is the same as the + # invite. + # + # Technically, a room admin could rescind the invite, but we + # have no way of knowing who is and isn't a room admin. + if invite_event and pdu.sender == invite_event.sender: + # Handle the rescission event + pdu.internal_metadata.outlier = True + pdu.internal_metadata.out_of_band_membership = True + context = EventContext.for_outlier(self._storage_controllers) + await self.persist_events_and_notify(room_id, [(pdu, context)]) + return + logger.info( "Ignoring PDU from %s as we're not in the room", origin, @@ -345,7 +383,7 @@ class FederationEventHandler: async def on_send_membership_event( self, origin: str, event: EventBase - ) -> Tuple[EventBase, EventContext]: + ) -> EventPersistencePair: """ We have received a join/leave/knock event for a room via send_join/leave/knock. @@ -580,7 +618,9 @@ class FederationEventHandler: room_version.identifier, state_maps_to_resolve, event_map=None, - state_res_store=StateResolutionStore(self._store), + state_res_store=StateResolutionStore( + self._store, self._state_deletion_store + ), ) ) else: @@ -694,7 +734,9 @@ class FederationEventHandler: if not events: return - with backfill_processing_after_timer.time(): + with backfill_processing_after_timer.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).time(): # if there are any events in the wrong room, the remote server is buggy and # should not be trusted. for ev in events: @@ -934,6 +976,7 @@ class FederationEventHandler: if len(events_with_failed_pull_attempts) > 0: run_as_background_process( "_process_new_pulled_events_with_failed_pull_attempts", + self.server_name, _process_new_pulled_events, events_with_failed_pull_attempts, ) @@ -1179,7 +1222,9 @@ class FederationEventHandler: room_version, state_maps, event_map={event_id: event}, - state_res_store=StateResolutionStore(self._store), + state_res_store=StateResolutionStore( + self._store, self._state_deletion_store + ), ) except Exception as e: @@ -1525,6 +1570,7 @@ class FederationEventHandler: if resync: run_as_background_process( "resync_device_due_to_pdu", + self.server_name, self._resync_device, event.sender, ) @@ -1538,12 +1584,7 @@ class FederationEventHandler: await self._store.mark_remote_users_device_caches_as_stale((sender,)) # Immediately attempt a resync in the background - if self._config.worker.worker_app: - await self._multi_user_device_resync(user_ids=[sender]) - else: - await self._device_list_updater.multi_user_device_resync( - user_ids=[sender] - ) + await self._device_list_updater.multi_user_device_resync(user_ids=[sender]) except Exception: logger.exception("Failed to resync device for %s", sender) @@ -1713,7 +1754,7 @@ class FederationEventHandler: ) auth_map.update(persisted_events) - events_and_contexts_to_persist: List[Tuple[EventBase, EventContext]] = [] + events_and_contexts_to_persist: List[EventPersistencePair] = [] async def prep(event: EventBase) -> None: with nested_logging_context(suffix=event.event_id): @@ -1729,6 +1770,9 @@ class FederationEventHandler: event, auth_event_id, ) + # Drop the event from the auth_map too, else we may incorrectly persist + # events which depend on this dropped event. + auth_map.pop(event.event_id, None) return auth.append(ae) @@ -1874,7 +1918,9 @@ class FederationEventHandler: room_version, [local_state_id_map, claimed_auth_events_id_map], event_map=None, - state_res_store=StateResolutionStore(self._store), + state_res_store=StateResolutionStore( + self._store, self._state_deletion_store + ), ) ) else: @@ -2014,7 +2060,9 @@ class FederationEventHandler: room_version, state_sets, event_map=None, - state_res_store=StateResolutionStore(self._store), + state_res_store=StateResolutionStore( + self._store, self._state_deletion_store + ), ) ) else: @@ -2052,7 +2100,9 @@ class FederationEventHandler: "hs": origin, }, ) - soft_failed_event_counter.inc() + soft_failed_event_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() event.internal_metadata.soft_failed = True async def _load_or_fetch_auth_events_for_event( @@ -2217,7 +2267,7 @@ class FederationEventHandler: async def persist_events_and_notify( self, room_id: str, - event_and_contexts: Sequence[Tuple[EventBase, EventContext]], + event_and_contexts: Sequence[EventPersistencePair], backfilled: bool = False, ) -> int: """Persists events and tells the notifier/pushers about them, if @@ -2272,8 +2322,9 @@ class FederationEventHandler: event_and_contexts, backfilled=backfilled ) - # After persistence we always need to notify replication there may - # be new data. + # After persistence, we never notify clients (wake up `/sync` streams) about + # backfilled events but it's important to let all the workers know about any + # new event (backfilled or not) because TODO self._notifier.notify_replication() if self._ephemeral_messages_enabled: diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 89191217d6..d96b585308 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -218,7 +218,7 @@ class IdentityHandler: return data except HttpResponseException as e: - logger.error("3PID bind failed with Matrix error: %r", e) + logger.exception("3PID bind failed with Matrix error: %r", e) raise e.to_synapse_error() except RequestTimedOutError: raise SynapseError(500, "Timed out contacting identity server") @@ -323,7 +323,7 @@ class IdentityHandler: # The remote server probably doesn't support unbinding (yet) logger.warning("Received %d response while unbinding threepid", e.code) else: - logger.error("Failed to unbind threepid on identity server: %s", e) + logger.exception("Failed to unbind threepid on identity server: %s", e) raise SynapseError(500, "Failed to contact identity server") except RequestTimedOutError: raise SynapseError(500, "Timed out contacting identity server") diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index bd3c87f5f4..75d64d2d50 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -60,6 +60,7 @@ logger = logging.getLogger(__name__) class InitialSyncHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.auth = hs.get_auth() self.state_handler = hs.get_state_handler() @@ -77,7 +78,11 @@ class InitialSyncHandler: bool, bool, ] - ] = ResponseCache(hs.get_clock(), "initial_sync_cache") + ] = ResponseCache( + clock=hs.get_clock(), + name="initial_sync_cache", + server_name=self.server_name, + ) self._event_serializer = hs.get_event_client_serializer() self._storage_controllers = hs.get_storage_controllers() self._state_storage_controller = self._storage_controllers.state diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index df3010ecf6..d850b617d8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -22,7 +22,7 @@ import logging import random from http import HTTPStatus -from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Sequence, Tuple from canonicaljson import encode_canonical_json @@ -55,7 +55,12 @@ from synapse.api.urls import ConsentURIBuilder from synapse.event_auth import validate_event_for_room_version from synapse.events import EventBase, relation_from_event from synapse.events.builder import EventBuilder -from synapse.events.snapshot import EventContext, UnpersistedEventContextBase +from synapse.events.snapshot import ( + EventContext, + EventPersistencePair, + UnpersistedEventContext, + UnpersistedEventContextBase, +) from synapse.events.utils import SerializeEventConfig, maybe_upsert_event_field from synapse.events.validator import EventValidator from synapse.handlers.directory import DirectoryHandler @@ -63,10 +68,10 @@ from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging import opentracing from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.replication.http.send_event import ReplicationSendEventRestServlet from synapse.replication.http.send_events import ReplicationSendEventsRestServlet from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( + JsonDict, PersistedEventPosition, Requester, RoomAlias, @@ -92,6 +97,7 @@ class MessageHandler: """Contains some read only APIs to get state about a room""" def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.auth = hs.get_auth() self.clock = hs.get_clock() self.state = hs.get_state_handler() @@ -107,7 +113,7 @@ class MessageHandler: if not hs.config.worker.worker_app: run_as_background_process( - "_schedule_next_expiry", self._schedule_next_expiry + "_schedule_next_expiry", self.server_name, self._schedule_next_expiry ) async def get_room_data( @@ -143,9 +149,9 @@ class MessageHandler: elif membership == Membership.LEAVE: key = (event_type, state_key) # If the membership is not JOIN, then the event ID should exist. - assert ( - membership_event_id is not None - ), "check_user_in_room_or_world_readable returned invalid data" + assert membership_event_id is not None, ( + "check_user_in_room_or_world_readable returned invalid data" + ) room_state = await self._state_storage_controller.get_state_for_events( [membership_event_id], StateFilter.from_types([key]) ) @@ -242,9 +248,9 @@ class MessageHandler: room_state = await self.store.get_events(state_ids.values()) elif membership == Membership.LEAVE: # If the membership is not JOIN, then the event ID should exist. - assert ( - membership_event_id is not None - ), "check_user_in_room_or_world_readable returned invalid data" + assert membership_event_id is not None, ( + "check_user_in_room_or_world_readable returned invalid data" + ) room_state_events = ( await self._state_storage_controller.get_state_for_events( [membership_event_id], state_filter=state_filter @@ -439,6 +445,7 @@ class MessageHandler: delay, run_as_background_process, "_expire_event", + self.server_name, self._expire_event, event_id, ) @@ -460,7 +467,7 @@ class MessageHandler: # date from the database in the same database transaction. await self.store.expire_event(event_id) except Exception as e: - logger.error("Could not expire event %s: %r", event_id, e) + logger.exception("Could not expire event %s: %r", event_id, e) # Schedule the expiry of the next event to expire. await self._schedule_next_expiry() @@ -476,16 +483,16 @@ _DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY = 7 * 24 * 60 * 60 * 1000 class EventCreationHandler: def __init__(self, hs: "HomeServer"): self.hs = hs + self.validator = EventValidator() + self.event_builder_factory = hs.get_event_builder_factory() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.auth_blocking = hs.get_auth_blocking() self._event_auth_handler = hs.get_event_auth_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.state = hs.get_state_handler() - self.clock = hs.get_clock() - self.validator = EventValidator() self.profile_handler = hs.get_profile_handler() - self.event_builder_factory = hs.get_event_builder_factory() - self.server_name = hs.hostname self.notifier = hs.get_notifier() self.config = hs.config self.require_membership_for_aliases = ( @@ -495,10 +502,10 @@ class EventCreationHandler: self._instance_name = hs.get_instance_name() self._notifier = hs.get_notifier() self._worker_lock_handler = hs.get_worker_locks_handler() + self._policy_handler = hs.get_room_policy_handler() self.room_prejoin_state_types = self.hs.config.api.room_prejoin_state - self.send_event = ReplicationSendEventRestServlet.make_client(hs) self.send_events = ReplicationSendEventsRestServlet.make_client(hs) self.request_ratelimiter = hs.get_request_ratelimiter() @@ -540,6 +547,7 @@ class EventCreationHandler: self.clock.looping_call( lambda: run_as_background_process( "send_dummy_events_to_fill_extremities", + self.server_name, self._send_dummy_events_to_fill_extremities, ), 5 * 60 * 1000, @@ -557,8 +565,9 @@ class EventCreationHandler: self._external_cache_joined_hosts_updates: Optional[ExpiringCache] = None if self._external_cache.is_enabled(): self._external_cache_joined_hosts_updates = ExpiringCache( - "_external_cache_joined_hosts_updates", - self.clock, + cache_name="_external_cache_joined_hosts_updates", + server_name=self.server_name, + clock=self.clock, expiry_ms=30 * 60 * 1000, ) @@ -567,7 +576,6 @@ class EventCreationHandler: requester: Requester, event_dict: dict, txn_id: Optional[str] = None, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, auth_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, @@ -593,10 +601,6 @@ class EventCreationHandler: requester event_dict: An entire event txn_id - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: the forward extremities to use as the prev_events for the new event. @@ -644,18 +648,51 @@ class EventCreationHandler: """ await self.auth_blocking.check_auth_blocking(requester=requester) - if event_dict["type"] == EventTypes.Message: + # The requester may be a regular user, but puppeted by the server. + request_by_server = ( + requester.authenticated_entity == self.hs.config.server.server_name + ) + + # If the request is initiated by the server, ignore whether the + # requester or target is suspended. + if not request_by_server: requester_suspended = await self.store.get_user_suspended_status( requester.user.to_string() ) if requester_suspended: - raise SynapseError( - 403, - "Sending messages while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) + # We want to allow suspended users to perform "corrective" actions + # asked of them by server admins, such as redact their messages and + # leave rooms. + if event_dict["type"] in ["m.room.redaction", "m.room.member"]: + if event_dict["type"] == "m.room.redaction": + event = await self.store.get_event( + event_dict["content"]["redacts"], allow_none=True + ) + if event: + if event.sender != requester.user.to_string(): + raise SynapseError( + 403, + "You can only redact your own events while account is suspended.", + Codes.USER_ACCOUNT_SUSPENDED, + ) + if event_dict["type"] == "m.room.member": + if event_dict["content"]["membership"] != "leave": + raise SynapseError( + 403, + "Changing membership while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) + else: + raise SynapseError( + 403, + "Sending messages while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) - if event_dict["type"] == EventTypes.Create and event_dict["state_key"] == "": + is_create_event = ( + event_dict["type"] == EventTypes.Create and event_dict["state_key"] == "" + ) + if is_create_event: room_version_id = event_dict["content"]["room_version"] maybe_room_version_obj = KNOWN_ROOM_VERSIONS.get(room_version_id) if not maybe_room_version_obj: @@ -694,7 +731,6 @@ class EventCreationHandler: event, unpersisted_context = await self.create_new_client_event( builder=builder, requester=requester, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids, state_event_ids=state_event_ids, @@ -762,6 +798,7 @@ class EventCreationHandler: """ # the only thing the user can do is join the server notices room. if builder.type == EventTypes.Member: + assert builder.room_id is not None membership = builder.content.get("membership", None) if membership == Membership.JOIN: return await self.store.is_server_notice_room(builder.room_id) @@ -922,7 +959,6 @@ class EventCreationHandler: self, requester: Requester, event_dict: dict, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, ratelimit: bool = True, @@ -939,10 +975,6 @@ class EventCreationHandler: Args: requester: The requester sending the event. event_dict: An entire event. - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events. Should normally be left as None to automatically request them @@ -1028,7 +1060,6 @@ class EventCreationHandler: return await self._create_and_send_nonmember_event_locked( requester=requester, event_dict=event_dict, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, ratelimit=ratelimit, @@ -1042,7 +1073,6 @@ class EventCreationHandler: self, requester: Requester, event_dict: dict, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, ratelimit: bool = True, @@ -1074,7 +1104,6 @@ class EventCreationHandler: requester, event_dict, txn_id=txn_id, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, outlier=outlier, @@ -1086,6 +1115,21 @@ class EventCreationHandler: event.sender, ) + policy_allowed = await self._policy_handler.is_event_allowed(event) + if not policy_allowed: + # We shouldn't need to set the metadata because the raise should + # cause the request to be denied, but just in case: + event.internal_metadata.policy_server_spammy = True + logger.warning( + "Event not allowed by policy server, rejecting %s", + event.event_id, + ) + raise SynapseError( + 403, + "This message has been rejected as probable spam", + Codes.FORBIDDEN, + ) + spam_check_result = ( await self._spam_checker_module_callbacks.check_event_for_spam( event @@ -1097,7 +1141,7 @@ class EventCreationHandler: [code, dict] = spam_check_result raise SynapseError( 403, - "This message had been rejected as probable spam", + "This message has been rejected as probable spam", code, dict, ) @@ -1145,7 +1189,6 @@ class EventCreationHandler: self, builder: EventBuilder, requester: Optional[Requester] = None, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, auth_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, @@ -1165,10 +1208,6 @@ class EventCreationHandler: Args: builder: requester: - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: the forward extremities to use as the prev_events for the new event. @@ -1206,7 +1245,6 @@ class EventCreationHandler: if state_event_ids is not None: # Do a quick check to make sure that prev_event_ids is present to # make the type-checking around `builder.build` happy. - # prev_event_ids could be an empty array though. assert prev_event_ids is not None temp_event = await builder.build( @@ -1226,30 +1264,49 @@ class EventCreationHandler: for_verification=False, ) + if ( + builder.room_version.msc4291_room_ids_as_hashes + and builder.type == EventTypes.Create + and builder.is_state() + ): + if builder.room_id is not None: + raise SynapseError( + 400, + "Cannot resend m.room.create event", + Codes.INVALID_PARAM, + ) + else: + assert builder.room_id is not None + if prev_event_ids is not None: assert len(prev_event_ids) <= 10, ( "Attempting to create an event with %i prev_events" % (len(prev_event_ids),) ) else: - prev_event_ids = await self.store.get_prev_events_for_room(builder.room_id) + if builder.room_id: + prev_event_ids = await self.store.get_prev_events_for_room( + builder.room_id + ) + else: + prev_event_ids = [] # can only happen for the create event in MSC4291 rooms + if builder.type == EventTypes.Create and builder.is_state(): + if len(prev_event_ids) != 0: + raise SynapseError( + 400, + "Cannot resend m.room.create event", + Codes.INVALID_PARAM, + ) + + # We now ought to have some `prev_events` (unless it's a create event). + # # Do a quick sanity check here, rather than waiting until we've created the # event and then try to auth it (which fails with a somewhat confusing "No # create event in auth events") - if allow_no_prev_events: - # We allow events with no `prev_events` but it better have some `auth_events` - assert ( - builder.type == EventTypes.Create - # Allow an event to have empty list of prev_event_ids - # only if it has auth_event_ids. - or auth_event_ids - ), "Attempting to create a non-m.room.create event with no prev_events or auth_event_ids" - else: - # we now ought to have some prev_events (unless it's a create event). - assert ( - builder.type == EventTypes.Create or prev_event_ids - ), "Attempting to create a non-m.room.create event with no prev_events" + assert builder.type == EventTypes.Create or len(prev_event_ids) > 0, ( + "Attempting to create an event with no prev_events" + ) if for_batch: assert prev_event_ids is not None @@ -1383,7 +1440,7 @@ class EventCreationHandler: async def handle_new_client_event( self, requester: Requester, - events_and_context: List[Tuple[EventBase, EventContext]], + events_and_context: List[EventPersistencePair], ratelimit: bool = True, extra_users: Optional[List[UserID]] = None, ignore_shadow_ban: bool = False, @@ -1440,6 +1497,12 @@ class EventCreationHandler: ) return prev_event + if not event.is_state() and event.type in [ + EventTypes.Message, + EventTypes.Encrypted, + ]: + await self.store.set_room_participation(event.user_id, event.room_id) + if event.internal_metadata.is_out_of_band_membership(): # the only sort of out-of-band-membership events we expect to see here are # invite rejections and rescinded knocks that we have generated ourselves. @@ -1494,10 +1557,102 @@ class EventCreationHandler: return result + async def create_and_send_new_client_events( + self, + requester: Requester, + room_id: str, + prev_event_id: Optional[str], + event_dicts: Sequence[JsonDict], + ratelimit: bool = True, + ignore_shadow_ban: bool = False, + ) -> None: + """Helper to create and send a batch of new client events. + + This supports sending membership events in very limited circumstances + (namely that the event is valid as is and doesn't need federation + requests or anything). Callers should prefer to use `update_membership`, + which correctly handles membership events in all cases. We allow + sending membership events here as its useful when copying e.g. bans + between rooms. + + All other events and state events are supported. + + Args: + requester: The requester sending the events. + room_id: The room ID to send the events in. + prev_event_id: The event ID to use as the previous event for the first + of the events, must have already been persisted. + event_dicts: A sequence of event dictionaries to create and send. + ratelimit: Whether to rate limit this send. + ignore_shadow_ban: True if shadow-banned users should be allowed to + send these events. + """ + + if not event_dicts: + # Nothing to do. + return + + if prev_event_id is None: + # Pick the latest forward extremity as the previous event ID. + prev_event_ids = await self.store.get_forward_extremities_for_room(room_id) + prev_event_ids.sort(key=lambda x: x[2]) # Sort by depth. + prev_event_id = prev_event_ids[-1][0] + + state_groups = await self._storage_controllers.state.get_state_group_for_events( + [prev_event_id] + ) + if prev_event_id not in state_groups: + # This should only happen if we got passed a prev event ID that + # hasn't been persisted yet. + raise Exception("Previous event ID not found ") + + current_state_group = state_groups[prev_event_id] + state_map = await self._storage_controllers.state.get_state_ids_for_group( + current_state_group + ) + + events_and_contexts_to_send = [] + state_map = dict(state_map) + depth = None + + for event_dict in event_dicts: + event, context = await self.create_event( + requester=requester, + event_dict=event_dict, + prev_event_ids=[prev_event_id], + depth=depth, + # Take a copy to ensure each event gets a unique copy of + # state_map since it is modified below. + state_map=dict(state_map), + for_batch=True, + ) + events_and_contexts_to_send.append((event, context)) + + prev_event_id = event.event_id + depth = event.depth + 1 + if event.is_state(): + # If this is a state event, we need to update the state map + # so that it can be used for the next event. + state_map[(event.type, event.state_key)] = event.event_id + + datastore = self.hs.get_datastores().state + events_and_context = ( + await UnpersistedEventContext.batch_persist_unpersisted_contexts( + events_and_contexts_to_send, room_id, current_state_group, datastore + ) + ) + + await self.handle_new_client_event( + requester, + events_and_context, + ignore_shadow_ban=ignore_shadow_ban, + ratelimit=ratelimit, + ) + async def _persist_events( self, requester: Requester, - events_and_context: List[Tuple[EventBase, EventContext]], + events_and_context: List[EventPersistencePair], ratelimit: bool = True, extra_users: Optional[List[UserID]] = None, ) -> EventBase: @@ -1583,7 +1738,7 @@ class EventCreationHandler: raise async def cache_joined_hosts_for_events( - self, events_and_context: List[Tuple[EventBase, EventContext]] + self, events_and_context: List[EventPersistencePair] ) -> None: """Precalculate the joined hosts at each of the given events, when using Redis, so that external federation senders don't have to recalculate it themselves. @@ -1689,7 +1844,7 @@ class EventCreationHandler: async def persist_and_notify_client_events( self, requester: Requester, - events_and_context: List[Tuple[EventBase, EventContext]], + events_and_context: List[EventPersistencePair], ratelimit: bool = True, extra_users: Optional[List[UserID]] = None, ) -> EventBase: @@ -1928,6 +2083,7 @@ class EventCreationHandler: # matters as sometimes presence code can take a while. run_as_background_process( "bump_presence_active_time", + self.server_name, self._bump_active_time, requester.user, requester.device_id, @@ -2018,7 +2174,8 @@ class EventCreationHandler: # dependent on _DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY logger.info( "Failed to send dummy event into room %s. Will exclude it from " - "future attempts until cache expires" % (room_id,) + "future attempts until cache expires", + room_id, ) now = self.clock.time_msec() self._rooms_to_exclude_from_dummy_event_insertion[room_id] = now @@ -2077,7 +2234,9 @@ class EventCreationHandler: except AuthError: logger.info( "Failed to send dummy event into room %s for user %s due to " - "lack of power. Will try another user" % (room_id, user_id) + "lack of power. Will try another user", + room_id, + user_id, ) return False @@ -2107,6 +2266,7 @@ class EventCreationHandler: original_event.room_version, third_party_result ) self.validator.validate_builder(builder) + assert builder.room_id is not None except SynapseError as e: raise Exception( "Third party rules module created an invalid event: " + e.msg, diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index 22b59829fa..8f3e717fb4 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -31,6 +31,7 @@ from typing import ( List, Optional, Type, + TypedDict, TypeVar, Union, ) @@ -52,7 +53,6 @@ from pymacaroons.exceptions import ( MacaroonInitException, MacaroonInvalidSignatureException, ) -from typing_extensions import TypedDict from twisted.web.client import readBody from twisted.web.http_headers import Headers @@ -382,7 +382,12 @@ class OidcProvider: self._macaroon_generaton = macaroon_generator self._config = provider - self._callback_url: str = hs.config.oidc.oidc_callback_url + + self._callback_url: str + if provider.redirect_uri is not None: + self._callback_url = provider.redirect_uri + else: + self._callback_url = hs.config.oidc.oidc_callback_url # Calculate the prefix for OIDC callback paths based on the public_baseurl. # We'll insert this into the Path= parameter of any session cookies we set. @@ -462,6 +467,10 @@ class OidcProvider: self._sso_handler.register_identity_provider(self) + self.passthrough_authorization_parameters = ( + provider.passthrough_authorization_parameters + ) + def _validate_metadata(self, m: OpenIDProviderMetadata) -> None: """Verifies the provider metadata. @@ -554,12 +563,13 @@ class OidcProvider: raise ValueError("Unexpected subject") except Exception: logger.warning( - f"OIDC Back-Channel Logout is enabled for issuer {self.issuer!r} " + "OIDC Back-Channel Logout is enabled for issuer %r " "but it looks like the configured `user_mapping_provider` " "does not use the `sub` claim as subject. If it is the case, " "and you want Synapse to ignore the `sub` claim in OIDC " "Back-Channel Logouts, set `backchannel_logout_ignore_sub` " - "to `true` in the issuer config." + "to `true` in the issuer config.", + self.issuer, ) @property @@ -577,6 +587,24 @@ class OidcProvider: or self._user_profile_method == "userinfo_endpoint" ) + @property + def _uses_access_token(self) -> bool: + """Return True if the `access_token` will be used during the login process. + + This is useful to determine whether the access token + returned by the identity provider, and + any related metadata (such as the `at_hash` field in + the ID token), should be validated. + """ + # Currently, Synapse only uses the access_token to fetch user metadata + # from the userinfo endpoint. Therefore we only have a single criteria + # to check right now but this may change in the future and this function + # should be updated if more usages are introduced. + # + # For example, if we start to use the access_token given to us by the + # IdP for more things, such as accessing Resource Server APIs. + return self._uses_userinfo + @property def issuer(self) -> str: """The issuer identifying this provider.""" @@ -640,6 +668,11 @@ class OidcProvider: elif self._config.pkce_method == "never": metadata.pop("code_challenge_methods_supported", None) + if self._config.id_token_signing_alg_values_supported: + metadata["id_token_signing_alg_values_supported"] = ( + self._config.id_token_signing_alg_values_supported + ) + self._validate_metadata(metadata) return metadata @@ -794,10 +827,10 @@ class OidcProvider: if response.code < 400: logger.debug( "Invalid response from the authorization server: " - 'responded with a "{status}" ' - "but body has an error field: {error!r}".format( - status=status, error=resp["error"] - ) + 'responded with a "%s" ' + "but body has an error field: %r", + status, + resp["error"], ) description = resp.get("error_description", error) @@ -943,9 +976,16 @@ class OidcProvider: "nonce": nonce, "client_id": self._client_auth.client_id, } - if "access_token" in token: + if self._uses_access_token and "access_token" in token: # If we got an `access_token`, there should be an `at_hash` claim - # in the `id_token` that we can check against. + # in the `id_token` that we can check against. Setting this + # instructs authlib to check the value of `at_hash` in the + # ID token. + # + # We only need to verify the access token if we actually make + # use of it. Which currently only happens when we need to fetch + # the user's information from the userinfo_endpoint. Thus, this + # check is also gated on self._uses_userinfo. claims_params["access_token"] = token["access_token"] claims_options = {"iss": {"values": [metadata["issuer"]]}} @@ -995,14 +1035,27 @@ class OidcProvider: when everything is done (or None for UI Auth) ui_auth_session_id: The session ID of the ongoing UI Auth (or None if this is a login). - Returns: The redirect URL to the authorization endpoint. """ state = generate_token() - nonce = generate_token() + + # Generate a nonce 32 characters long. When encoded with base64url later on, + # the nonce will be 43 characters when sent to the identity provider. + # + # While RFC7636 does not specify a minimum length for the `nonce` + # parameter, the TI-Messenger IDP_FD spec v1.7.3 does require it to be + # between 43 and 128 characters. This spec concerns using Matrix for + # communication in German healthcare. + # + # As increasing the length only strengthens security, we use this length + # to allow TI-Messenger deployments using Synapse to satisfy this + # external spec. + # + # See https://github.com/element-hq/synapse/pull/18109 for more context. + nonce = generate_token(length=32) code_verifier = "" if not client_redirect_url: @@ -1054,6 +1107,13 @@ class OidcProvider: ) ) + # add passthrough additional authorization parameters + passthrough_authorization_parameters = self.passthrough_authorization_parameters + for parameter in passthrough_authorization_parameters: + parameter_value = parse_string(request, parameter) + if parameter_value: + additional_authorization_parameters.update({parameter: parameter_value}) + authorization_endpoint = metadata.get("authorization_endpoint") return prepare_grant_uri( authorization_endpoint, @@ -1326,7 +1386,8 @@ class OidcProvider: # support dynamic registration in Synapse at some point. if not self._config.backchannel_logout_enabled: logger.warning( - f"Received an OIDC Back-Channel Logout request from issuer {self.issuer!r} but it is disabled in config" + "Received an OIDC Back-Channel Logout request from issuer %r but it is disabled in config", + self.issuer, ) # TODO: this responds with a 400 status code, which is what the OIDC @@ -1738,5 +1799,5 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]): extras[key] = template.render(user=userinfo).strip() except Exception as e: # Log an error and skip this value (don't break login for this). - logger.error("Failed to render OIDC extra attribute %s: %s" % (key, e)) + logger.exception("Failed to render OIDC extra attribute %s: %s", key, e) return extras diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 4070b74b7a..df1a7e714c 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -79,12 +79,12 @@ class PaginationHandler: def __init__(self, hs: "HomeServer"): self.hs = hs + self.server_name = hs.hostname self.auth = hs.get_auth() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self._state_storage_controller = self._storage_controllers.state self.clock = hs.get_clock() - self._server_name = hs.hostname self._room_shutdown_handler = hs.get_room_shutdown_handler() self._relations_handler = hs.get_relations_handler() self._worker_locks = hs.get_worker_locks_handler() @@ -119,6 +119,7 @@ class PaginationHandler: run_as_background_process, job.interval, "purge_history_for_rooms_in_range", + self.server_name, self.purge_history_for_rooms_in_range, job.shortest_max_lifetime, job.longest_max_lifetime, @@ -245,6 +246,7 @@ class PaginationHandler: # other purges in the same room. run_as_background_process( PURGE_HISTORY_ACTION_NAME, + self.server_name, self.purge_history, room_id, token, @@ -395,7 +397,7 @@ class PaginationHandler: write=True, ): # first check that we have no users in this room - joined = await self.store.is_host_joined(room_id, self._server_name) + joined = await self.store.is_host_joined(room_id, self.server_name) if joined: if force: logger.info( @@ -604,6 +606,7 @@ class PaginationHandler: # for a costly federation call and processing. run_as_background_process( "maybe_backfill_in_the_background", + self.server_name, self.hs.get_federation_handler().maybe_backfill, room_id, curr_topo, diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 390cafa8f6..d7de20f884 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -105,7 +105,7 @@ from synapse.api.presence import UserDevicePresenceState, UserPresenceState from synapse.appservice import ApplicationService from synapse.events.presence_router import PresenceRouter from synapse.logging.context import run_in_background -from synapse.metrics import LaterGauge +from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import ( run_as_background_process, wrap_as_background_process, @@ -137,24 +137,52 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -notified_presence_counter = Counter("synapse_handler_presence_notified_presence", "") +notified_presence_counter = Counter( + "synapse_handler_presence_notified_presence", "", labelnames=[SERVER_NAME_LABEL] +) federation_presence_out_counter = Counter( - "synapse_handler_presence_federation_presence_out", "" + "synapse_handler_presence_federation_presence_out", + "", + labelnames=[SERVER_NAME_LABEL], +) +presence_updates_counter = Counter( + "synapse_handler_presence_presence_updates", "", labelnames=[SERVER_NAME_LABEL] +) +timers_fired_counter = Counter( + "synapse_handler_presence_timers_fired", "", labelnames=[SERVER_NAME_LABEL] ) -presence_updates_counter = Counter("synapse_handler_presence_presence_updates", "") -timers_fired_counter = Counter("synapse_handler_presence_timers_fired", "") federation_presence_counter = Counter( - "synapse_handler_presence_federation_presence", "" + "synapse_handler_presence_federation_presence", "", labelnames=[SERVER_NAME_LABEL] +) +bump_active_time_counter = Counter( + "synapse_handler_presence_bump_active_time", "", labelnames=[SERVER_NAME_LABEL] ) -bump_active_time_counter = Counter("synapse_handler_presence_bump_active_time", "") -get_updates_counter = Counter("synapse_handler_presence_get_updates", "", ["type"]) +get_updates_counter = Counter( + "synapse_handler_presence_get_updates", "", labelnames=["type", SERVER_NAME_LABEL] +) notify_reason_counter = Counter( - "synapse_handler_presence_notify_reason", "", ["locality", "reason"] + "synapse_handler_presence_notify_reason", + "", + labelnames=["locality", "reason", SERVER_NAME_LABEL], ) state_transition_counter = Counter( - "synapse_handler_presence_state_transition", "", ["locality", "from", "to"] + "synapse_handler_presence_state_transition", + "", + labelnames=["locality", "from", "to", SERVER_NAME_LABEL], +) + +presence_user_to_current_state_size_gauge = LaterGauge( + name="synapse_handlers_presence_user_to_current_state_size", + desc="", + labelnames=[SERVER_NAME_LABEL], +) + +presence_wheel_timer_size_gauge = LaterGauge( + name="synapse_handlers_presence_wheel_timer_size", + desc="", + labelnames=[SERVER_NAME_LABEL], ) # If a user was last active in the last LAST_ACTIVE_GRANULARITY, consider them @@ -484,6 +512,7 @@ class _NullContextManager(ContextManager[None]): class WorkerPresenceHandler(BasePresenceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self._presence_writer_instance = hs.config.worker.writers.presence[0] # Route presence EDUs to the right worker @@ -517,6 +546,7 @@ class WorkerPresenceHandler(BasePresenceHandler): "shutdown", run_as_background_process, "generic_presence.on_shutdown", + self.server_name, self._on_shutdown, ) @@ -666,7 +696,9 @@ class WorkerPresenceHandler(BasePresenceHandler): old_state = self.user_to_current_state.get(new_state.user_id) self.user_to_current_state[new_state.user_id] = new_state is_mine = self.is_mine_id(new_state.user_id) - if not old_state or should_notify(old_state, new_state, is_mine): + if not old_state or should_notify( + old_state, new_state, is_mine, self.server_name + ): state_to_notify.append(new_state) stream_id = token @@ -747,6 +779,9 @@ class WorkerPresenceHandler(BasePresenceHandler): class PresenceHandler(BasePresenceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = ( + hs.hostname + ) # nb must be called this for @wrap_as_background_process self.wheel_timer: WheelTimer[str] = WheelTimer() self.notifier = hs.get_notifier() @@ -756,11 +791,9 @@ class PresenceHandler(BasePresenceHandler): EduTypes.PRESENCE, self.incoming_presence ) - LaterGauge( - "synapse_handlers_presence_user_to_current_state_size", - "", - [], - lambda: len(self.user_to_current_state), + presence_user_to_current_state_size_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: {(self.server_name,): len(self.user_to_current_state)}, ) # The per-device presence state, maps user to devices to per-device presence state. @@ -814,6 +847,7 @@ class PresenceHandler(BasePresenceHandler): "shutdown", run_as_background_process, "presence.on_shutdown", + self.server_name, self._on_shutdown, ) @@ -858,11 +892,9 @@ class PresenceHandler(BasePresenceHandler): 60 * 1000, ) - LaterGauge( - "synapse_handlers_presence_wheel_timer_size", - "", - [], - lambda: len(self.wheel_timer), + presence_wheel_timer_size_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: {(self.server_name,): len(self.wheel_timer)}, ) # Used to handle sending of presence to newly joined users/servers @@ -941,7 +973,9 @@ class PresenceHandler(BasePresenceHandler): now = self.clock.time_msec() - with Measure(self.clock, "presence_update_states"): + with Measure( + self.clock, name="presence_update_states", server_name=self.server_name + ): # NOTE: We purposefully don't await between now and when we've # calculated what we want to do with the new states, to avoid races. @@ -969,6 +1003,7 @@ class PresenceHandler(BasePresenceHandler): prev_state, new_state, is_mine=self.is_mine_id(user_id), + our_server_name=self.server_name, wheel_timer=self.wheel_timer, now=now, # When overriding disabled presence, don't kick off all the @@ -988,10 +1023,14 @@ class PresenceHandler(BasePresenceHandler): # TODO: We should probably ensure there are no races hereafter - presence_updates_counter.inc(len(new_states)) + presence_updates_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(len(new_states)) if to_notify: - notified_presence_counter.inc(len(to_notify)) + notified_presence_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(len(to_notify)) await self._persist_and_notify(list(to_notify.values())) self.unpersisted_users_changes |= {s.user_id for s in new_states} @@ -1010,7 +1049,9 @@ class PresenceHandler(BasePresenceHandler): if user_id not in to_notify } if to_federation_ping: - federation_presence_out_counter.inc(len(to_federation_ping)) + federation_presence_out_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(len(to_federation_ping)) hosts_to_states = await get_interested_remotes( self.store, @@ -1060,7 +1101,9 @@ class PresenceHandler(BasePresenceHandler): for user_id in users_to_check ] - timers_fired_counter.inc(len(states)) + timers_fired_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc( + len(states) + ) # Set of user ID & device IDs which are currently syncing. syncing_user_devices = { @@ -1094,7 +1137,7 @@ class PresenceHandler(BasePresenceHandler): user_id = user.to_string() - bump_active_time_counter.inc() + bump_active_time_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc() now = self.clock.time_msec() @@ -1346,7 +1389,9 @@ class PresenceHandler(BasePresenceHandler): updates.append(prev_state.copy_and_replace(**new_fields)) if updates: - federation_presence_counter.inc(len(updates)) + federation_presence_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(len(updates)) await self._update_states(updates) async def set_state( @@ -1405,7 +1450,7 @@ class PresenceHandler(BasePresenceHandler): # Based on the state of each user's device calculate the new presence state. presence = _combine_device_states(devices.values()) - new_fields = {"state": presence} + new_fields: JsonDict = {"state": presence} if presence == PresenceState.ONLINE or presence == PresenceState.BUSY: new_fields["last_active_ts"] = now @@ -1492,12 +1537,16 @@ class PresenceHandler(BasePresenceHandler): finally: self._event_processing = False - run_as_background_process("presence.notify_new_event", _process_presence) + run_as_background_process( + "presence.notify_new_event", self.server_name, _process_presence + ) async def _unsafe_process(self) -> None: # Loop round handling deltas until we're up to date while True: - with Measure(self.clock, "presence_delta"): + with Measure( + self.clock, name="presence_delta", server_name=self.server_name + ): room_max_stream_ordering = self.store.get_room_max_stream_ordering() if self._event_pos == room_max_stream_ordering: return @@ -1527,9 +1576,9 @@ class PresenceHandler(BasePresenceHandler): self._event_pos = max_pos # Expose current event processing position to prometheus - synapse.metrics.event_processing_positions.labels("presence").set( - max_pos - ) + synapse.metrics.event_processing_positions.labels( + name="presence", **{SERVER_NAME_LABEL: self.server_name} + ).set(max_pos) async def _handle_state_delta(self, room_id: str, deltas: List[StateDelta]) -> None: """Process current state deltas for the room to find new joins that need @@ -1655,7 +1704,10 @@ class PresenceHandler(BasePresenceHandler): def should_notify( - old_state: UserPresenceState, new_state: UserPresenceState, is_mine: bool + old_state: UserPresenceState, + new_state: UserPresenceState, + is_mine: bool, + our_server_name: str, ) -> bool: """Decides if a presence state change should be sent to interested parties.""" user_location = "remote" @@ -1666,19 +1718,38 @@ def should_notify( return False if old_state.status_msg != new_state.status_msg: - notify_reason_counter.labels(user_location, "status_msg_change").inc() + notify_reason_counter.labels( + locality=user_location, + reason="status_msg_change", + **{SERVER_NAME_LABEL: our_server_name}, + ).inc() return True if old_state.state != new_state.state: - notify_reason_counter.labels(user_location, "state_change").inc() + notify_reason_counter.labels( + locality=user_location, + reason="state_change", + **{SERVER_NAME_LABEL: our_server_name}, + ).inc() state_transition_counter.labels( - user_location, old_state.state, new_state.state + **{ + "locality": user_location, + # `from` is a reserved word in Python so we have to label it this way if + # we want to use keyword args. + "from": old_state.state, + "to": new_state.state, + SERVER_NAME_LABEL: our_server_name, + }, ).inc() return True if old_state.state == PresenceState.ONLINE: if new_state.currently_active != old_state.currently_active: - notify_reason_counter.labels(user_location, "current_active_change").inc() + notify_reason_counter.labels( + locality=user_location, + reason="current_active_change", + **{SERVER_NAME_LABEL: our_server_name}, + ).inc() return True if ( @@ -1688,14 +1759,18 @@ def should_notify( # Only notify about last active bumps if we're not currently active if not new_state.currently_active: notify_reason_counter.labels( - user_location, "last_active_change_online" + locality=user_location, + reason="last_active_change_online", + **{SERVER_NAME_LABEL: our_server_name}, ).inc() return True elif new_state.last_active_ts - old_state.last_active_ts > LAST_ACTIVE_GRANULARITY: # Always notify for a transition where last active gets bumped. notify_reason_counter.labels( - user_location, "last_active_change_not_online" + locality=user_location, + reason="last_active_change_not_online", + **{SERVER_NAME_LABEL: our_server_name}, ).inc() return True @@ -1759,8 +1834,10 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): # Same with get_presence_router: # # AuthHandler -> Notifier -> PresenceEventSource -> ModuleApi -> AuthHandler + self.server_name = hs.hostname self.get_presence_handler = hs.get_presence_handler self.get_presence_router = hs.get_presence_router + self.server_name = hs.hostname self.clock = hs.get_clock() self.store = hs.get_datastores().main @@ -1792,7 +1869,9 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): user_id = user.to_string() stream_change_cache = self.store.presence_stream_cache - with Measure(self.clock, "presence.get_new_events"): + with Measure( + self.clock, name="presence.get_new_events", server_name=self.server_name + ): if from_key is not None: from_key = int(from_key) @@ -1870,7 +1949,10 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): # If we have the full list of changes for presence we can # simply check which ones share a room with the user. - get_updates_counter.labels("stream").inc() + get_updates_counter.labels( + type="stream", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() sharing_users = await self.store.do_users_share_a_room( user_id, updated_users @@ -1883,7 +1965,10 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): else: # Too many possible updates. Find all users we can see and check # if any of them have changed. - get_updates_counter.labels("full").inc() + get_updates_counter.labels( + type="full", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() users_interested_in = ( await self.store.get_users_who_share_room_with_user(user_id) @@ -2133,6 +2218,7 @@ def handle_update( prev_state: UserPresenceState, new_state: UserPresenceState, is_mine: bool, + our_server_name: str, wheel_timer: WheelTimer, now: int, persist: bool, @@ -2145,6 +2231,7 @@ def handle_update( prev_state new_state is_mine: Whether the user is ours + our_server_name: The homeserver name of the our server (`hs.hostname`) wheel_timer now: Time now in ms persist: True if this state should persist until another update occurs. @@ -2213,7 +2300,7 @@ def handle_update( ) # Check whether the change was something worth notifying about - if should_notify(prev_state, new_state, is_mine): + if should_notify(prev_state, new_state, is_mine, our_server_name): new_state = new_state.copy_and_replace(last_federation_update_ts=now) persist_and_notify = True diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index ac4544ca4c..dbff28e7fb 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -22,6 +22,7 @@ import logging import random from typing import TYPE_CHECKING, List, Optional, Union +from synapse.api.constants import ProfileFields from synapse.api.errors import ( AuthError, Codes, @@ -31,7 +32,7 @@ from synapse.api.errors import ( SynapseError, ) from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia -from synapse.types import JsonDict, Requester, UserID, create_requester +from synapse.types import JsonDict, JsonValue, Requester, UserID, create_requester from synapse.util.caches.descriptors import cached from synapse.util.stringutils import parse_and_validate_mxc_uri @@ -42,6 +43,8 @@ logger = logging.getLogger(__name__) MAX_DISPLAYNAME_LEN = 256 MAX_AVATAR_URL_LEN = 1000 +# Field name length is specced at 255 bytes. +MAX_CUSTOM_FIELD_LEN = 255 class ProfileHandler: @@ -52,6 +55,7 @@ class ProfileHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self.clock = hs.get_clock() self.hs = hs @@ -83,19 +87,31 @@ class ProfileHandler: Returns: A JSON dictionary. For local queries this will include the displayname and avatar_url - fields. For remote queries it may contain arbitrary information. + fields, if set. For remote queries it may contain arbitrary information. """ target_user = UserID.from_string(user_id) if self.hs.is_mine(target_user): profileinfo = await self.store.get_profileinfo(target_user) - if profileinfo.display_name is None and profileinfo.avatar_url is None: + extra_fields = await self.store.get_profile_fields(target_user) + + if ( + profileinfo.display_name is None + and profileinfo.avatar_url is None + and not extra_fields + ): raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND) - return { - "displayname": profileinfo.display_name, - "avatar_url": profileinfo.avatar_url, - } + # Do not include display name or avatar if unset. + ret = {} + if profileinfo.display_name is not None: + ret[ProfileFields.DISPLAYNAME] = profileinfo.display_name + if profileinfo.avatar_url is not None: + ret[ProfileFields.AVATAR_URL] = profileinfo.avatar_url + if extra_fields: + ret.update(extra_fields) + + return ret else: try: result = await self.federation.make_query( @@ -108,7 +124,7 @@ class ProfileHandler: except RequestSendFailed as e: raise SynapseError(502, "Failed to fetch profile") from e except HttpResponseException as e: - if e.code < 500 and e.code != 404: + if e.code < 500 and e.code not in (403, 404): # Other codes are not allowed in c2s API logger.info( "Server replied with wrong response: %s %s", e.code, e.msg @@ -399,6 +415,110 @@ class ProfileHandler: return True + async def get_profile_field( + self, target_user: UserID, field_name: str + ) -> JsonValue: + """ + Fetch a user's profile from the database for local users and over federation + for remote users. + + Args: + target_user: The user ID to fetch the profile for. + field_name: The field to fetch the profile for. + + Returns: + The value for the profile field or None if the field does not exist. + """ + if self.hs.is_mine(target_user): + try: + field_value = await self.store.get_profile_field( + target_user, field_name + ) + except StoreError as e: + if e.code == 404: + raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND) + raise + + return field_value + else: + try: + result = await self.federation.make_query( + destination=target_user.domain, + query_type="profile", + args={"user_id": target_user.to_string(), "field": field_name}, + ignore_backoff=True, + ) + except RequestSendFailed as e: + raise SynapseError(502, "Failed to fetch profile") from e + except HttpResponseException as e: + raise e.to_synapse_error() + + return result.get(field_name) + + async def set_profile_field( + self, + target_user: UserID, + requester: Requester, + field_name: str, + new_value: JsonValue, + by_admin: bool = False, + deactivation: bool = False, + ) -> None: + """Set a new profile field for a user. + + Args: + target_user: the user whose profile is to be changed. + requester: The user attempting to make this change. + field_name: The name of the profile field to update. + new_value: The new field value for this user. + by_admin: Whether this change was made by an administrator. + deactivation: Whether this change was made while deactivating the user. + """ + if not self.hs.is_mine(target_user): + raise SynapseError(400, "User is not hosted on this homeserver") + + if not by_admin and target_user != requester.user: + raise AuthError(403, "Cannot set another user's profile") + + await self.store.set_profile_field(target_user, field_name, new_value) + + # Custom fields do not propagate into the user directory *or* rooms. + profile = await self.store.get_profileinfo(target_user) + await self._third_party_rules.on_profile_update( + target_user.to_string(), profile, by_admin, deactivation + ) + + async def delete_profile_field( + self, + target_user: UserID, + requester: Requester, + field_name: str, + by_admin: bool = False, + deactivation: bool = False, + ) -> None: + """Delete a field from a user's profile. + + Args: + target_user: the user whose profile is to be changed. + requester: The user attempting to make this change. + field_name: The name of the profile field to remove. + by_admin: Whether this change was made by an administrator. + deactivation: Whether this change was made while deactivating the user. + """ + if not self.hs.is_mine(target_user): + raise SynapseError(400, "User is not hosted on this homeserver") + + if not by_admin and target_user != requester.user: + raise AuthError(400, "Cannot set another user's profile") + + await self.store.delete_profile_field(target_user, field_name) + + # Custom fields do not propagate into the user directory *or* rooms. + profile = await self.store.get_profileinfo(target_user) + await self._third_party_rules.on_profile_update( + target_user.to_string(), profile, by_admin, deactivation + ) + async def on_profile_query(self, args: JsonDict) -> JsonDict: """Handles federation profile query requests.""" @@ -415,13 +535,30 @@ class ProfileHandler: just_field = args.get("field", None) - response = {} + response: JsonDict = {} try: - if just_field is None or just_field == "displayname": - response["displayname"] = await self.store.get_profile_displayname(user) + if just_field is None or just_field == ProfileFields.DISPLAYNAME: + displayname = await self.store.get_profile_displayname(user) + # do not set the displayname field if it is None, + # since then we send a null in the JSON response + if displayname is not None: + response["displayname"] = displayname + if just_field is None or just_field == ProfileFields.AVATAR_URL: + avatar_url = await self.store.get_profile_avatar_url(user) + # do not set the avatar_url field if it is None, + # since then we send a null in the JSON response + if avatar_url is not None: + response["avatar_url"] = avatar_url - if just_field is None or just_field == "avatar_url": - response["avatar_url"] = await self.store.get_profile_avatar_url(user) + if just_field is None: + response.update(await self.store.get_profile_fields(user)) + elif just_field not in ( + ProfileFields.DISPLAYNAME, + ProfileFields.AVATAR_URL, + ): + response[just_field] = await self.store.get_profile_field( + user, just_field + ) except StoreError as e: if e.code == 404: raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c200e29569..5761a7f70b 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -23,10 +23,9 @@ """Contains functions for registering clients.""" import logging -from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, TypedDict from prometheus_client import Counter -from typing_extensions import TypedDict from synapse import types from synapse.api.constants import ( @@ -45,12 +44,11 @@ from synapse.api.errors import ( ) from synapse.appservice import ApplicationService from synapse.config.server import is_threepid_reserved -from synapse.handlers.device import DeviceHandler from synapse.http.servlet import assert_params_in_dict +from synapse.metrics import SERVER_NAME_LABEL from synapse.replication.http.login import RegisterDeviceReplicationServlet from synapse.replication.http.register import ( ReplicationPostRegisterActionsServlet, - ReplicationRegisterServlet, ) from synapse.spam_checker_api import RegistrationBehaviour from synapse.types import GUEST_USER_ID_PATTERN, RoomAlias, UserID, create_requester @@ -65,29 +63,38 @@ logger = logging.getLogger(__name__) registration_counter = Counter( "synapse_user_registrations_total", "Number of new users registered (since restart)", - ["guest", "shadow_banned", "auth_provider"], + labelnames=["guest", "shadow_banned", "auth_provider", SERVER_NAME_LABEL], ) login_counter = Counter( "synapse_user_logins_total", "Number of user logins (since restart)", - ["guest", "auth_provider"], + labelnames=["guest", "auth_provider", SERVER_NAME_LABEL], ) -def init_counters_for_auth_provider(auth_provider_id: str) -> None: +def init_counters_for_auth_provider(auth_provider_id: str, server_name: str) -> None: """Ensure the prometheus counters for the given auth provider are initialised This fixes a problem where the counters are not reported for a given auth provider until the user first logs in/registers. + + Args: + auth_provider_id: The ID of the auth provider to initialise counters for. + server_name: Our server name (used to label metrics) (this should be `hs.hostname`). """ for is_guest in (True, False): - login_counter.labels(guest=is_guest, auth_provider=auth_provider_id) + login_counter.labels( + guest=is_guest, + auth_provider=auth_provider_id, + **{SERVER_NAME_LABEL: server_name}, + ) for shadow_banned in (True, False): registration_counter.labels( guest=is_guest, shadow_banned=shadow_banned, auth_provider=auth_provider_id, + **{SERVER_NAME_LABEL: server_name}, ) @@ -100,6 +107,7 @@ class LoginDict(TypedDict): class RegistrationHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.clock = hs.get_clock() @@ -115,12 +123,11 @@ class RegistrationHandler: self._account_validity_handler = hs.get_account_validity_handler() self._user_consent_version = self.hs.config.consent.user_consent_version self._server_notices_mxid = hs.config.servernotices.server_notices_mxid - self._server_name = hs.hostname + self._user_types_config = hs.config.user_types self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker if hs.config.worker.worker_app: - self._register_client = ReplicationRegisterServlet.make_client(hs) self._register_device_client = RegisterDeviceReplicationServlet.make_client( hs ) @@ -141,7 +148,9 @@ class RegistrationHandler: ) self.refresh_token_lifetime = hs.config.registration.refresh_token_lifetime - init_counters_for_auth_provider("") + init_counters_for_auth_provider( + auth_provider_id="", server_name=self.server_name + ) async def check_username( self, @@ -160,7 +169,10 @@ class RegistrationHandler: if not localpart: raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME) - if localpart[0] == "_": + if ( + localpart[0] == "_" + and not self.hs.config.registration.allow_underscore_prefixed_localpart + ): raise SynapseError( 400, "User ID may not begin with _", Codes.INVALID_USERNAME ) @@ -304,6 +316,9 @@ class RegistrationHandler: elif default_display_name is None: default_display_name = localpart + if user_type is None: + user_type = self._user_types_config.default_user_type + await self.register_with_store( user_id=user_id, password_hash=password_hash, @@ -359,6 +374,7 @@ class RegistrationHandler: guest=make_guest, shadow_banned=shadow_banned, auth_provider=(auth_provider_id or ""), + **{SERVER_NAME_LABEL: self.server_name}, ).inc() # If the user does not need to consent at registration, auto-join any @@ -419,7 +435,7 @@ class RegistrationHandler: if self.hs.config.registration.auto_join_user_id: fake_requester = create_requester( self.hs.config.registration.auto_join_user_id, - authenticated_entity=self._server_name, + authenticated_entity=self.server_name, ) # If the room requires an invite, add the user to the list of invites. @@ -432,7 +448,7 @@ class RegistrationHandler: requires_join = True else: fake_requester = create_requester( - user_id, authenticated_entity=self._server_name + user_id, authenticated_entity=self.server_name ) # Choose whether to federate the new room. @@ -464,7 +480,7 @@ class RegistrationHandler: await room_member_handler.update_membership( requester=create_requester( - user_id, authenticated_entity=self._server_name + user_id, authenticated_entity=self.server_name ), target=UserID.from_string(user_id), room_id=room_id, @@ -490,7 +506,7 @@ class RegistrationHandler: if requires_join: await room_member_handler.update_membership( requester=create_requester( - user_id, authenticated_entity=self._server_name + user_id, authenticated_entity=self.server_name ), target=UserID.from_string(user_id), room_id=room_id, @@ -500,7 +516,7 @@ class RegistrationHandler: ratelimit=False, ) except Exception as e: - logger.error("Failed to join new user to %r: %r", r, e) + logger.exception("Failed to join new user to %r: %r", r, e) async def _join_rooms(self, user_id: str) -> None: """ @@ -536,7 +552,7 @@ class RegistrationHandler: # we don't have a local user in the room to craft up an invite with. requires_invite = await self.store.is_host_joined( room_id, - self._server_name, + self.server_name, ) if requires_invite: @@ -553,7 +569,7 @@ class RegistrationHandler: if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) requires_invite = ( - join_rule and join_rule != JoinRules.PUBLIC + join_rule is not None and join_rule != JoinRules.PUBLIC ) # Send the invite, if necessary. @@ -564,7 +580,7 @@ class RegistrationHandler: await room_member_handler.update_membership( requester=create_requester( self.hs.config.registration.auto_join_user_id, - authenticated_entity=self._server_name, + authenticated_entity=self.server_name, ), target=UserID.from_string(user_id), room_id=room_id, @@ -576,7 +592,7 @@ class RegistrationHandler: # Send the join. await room_member_handler.update_membership( requester=create_requester( - user_id, authenticated_entity=self._server_name + user_id, authenticated_entity=self.server_name ), target=UserID.from_string(user_id), room_id=room_id, @@ -590,7 +606,7 @@ class RegistrationHandler: # moving away from bare excepts is a good thing to do. logger.error("Failed to join new user to %r: %r", r, e) except Exception as e: - logger.error("Failed to join new user to %r: %r", r, e, exc_info=True) + logger.exception("Failed to join new user to %r: %r", r, e) async def _auto_join_rooms(self, user_id: str) -> None: """Automatically joins users to auto join rooms - creating the room in the first place @@ -630,7 +646,9 @@ class RegistrationHandler: """ await self._auto_join_rooms(user_id) - async def appservice_register(self, user_localpart: str, as_token: str) -> str: + async def appservice_register( + self, user_localpart: str, as_token: str + ) -> Tuple[str, ApplicationService]: user = UserID(user_localpart, self.hs.hostname) user_id = user.to_string() service = self.store.get_app_service_by_token(as_token) @@ -653,7 +671,7 @@ class RegistrationHandler: appservice_id=service_id, create_profile_with_displayname=user.localpart, ) - return user_id + return (user_id, service) def check_user_id_not_appservice_exclusive( self, user_id: str, allowed_appservice: Optional[ApplicationService] = None @@ -730,37 +748,20 @@ class RegistrationHandler: shadow_banned: Whether to shadow-ban the user approved: Whether to mark the user as approved by an administrator """ - if self.hs.config.worker.worker_app: - await self._register_client( - user_id=user_id, - password_hash=password_hash, - was_guest=was_guest, - make_guest=make_guest, - appservice_id=appservice_id, - create_profile_with_displayname=create_profile_with_displayname, - admin=admin, - user_type=user_type, - address=address, - shadow_banned=shadow_banned, - approved=approved, - ) - else: - await self.store.register_user( - user_id=user_id, - password_hash=password_hash, - was_guest=was_guest, - make_guest=make_guest, - appservice_id=appservice_id, - create_profile_with_displayname=create_profile_with_displayname, - admin=admin, - user_type=user_type, - shadow_banned=shadow_banned, - approved=approved, - ) + await self.store.register_user( + user_id=user_id, + password_hash=password_hash, + was_guest=was_guest, + make_guest=make_guest, + appservice_id=appservice_id, + create_profile_with_displayname=create_profile_with_displayname, + admin=admin, + user_type=user_type, + shadow_banned=shadow_banned, + approved=approved, + ) - # Only call the account validity module(s) on the main process, to avoid - # repeating e.g. database writes on all of the workers. - await self._account_validity_handler.on_user_registration(user_id) + await self._account_validity_handler.on_user_registration(user_id) async def register_device( self, @@ -802,6 +803,7 @@ class RegistrationHandler: login_counter.labels( guest=is_guest, auth_provider=(auth_provider_id or ""), + **{SERVER_NAME_LABEL: self.server_name}, ).inc() return ( @@ -851,9 +853,6 @@ class RegistrationHandler: refresh_token = None refresh_token_id = None - # This can only run on the main process. - assert isinstance(self.device_handler, DeviceHandler) - registered_device_id = await self.device_handler.check_device_registered( user_id, device_id, diff --git a/synapse/handlers/reports.py b/synapse/handlers/reports.py new file mode 100644 index 0000000000..a7b8a4bed7 --- /dev/null +++ b/synapse/handlers/reports.py @@ -0,0 +1,98 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2015, 2016 OpenMarket Ltd +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING + +from synapse.api.errors import Codes, SynapseError +from synapse.api.ratelimiting import Ratelimiter +from synapse.types import ( + Requester, +) + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ReportsHandler: + def __init__(self, hs: "HomeServer"): + self._hs = hs + self._store = hs.get_datastores().main + self._clock = hs.get_clock() + + # Ratelimiter for management of existing delayed events, + # keyed by the requesting user ID. + self._reports_ratelimiter = Ratelimiter( + store=self._store, + clock=self._clock, + cfg=hs.config.ratelimiting.rc_reports, + ) + + async def report_user( + self, requester: Requester, target_user_id: str, reason: str + ) -> None: + """Files a report against a user from a user. + + Rate and size limits are applied to the report. If the user being reported + does not belong to this server, the report is ignored. This check is done + after the limits to reduce DoS potential. + + If the user being reported belongs to this server, but doesn't exist, we + similarly ignore the report. The spec allows us to return an error if we + want to, but we choose to hide that user's existence instead. + + If the report is otherwise valid (for a user which exists on our server), + we append it to the database for later processing. + + Args: + requester - The user filing the report. + target_user_id - The user being reported. + reason - The user-supplied reason the user is being reported. + + Raises: + SynapseError for BAD_REQUEST/BAD_JSON if the reason is too long. + """ + + await self._check_limits(requester) + + if len(reason) > 1000: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Reason must be less than 1000 characters", + Codes.BAD_JSON, + ) + + if not self._hs.is_mine_id(target_user_id): + return # hide that they're not ours/that we can't do anything about them + + user = await self._store.get_user_by_id(target_user_id) + if user is None: + return # hide that they don't exist + + await self._store.add_user_report( + target_user_id=target_user_id, + user_id=requester.user.to_string(), + reason=reason, + received_ts=self._clock.time_msec(), + ) + + async def _check_limits(self, requester: Requester) -> None: + await self._reports_ratelimiter.ratelimit( + requester, + requester.user.to_string(), + ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 386375d64b..47bd139ca7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -51,6 +51,7 @@ from synapse.api.constants import ( HistoryVisibility, JoinRules, Membership, + MTextFields, RoomCreationPreset, RoomEncryptionAlgorithms, RoomTypes, @@ -65,6 +66,7 @@ from synapse.api.errors import ( SynapseError, ) from synapse.api.filtering import Filter +from synapse.api.ratelimiting import Ratelimiter from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.event_auth import validate_event_for_room_version from synapse.events import EventBase @@ -80,6 +82,7 @@ from synapse.types import ( Requester, RoomAlias, RoomID, + RoomIdWithDomain, RoomStreamToken, StateMap, StrCollection, @@ -91,7 +94,9 @@ from synapse.types import ( from synapse.types.handlers import ShutdownRoomParams, ShutdownRoomResponse from synapse.types.state import StateFilter from synapse.util import stringutils +from synapse.util.async_helpers import concurrently_execute from synapse.util.caches.response_cache import ResponseCache +from synapse.util.iterutils import batch_iter from synapse.util.stringutils import parse_and_validate_server_name from synapse.visibility import filter_events_for_client @@ -118,6 +123,7 @@ class EventContext: class RoomCreationHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.auth = hs.get_auth() @@ -129,7 +135,12 @@ class RoomCreationHandler: self.room_member_handler = hs.get_room_member_handler() self._event_auth_handler = hs.get_event_auth_handler() self.config = hs.config - self.request_ratelimiter = hs.get_request_ratelimiter() + self.common_request_ratelimiter = hs.get_request_ratelimiter() + self.creation_ratelimiter = Ratelimiter( + store=self.store, + clock=self.clock, + cfg=self.config.ratelimiting.rc_room_creation, + ) # Room state based off defined presets self._presets_dict: Dict[str, Dict[str, Any]] = { @@ -174,7 +185,10 @@ class RoomCreationHandler: # succession, only process the first attempt and return its result to # subsequent requests self._upgrade_response_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "room_upgrade", timeout_ms=FIVE_MINUTES_IN_MS + clock=hs.get_clock(), + name="room_upgrade", + server_name=self.server_name, + timeout_ms=FIVE_MINUTES_IN_MS, ) self._server_notices_mxid = hs.config.servernotices.server_notices_mxid @@ -183,7 +197,13 @@ class RoomCreationHandler: ) async def upgrade_room( - self, requester: Requester, old_room_id: str, new_version: RoomVersion + self, + requester: Requester, + old_room_id: str, + new_version: RoomVersion, + additional_creators: Optional[List[str]], + auto_member: bool = False, + ratelimit: bool = True, ) -> str: """Replace a room with a new room with a different version @@ -191,6 +211,9 @@ class RoomCreationHandler: requester: the user requesting the upgrade old_room_id: the id of the room to be replaced new_version: the new room version to use + additional_creators: additional room creators, for MSC4289. + auto_member: Whether to automatically join local users to the new + room and send out invites to remote users. Returns: the new room id @@ -198,7 +221,12 @@ class RoomCreationHandler: Raises: ShadowBanError if the requester is shadow-banned. """ - await self.request_ratelimiter.ratelimit(requester) + if ratelimit: + await self.creation_ratelimiter.ratelimit(requester, update=False) + + # then apply the ratelimits + await self.common_request_ratelimiter.ratelimit(requester) + await self.creation_ratelimiter.ratelimit(requester) user_id = requester.user.to_string() @@ -219,8 +247,29 @@ class RoomCreationHandler: old_room = await self.store.get_room(old_room_id) if old_room is None: raise NotFoundError("Unknown room id %s" % (old_room_id,)) + old_room_is_public, _ = old_room - new_room_id = self._generate_room_id() + creation_event_with_context = None + if new_version.msc4291_room_ids_as_hashes: + old_room_create_event = await self.store.get_create_event_for_room( + old_room_id + ) + creation_content = self._calculate_upgraded_room_creation_content( + old_room_create_event, + tombstone_event_id=None, + new_room_version=new_version, + additional_creators=additional_creators, + ) + creation_event_with_context = await self._generate_create_event_for_room_id( + requester, + creation_content, + old_room_is_public, + new_version, + ) + (create_event, _) = creation_event_with_context + new_room_id = create_event.room_id + else: + new_room_id = self._generate_room_id() # Try several times, it could fail with PartialStateConflictError # in _upgrade_room, cf comment in except block. @@ -269,6 +318,9 @@ class RoomCreationHandler: new_version, tombstone_event, tombstone_context, + additional_creators, + creation_event_with_context, + auto_member=auto_member, ) return ret @@ -292,6 +344,11 @@ class RoomCreationHandler: new_version: RoomVersion, tombstone_event: EventBase, tombstone_context: synapse.events.snapshot.EventContext, + additional_creators: Optional[List[str]], + creation_event_with_context: Optional[ + Tuple[EventBase, synapse.events.snapshot.EventContext] + ] = None, + auto_member: bool = False, ) -> str: """ Args: @@ -303,6 +360,10 @@ class RoomCreationHandler: new_version: the version to upgrade the room to tombstone_event: the tombstone event to send to the old room tombstone_context: the context for the tombstone event + additional_creators: additional room creators, for MSC4289. + creation_event_with_context: The new room's create event, for room IDs as create event IDs. + auto_member: Whether to automatically join local users to the new + room and send out invites to remote users. Raises: ShadowBanError if the requester is shadow-banned. @@ -312,14 +373,16 @@ class RoomCreationHandler: logger.info("Creating new room %s to replace %s", new_room_id, old_room_id) - # create the new room. may raise a `StoreError` in the exceedingly unlikely - # event of a room ID collision. - await self.store.store_room( - room_id=new_room_id, - room_creator_user_id=user_id, - is_public=old_room[0], - room_version=new_version, - ) + # We've already stored the room if we have the create event + if not creation_event_with_context: + # create the new room. may raise a `StoreError` in the exceedingly unlikely + # event of a room ID collision. + await self.store.store_room( + room_id=new_room_id, + room_creator_user_id=user_id, + is_public=old_room[0], + room_version=new_version, + ) await self.clone_existing_room( requester, @@ -327,6 +390,9 @@ class RoomCreationHandler: new_room_id=new_room_id, new_room_version=new_version, tombstone_event_id=tombstone_event.event_id, + additional_creators=additional_creators, + creation_event_with_context=creation_event_with_context, + auto_member=auto_member, ) # now send the tombstone @@ -360,6 +426,7 @@ class RoomCreationHandler: old_room_id, new_room_id, old_room_state, + additional_creators, ) return new_room_id @@ -370,6 +437,7 @@ class RoomCreationHandler: old_room_id: str, new_room_id: str, old_room_state: StateMap[str], + additional_creators: Optional[List[str]], ) -> None: """Send updated power levels in both rooms after an upgrade @@ -378,7 +446,7 @@ class RoomCreationHandler: old_room_id: the id of the room to be replaced new_room_id: the id of the replacement room old_room_state: the state map for the old room - + additional_creators: Additional creators in the new room. Raises: ShadowBanError if the requester is shadow-banned. """ @@ -434,6 +502,14 @@ class RoomCreationHandler: except AuthError as e: logger.warning("Unable to update PLs in old room: %s", e) + new_room_version = await self.store.get_room_version(new_room_id) + if new_room_version.msc4289_creator_power_enabled: + self._remove_creators_from_pl_users_map( + old_room_pl_state.content.get("users", {}), + requester.user.to_string(), + additional_creators, + ) + await self.event_creation_handler.create_and_send_nonmember_event( requester, { @@ -448,6 +524,36 @@ class RoomCreationHandler: ratelimit=False, ) + def _calculate_upgraded_room_creation_content( + self, + old_room_create_event: EventBase, + tombstone_event_id: Optional[str], + new_room_version: RoomVersion, + additional_creators: Optional[List[str]], + ) -> JsonDict: + creation_content: JsonDict = { + "room_version": new_room_version.identifier, + "predecessor": { + "room_id": old_room_create_event.room_id, + }, + } + if tombstone_event_id is not None: + creation_content["predecessor"]["event_id"] = tombstone_event_id + if ( + additional_creators is not None + and new_room_version.msc4289_creator_power_enabled + ): + creation_content["additional_creators"] = additional_creators + # Check if old room was non-federatable + if not old_room_create_event.content.get(EventContentFields.FEDERATE, True): + # If so, mark the new room as non-federatable as well + creation_content[EventContentFields.FEDERATE] = False + # Copy the room type as per MSC3818. + room_type = old_room_create_event.content.get(EventContentFields.ROOM_TYPE) + if room_type is not None: + creation_content[EventContentFields.ROOM_TYPE] = room_type + return creation_content + async def clone_existing_room( self, requester: Requester, @@ -455,6 +561,11 @@ class RoomCreationHandler: new_room_id: str, new_room_version: RoomVersion, tombstone_event_id: str, + additional_creators: Optional[List[str]], + creation_event_with_context: Optional[ + Tuple[EventBase, synapse.events.snapshot.EventContext] + ] = None, + auto_member: bool = False, ) -> None: """Populate a new room based on an old room @@ -465,35 +576,27 @@ class RoomCreationHandler: created with _generate_room_id()) new_room_version: the new room version to use tombstone_event_id: the ID of the tombstone event in the old room. + additional_creators: additional room creators, for MSC4289. + creation_event_with_context: The create event of the new room, if the new room supports + room ID as create event ID hash. + auto_member: Whether to automatically join local users to the new + room and send out invites to remote users. """ user_id = requester.user.to_string() - spam_check = await self._spam_checker_module_callbacks.user_may_create_room( - user_id - ) - if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: - raise SynapseError( - 403, - "You are not permitted to create rooms", - errcode=spam_check[0], - additional_fields=spam_check[1], - ) - - creation_content: JsonDict = { - "room_version": new_room_version.identifier, - "predecessor": {"room_id": old_room_id, "event_id": tombstone_event_id}, - } - - # Check if old room was non-federatable - # Get old room's create event old_room_create_event = await self.store.get_create_event_for_room(old_room_id) - # Check if the create event specified a non-federatable room - if not old_room_create_event.content.get(EventContentFields.FEDERATE, True): - # If so, mark the new room as non-federatable as well - creation_content[EventContentFields.FEDERATE] = False - + if creation_event_with_context: + create_event, _ = creation_event_with_context + creation_content = create_event.content + else: + creation_content = self._calculate_upgraded_room_creation_content( + old_room_create_event, + tombstone_event_id, + new_room_version, + additional_creators=additional_creators, + ) initial_state = {} # Replicate relevant room events @@ -509,11 +612,8 @@ class RoomCreationHandler: (EventTypes.PowerLevels, ""), ] - # Copy the room type as per MSC3818. room_type = old_room_create_event.content.get(EventContentFields.ROOM_TYPE) if room_type is not None: - creation_content[EventContentFields.ROOM_TYPE] = room_type - # If the old room was a space, copy over the rooms in the space. if room_type == RoomTypes.SPACE: types_to_copy.append((EventTypes.SpaceChild, None)) @@ -585,7 +685,33 @@ class RoomCreationHandler: if current_power_level_int < needed_power_level: user_power_levels[user_id] = needed_power_level - await self._send_events_for_new_room( + if new_room_version.msc4289_creator_power_enabled: + # the creator(s) cannot be in the users map + self._remove_creators_from_pl_users_map( + user_power_levels, + user_id, + additional_creators, + ) + + # We construct what the body of a call to /createRoom would look like for passing + # to the spam checker. We don't include a preset here, as we expect the + # initial state to contain everything we need. + spam_check = await self._spam_checker_module_callbacks.user_may_create_room( + user_id, + { + "creation_content": creation_content, + "initial_state": list(initial_state.items()), + }, + ) + if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: + raise SynapseError( + 403, + "You are not permitted to create rooms", + errcode=spam_check[0], + additional_fields=spam_check[1], + ) + + _, last_event_id, _ = await self._send_events_for_new_room( requester, new_room_id, new_room_version, @@ -595,36 +721,228 @@ class RoomCreationHandler: invite_list=[], initial_state=initial_state, creation_content=creation_content, + creation_event_with_context=creation_event_with_context, ) # Transfer membership events - old_room_member_state_ids = ( - await self._storage_controllers.state.get_current_state_ids( - old_room_id, StateFilter.from_types([(EventTypes.Member, None)]) - ) - ) + ban_event_ids = await self.store.get_ban_event_ids_in_room(old_room_id) + if ban_event_ids: + ban_events = await self.store.get_events_as_list(ban_event_ids) - # map from event_id to BaseEvent - old_room_member_state_events = await self.store.get_events( - old_room_member_state_ids.values() - ) - for old_event in old_room_member_state_events.values(): - # Only transfer ban events - if ( - "membership" in old_event.content - and old_event.content["membership"] == "ban" - ): - await self.room_member_handler.update_membership( - requester, - UserID.from_string(old_event.state_key), - new_room_id, - "ban", - ratelimit=False, - content=old_event.content, + # Add any banned users to the new room. + # + # Note generally we should send membership events via + # `update_membership`, however in this case its fine to bypass as + # these bans don't need any special treatment, i.e. the sender is in + # the room and they don't need any extra signatures, etc. + for batched_ban_events in batch_iter(ban_events, 1000): + await self.event_creation_handler.create_and_send_new_client_events( + requester=requester, + room_id=new_room_id, + prev_event_id=last_event_id, + event_dicts=[ + { + "type": EventTypes.Member, + "state_key": ban_event.state_key, + "room_id": new_room_id, + "sender": requester.user.to_string(), + "content": ban_event.content, + } + for ban_event in batched_ban_events + ], + ratelimit=False, # We ratelimit the entire upgrade, not individual events. ) - # XXX invites/joins - # XXX 3pid invites + if auto_member: + logger.info("Joining local users to %s", new_room_id) + + # 1. Copy over all joins for local + joined_profiles = await self.store.get_users_in_room_with_profiles( + old_room_id + ) + + local_user_ids = [ + user_id for user_id in joined_profiles if self.hs.is_mine_id(user_id) + ] + + logger.info("Local user IDs %s", local_user_ids) + + for batched_local_user_ids in batch_iter(local_user_ids, 1000): + invites_to_send = [] + + # For each local user we create an invite event (from the + # upgrading user) plus a join event. + for local_user_id in batched_local_user_ids: + if local_user_id == user_id: + # Ignore the upgrading user, as they are already in the + # new room. + continue + + invites_to_send.append( + { + "type": EventTypes.Member, + "state_key": local_user_id, + "room_id": new_room_id, + "sender": requester.user.to_string(), + "content": { + "membership": Membership.INVITE, + }, + } + ) + + # If the user has profile information in the previous join, + # add it to the content. + # + # We could instead copy over the contents from the old join + # event, however a) that would require us to fetch all the + # old join events (which is slow), and b) generally the join + # events have no extra information in them. (We also believe + # that most clients don't copy this information over either, + # but we could be wrong.) + content_profile = {} + user_profile = joined_profiles[local_user_id] + if user_profile.display_name: + content_profile["displayname"] = user_profile.display_name + if user_profile.avatar_url: + content_profile["avatar_url"] = user_profile.avatar_url + + invites_to_send.append( + { + "type": EventTypes.Member, + "state_key": local_user_id, + "room_id": new_room_id, + "sender": local_user_id, + "content": { + "membership": Membership.JOIN, + **content_profile, + }, + } + ) + + await self.event_creation_handler.create_and_send_new_client_events( + requester=requester, + room_id=new_room_id, + prev_event_id=None, + event_dicts=invites_to_send, + ratelimit=False, # We ratelimit the entire upgrade, not individual events. + ) + + # Invite other users if the room is not public. If the room *is* + # public then users can simply directly join, and inviting them as + # well may lead to confusion. + + join_rule_content = initial_state.get((EventTypes.JoinRules, ""), None) + is_public = False + if join_rule_content: + is_public = join_rule_content["join_rule"] == JoinRules.PUBLIC + + if not is_public: + # Copy invites + # TODO: Copy over 3pid invites as well. + invited_users = await self.store.get_invited_users_in_room( + room_id=old_room_id + ) + + # For local users we can just batch send the invites. + local_invited_users = [ + user_id for user_id in invited_users if self.hs.is_mine_id(user_id) + ] + + logger.info( + "Joining local user IDs %s to new room %s", + local_invited_users, + new_room_id, + ) + + for batched_local_invited_users in batch_iter( + local_invited_users, 1000 + ): + invites_to_send = [] + leaves_to_send = [] + + # For each local user we create an invite event (from the + # upgrading user), and reject the invite event in the old + # room. + # + # This ensures that the user ends up with a single invite to + # the new room (rather than multiple invites which may be + # noisy and confusing). + for local_user_id in batched_local_invited_users: + leaves_to_send.append( + { + "type": EventTypes.Member, + "state_key": local_user_id, + "room_id": old_room_id, + "sender": local_user_id, + "content": { + "membership": Membership.LEAVE, + }, + } + ) + invites_to_send.append( + { + "type": EventTypes.Member, + "state_key": local_user_id, + "room_id": new_room_id, + "sender": requester.user.to_string(), + "content": { + "membership": Membership.INVITE, + }, + } + ) + + await self.event_creation_handler.create_and_send_new_client_events( + requester=requester, + room_id=old_room_id, + prev_event_id=None, + event_dicts=leaves_to_send, + ratelimit=False, # We ratelimit the entire upgrade, not individual events. + ) + await self.event_creation_handler.create_and_send_new_client_events( + requester=requester, + room_id=new_room_id, + prev_event_id=None, + event_dicts=invites_to_send, + ratelimit=False, + ) + + # For remote users we send invites one by one, as we need to + # send each one to the remote server. + # + # We also invite joined remote users who were in the old room. + remote_user_ids = [ + user_id + for user_id in itertools.chain(invited_users, joined_profiles) + if not self.hs.is_mine_id(user_id) + ] + + logger.debug("Inviting remote user IDs %s", remote_user_ids) + + async def remote_invite(remote_user: str) -> None: + try: + await self.room_member_handler.update_membership( + requester, + UserID.from_string(remote_user), + new_room_id, + Membership.INVITE, + ratelimit=False, # We ratelimit the entire upgrade, not individual events. + ) + except SynapseError as e: + # If we fail to invite a remote user, we log it but continue + # on with the upgrade. + logger.warning( + "Failed to invite remote user %s to new room %s: %s", + remote_user, + new_room_id, + e, + ) + + # We do this concurrently, as it can take a while to invite + await concurrently_execute( + remote_invite, + remote_user_ids, + 10, + ) async def _move_aliases_to_new_room( self, @@ -691,7 +1009,7 @@ class RoomCreationHandler: except SynapseError as e: # again I'm not really expecting this to fail, but if it does, I'd rather # we returned the new room to the client at this point. - logger.error("Unable to send updated alias events in old room: %s", e) + logger.exception("Unable to send updated alias events in old room: %s", e) try: await self.event_creation_handler.create_and_send_nonmember_event( @@ -708,7 +1026,7 @@ class RoomCreationHandler: except SynapseError as e: # again I'm not really expecting this to fail, but if it does, I'd rather # we returned the new room to the client at this point. - logger.error("Unable to send updated alias events in new room: %s", e) + logger.exception("Unable to send updated alias events in new room: %s", e) async def create_room( self, @@ -753,6 +1071,25 @@ class RoomCreationHandler: await self.auth_blocking.check_auth_blocking(requester=requester) + if ratelimit: + # Limit the rate of room creations, + # using both the limiter specific to room creations as well + # as the general request ratelimiter. + # + # Note that we don't rate limit the individual + # events in the room — room creation isn't atomic and + # historically it was very janky if half the events in the + # initial state don't make it because of rate limiting. + + # First check the room creation ratelimiter without updating it + # (this is so we don't consume a token if the other ratelimiter doesn't + # allow us to proceed) + await self.creation_ratelimiter.ratelimit(requester, update=False) + + # then apply the ratelimits + await self.common_request_ratelimiter.ratelimit(requester) + await self.creation_ratelimiter.ratelimit(requester) + if ( self._server_notices_mxid is not None and user_id == self._server_notices_mxid @@ -784,25 +1121,6 @@ class RoomCreationHandler: Codes.MISSING_PARAM, ) - if not is_requester_admin: - spam_check = await self._spam_checker_module_callbacks.user_may_create_room( - user_id - ) - if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: - raise SynapseError( - 403, - "You are not permitted to create rooms", - errcode=spam_check[0], - additional_fields=spam_check[1], - ) - - if ratelimit: - # Rate limit once in advance, but don't rate limit the individual - # events in the room — room creation isn't atomic and it's very - # janky if half the events in the initial state don't make it because - # of rate limiting. - await self.request_ratelimiter.ratelimit(requester) - room_version_id = config.get( "room_version", self.config.server.default_room_version.identifier ) @@ -878,6 +1196,7 @@ class RoomCreationHandler: power_level_content_override = config.get("power_level_content_override") if ( power_level_content_override + and not room_version.msc4289_creator_power_enabled # this validation doesn't apply in MSC4289 rooms and "users" in power_level_content_override and user_id not in power_level_content_override["users"] ): @@ -894,11 +1213,54 @@ class RoomCreationHandler: self._validate_room_config(config, visibility) - room_id = await self._generate_and_create_room_id( - creator_id=user_id, - is_public=is_public, - room_version=room_version, - ) + # Run the spam checker after other validation + if not is_requester_admin: + spam_check = await self._spam_checker_module_callbacks.user_may_create_room( + user_id, config + ) + if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: + raise SynapseError( + 403, + "You are not permitted to create rooms", + errcode=spam_check[0], + additional_fields=spam_check[1], + ) + + creation_content = config.get("creation_content", {}) + # override any attempt to set room versions via the creation_content + creation_content["room_version"] = room_version.identifier + + # trusted private chats have the invited users marked as additional creators + if ( + room_version.msc4289_creator_power_enabled + and config.get("preset", None) == RoomCreationPreset.TRUSTED_PRIVATE_CHAT + and len(config.get("invite", [])) > 0 + ): + # the other user(s) are additional creators + invitees = config.get("invite", []) + # we don't want to replace any additional_creators additionally specified, and we want + # to remove duplicates. + creation_content[EventContentFields.ADDITIONAL_CREATORS] = list( + set(creation_content.get(EventContentFields.ADDITIONAL_CREATORS, [])) + | set(invitees) + ) + + creation_event_with_context = None + if room_version.msc4291_room_ids_as_hashes: + creation_event_with_context = await self._generate_create_event_for_room_id( + requester, + creation_content, + is_public, + room_version, + ) + (create_event, _) = creation_event_with_context + room_id = create_event.room_id + else: + room_id = await self._generate_and_create_room_id( + creator_id=user_id, + is_public=is_public, + room_version=room_version, + ) # Check whether this visibility value is blocked by a third party module allowed_by_third_party_rules = await ( @@ -935,11 +1297,6 @@ class RoomCreationHandler: for val in raw_initial_state: initial_state[(val["type"], val.get("state_key", ""))] = val["content"] - creation_content = config.get("creation_content", {}) - - # override any attempt to set room versions via the creation_content - creation_content["room_version"] = room_version.identifier - ( last_stream_id, last_sent_event_id, @@ -956,6 +1313,7 @@ class RoomCreationHandler: power_level_content_override=power_level_content_override, creator_join_profile=creator_join_profile, ignore_forced_encryption=ignore_forced_encryption, + creation_event_with_context=creation_event_with_context, ) # we avoid dropping the lock between invites, as otherwise joins can @@ -1021,6 +1379,38 @@ class RoomCreationHandler: return room_id, room_alias, last_stream_id + async def _generate_create_event_for_room_id( + self, + creator: Requester, + creation_content: JsonDict, + is_public: bool, + room_version: RoomVersion, + ) -> Tuple[EventBase, synapse.events.snapshot.EventContext]: + ( + creation_event, + new_unpersisted_context, + ) = await self.event_creation_handler.create_event( + creator, + { + "content": creation_content, + "sender": creator.user.to_string(), + "type": EventTypes.Create, + "state_key": "", + }, + prev_event_ids=[], + depth=1, + state_map={}, + for_batch=False, + ) + await self.store.store_room( + room_id=creation_event.room_id, + room_creator_user_id=creator.user.to_string(), + is_public=is_public, + room_version=room_version, + ) + creation_context = await new_unpersisted_context.persist(creation_event) + return (creation_event, creation_context) + async def _send_events_for_new_room( self, creator: Requester, @@ -1034,6 +1424,9 @@ class RoomCreationHandler: power_level_content_override: Optional[JsonDict] = None, creator_join_profile: Optional[JsonDict] = None, ignore_forced_encryption: bool = False, + creation_event_with_context: Optional[ + Tuple[EventBase, synapse.events.snapshot.EventContext] + ] = None, ) -> Tuple[int, str, int]: """Sends the initial events into a new room. Sends the room creation, membership, and power level events into the room sequentially, then creates and batches up the @@ -1070,7 +1463,10 @@ class RoomCreationHandler: user in this room. ignore_forced_encryption: Ignore encryption forced by `encryption_enabled_by_default_for_room_type` setting. - + creation_event_with_context: + Set in MSC4291 rooms where the create event determines the room ID. If provided, + does not create an additional create event but instead appends the remaining new + events onto the provided create event. Returns: A tuple containing the stream ID, event ID and depth of the last event sent to the room. @@ -1135,13 +1531,26 @@ class RoomCreationHandler: preset_config, config = self._room_preset_config(room_config) - # MSC2175 removes the creator field from the create event. - if not room_version.implicit_room_creator: - creation_content["creator"] = creator_id - creation_event, unpersisted_creation_context = await create_event( - EventTypes.Create, creation_content, False - ) - creation_context = await unpersisted_creation_context.persist(creation_event) + if creation_event_with_context is None: + # MSC2175 removes the creator field from the create event. + if not room_version.implicit_room_creator: + creation_content["creator"] = creator_id + creation_event, unpersisted_creation_context = await create_event( + EventTypes.Create, creation_content, False + ) + creation_context = await unpersisted_creation_context.persist( + creation_event + ) + else: + (creation_event, creation_context) = creation_event_with_context + # we had to do the above already in order to have a room ID, so just updates local vars + # and continue. + depth = 2 + prev_event = [creation_event.event_id] + state_map[(creation_event.type, creation_event.state_key)] = ( + creation_event.event_id + ) + logger.debug("Sending %s in new room", EventTypes.Member) ev = await self.event_creation_handler.handle_new_client_event( requester=creator, @@ -1190,7 +1599,9 @@ class RoomCreationHandler: # Please update the docs for `default_power_level_content_override` when # updating the `events` dict below power_level_content: JsonDict = { - "users": {creator_id: 100}, + "users": {creator_id: 100} + if not room_version.msc4289_creator_power_enabled + else {}, "users_default": 0, "events": { EventTypes.Name: 50, @@ -1198,7 +1609,9 @@ class RoomCreationHandler: EventTypes.RoomHistoryVisibility: 100, EventTypes.CanonicalAlias: 50, EventTypes.RoomAvatar: 50, - EventTypes.Tombstone: 100, + EventTypes.Tombstone: 150 + if room_version.msc4289_creator_power_enabled + else 100, EventTypes.ServerACL: 100, EventTypes.RoomEncryption: 100, }, @@ -1211,7 +1624,13 @@ class RoomCreationHandler: "historical": 100, } - if config["original_invitees_have_ops"]: + # original_invitees_have_ops is set on preset:trusted_private_chat which will already + # have set these users as additional_creators, hence don't set the PL for creators as + # that is invalid. + if ( + config["original_invitees_have_ops"] + and not room_version.msc4289_creator_power_enabled + ): for invitee in invite_list: power_level_content["users"][invitee] = 100 @@ -1296,7 +1715,13 @@ class RoomCreationHandler: topic = room_config["topic"] topic_event, topic_context = await create_event( EventTypes.Topic, - {"topic": topic}, + { + EventContentFields.TOPIC: topic, + EventContentFields.M_TOPIC: { + # The mimetype property defaults to `text/plain` if omitted. + EventContentFields.M_TEXT: [{MTextFields.BODY: topic}] + }, + }, True, ) events_to_send.append((topic_event, topic_context)) @@ -1378,6 +1803,19 @@ class RoomCreationHandler: ) return preset_name, preset_config + def _remove_creators_from_pl_users_map( + self, + users_map: Dict[str, int], + creator: str, + additional_creators: Optional[List[str]], + ) -> None: + creators = [creator] + if additional_creators: + creators.extend(additional_creators) + for creator in creators: + # the creator(s) cannot be in the users map + users_map.pop(creator, None) + def _generate_room_id(self) -> str: """Generates a random room ID. @@ -1395,7 +1833,7 @@ class RoomCreationHandler: A random room ID of the form "!opaque_id:domain". """ random_string = stringutils.random_string(18) - return RoomID(random_string, self.hs.hostname).to_string() + return RoomIdWithDomain(random_string, self.hs.hostname).to_string() async def _generate_and_create_room_id( self, @@ -1806,7 +2244,7 @@ class RoomShutdownHandler: ] = None, ) -> Optional[ShutdownRoomResponse]: """ - Shuts down a room. Moves all local users and room aliases automatically + Shuts down a room. Moves all joined local users and room aliases automatically to a new room if `new_room_user_id` is set. Otherwise local users only leave the room without any information. @@ -1949,16 +2387,17 @@ class RoomShutdownHandler: # Join users to new room if new_room_user_id: - assert new_room_id is not None - await self.room_member_handler.update_membership( - requester=target_requester, - target=target_requester.user, - room_id=new_room_id, - action=Membership.JOIN, - content={}, - ratelimit=False, - require_consent=False, - ) + if membership == Membership.JOIN: + assert new_room_id is not None + await self.room_member_handler.update_membership( + requester=target_requester, + target=target_requester.user, + room_id=new_room_id, + action=Membership.JOIN, + content={}, + ratelimit=False, + require_consent=False, + ) result["kicked_users"].append(user_id) if update_result_fct: diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 07eac71e2a..9d4307fb07 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -61,16 +61,26 @@ MAX_PUBLIC_ROOMS_IN_RESPONSE = 100 class RoomListHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.hs = hs self.enable_room_list_search = hs.config.roomdirectory.enable_room_list_search self.response_cache: ResponseCache[ Tuple[Optional[int], Optional[str], Optional[ThirdPartyInstanceID]] - ] = ResponseCache(hs.get_clock(), "room_list") + ] = ResponseCache( + clock=hs.get_clock(), + name="room_list", + server_name=self.server_name, + ) self.remote_response_cache: ResponseCache[ Tuple[str, Optional[int], Optional[str], bool, Optional[str]] - ] = ResponseCache(hs.get_clock(), "remote_room_list", timeout_ms=30 * 1000) + ] = ResponseCache( + clock=hs.get_clock(), + name="remote_room_list", + server_name=self.server_name, + timeout_ms=30 * 1000, + ) async def get_local_public_room_list( self, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 70cbbc352b..5ba64912c9 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -42,17 +42,18 @@ from synapse.api.errors import ( ) from synapse.api.ratelimiting import Ratelimiter from synapse.event_auth import get_named_level, get_power_level_event -from synapse.events import EventBase +from synapse.events import EventBase, is_creator from synapse.events.snapshot import EventContext from synapse.handlers.pagination import PURGE_ROOM_ACTION_NAME from synapse.handlers.profile import MAX_AVATAR_URL_LEN, MAX_DISPLAYNAME_LEN from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging import opentracing -from synapse.metrics import event_processing_positions +from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.push import ReplicationCopyPusherRestServlet from synapse.storage.databases.main.state_deltas import StateDelta +from synapse.storage.invite_rule import InviteRule from synapse.types import ( JsonDict, Requester, @@ -158,6 +159,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_room, + ratelimit_callbacks=hs.get_module_api_callbacks().ratelimit, ) # Ratelimiter for invites, keyed by recipient (across all rooms, all @@ -166,6 +168,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_user, + ratelimit_callbacks=hs.get_module_api_callbacks().ratelimit, ) # Ratelimiter for invites, keyed by issuer (across all rooms, all @@ -174,6 +177,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_issuer, + ratelimit_callbacks=hs.get_module_api_callbacks().ratelimit, ) self._third_party_invite_limiter = Ratelimiter( @@ -384,11 +388,11 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): async def _local_membership_update( self, + *, requester: Requester, target: UserID, room_id: str, membership: str, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, @@ -410,11 +414,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): desired membership event. room_id: membership: - - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events state_event_ids: The full state at a given event. This was previously used particularly @@ -482,7 +481,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): "origin_server_ts": origin_server_ts, }, txn_id=txn_id, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, depth=depth, @@ -579,7 +577,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): new_room: bool = False, require_consent: bool = True, outlier: bool = False, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, @@ -603,10 +600,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): outlier: Indicates whether the event is an `outlier`, i.e. if it's from an arbitrary point and floating in the DAG as opposed to being inline with the current DAG. - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events state_event_ids: The full state at a given event. This was previously used particularly @@ -676,7 +669,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): new_room=new_room, require_consent=require_consent, outlier=outlier, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, depth=depth, @@ -699,7 +691,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): new_room: bool = False, require_consent: bool = True, outlier: bool = False, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, @@ -725,10 +716,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): outlier: Indicates whether the event is an `outlier`, i.e. if it's from an arbitrary point and floating in the DAG as opposed to being inline with the current DAG. - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events state_event_ids: The full state at a given event. This was previously used particularly @@ -759,35 +746,41 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): and requester.user.to_string() == self._server_notices_mxid ) - requester_suspended = await self.store.get_user_suspended_status( - requester.user.to_string() - ) - if action == Membership.INVITE and requester_suspended: - raise SynapseError( - 403, - "Sending invites while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) + # The requester may be a regular user, but puppeted by the server. + request_by_server = requester.authenticated_entity == self._server_name - if target.to_string() != requester.user.to_string(): - target_suspended = await self.store.get_user_suspended_status( - target.to_string() + # If the request is initiated by the server, ignore whether the + # requester or target is suspended. + if not request_by_server: + requester_suspended = await self.store.get_user_suspended_status( + requester.user.to_string() ) - else: - target_suspended = requester_suspended + if action == Membership.INVITE and requester_suspended: + raise SynapseError( + 403, + "Sending invites while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) - if action == Membership.JOIN and target_suspended: - raise SynapseError( - 403, - "Joining rooms while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) - if action == Membership.KNOCK and target_suspended: - raise SynapseError( - 403, - "Knocking on rooms while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) + if target.to_string() != requester.user.to_string(): + target_suspended = await self.store.get_user_suspended_status( + target.to_string() + ) + else: + target_suspended = requester_suspended + + if action == Membership.JOIN and target_suspended: + raise SynapseError( + 403, + "Joining rooms while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) + if action == Membership.KNOCK and target_suspended: + raise SynapseError( + 403, + "Knocking on rooms while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) if ( not self.allow_per_room_profiles and not is_requester_server_notices_user @@ -912,7 +905,23 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): additional_fields=block_invite_result[1], ) - # An empty prev_events list is allowed as long as the auth_event_ids are present + # check the invitee's configuration and apply rules. Admins on the server can bypass. + if not is_requester_admin: + invite_config = await self.store.get_invite_config_for_user(target_id) + rule = invite_config.get_invite_rule(requester.user.to_string()) + if rule == InviteRule.BLOCK: + logger.info( + "Automatically rejecting invite from %s due to the the invite filtering rules of %s", + target_id, + requester.user, + ) + raise SynapseError( + 403, + "You are not permitted to invite this user.", + errcode=Codes.INVITE_BLOCKED, + ) + # InviteRule.IGNORE is handled at the sync layer. + if prev_event_ids is not None: return await self._local_membership_update( requester=requester, @@ -921,7 +930,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): membership=effective_membership_state, txn_id=txn_id, ratelimit=ratelimit, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, depth=depth, @@ -1152,9 +1160,8 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): elif effective_membership_state == Membership.KNOCK: if not is_host_in_room: - # The knock needs to be sent over federation instead - remote_room_hosts.append(get_domain_from_id(room_id)) - + # we used to add the domain of the room ID to remote_room_hosts. + # This is not safe in MSC4291 rooms which do not have a domain. content["membership"] = Membership.KNOCK try: @@ -1551,7 +1558,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): require_consent=False, ) except Exception as e: - logger.exception("Error kicking guest user: %s" % (e,)) + logger.exception("Error kicking guest user: %s", e) async def lookup_room_alias( self, room_alias: RoomAlias @@ -1913,7 +1920,7 @@ class RoomMemberMasterHandler(RoomMemberHandler): check_complexity and self.hs.config.server.limit_remote_rooms.admins_can_join ): - check_complexity = not await self.store.is_server_admin(user) + check_complexity = not await self.store.is_server_admin(user.to_string()) if check_complexity: # Fetch the room complexity @@ -2162,6 +2169,7 @@ class RoomForgetterHandler(StateDeltasHandler): super().__init__(hs) self._hs = hs + self.server_name = hs.hostname self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self._clock = hs.get_clock() @@ -2193,7 +2201,9 @@ class RoomForgetterHandler(StateDeltasHandler): finally: self._is_processing = False - run_as_background_process("room_forgetter.notify_new_event", process) + run_as_background_process( + "room_forgetter.notify_new_event", self.server_name, process + ) async def _unsafe_process(self) -> None: # If self.pos is None then means we haven't fetched it from DB @@ -2250,7 +2260,9 @@ class RoomForgetterHandler(StateDeltasHandler): self.pos = max_pos # Expose current event processing position to prometheus - event_processing_positions.labels("room_forgetter").set(max_pos) + event_processing_positions.labels( + name="room_forgetter", **{SERVER_NAME_LABEL: self.server_name} + ).set(max_pos) await self._store.update_room_forgetter_stream_pos(max_pos) @@ -2311,6 +2323,7 @@ def get_users_which_can_issue_invite(auth_events: StateMap[EventBase]) -> List[s # Check which members are able to invite by ensuring they're joined and have # the necessary power level. + create_event = auth_events[(EventTypes.Create, "")] for (event_type, state_key), event in auth_events.items(): if event_type != EventTypes.Member: continue @@ -2318,8 +2331,12 @@ def get_users_which_can_issue_invite(auth_events: StateMap[EventBase]) -> List[s if event.membership != Membership.JOIN: continue + if create_event.room_version.msc4289_creator_power_enabled and is_creator( + create_event, state_key + ): + result.append(state_key) # Check if the user has a custom power level. - if users.get(state_key, users_default_level) >= invite_level: + elif users.get(state_key, users_default_level) >= invite_level: result.append(state_key) return result diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py new file mode 100644 index 0000000000..170c477d6f --- /dev/null +++ b/synapse/handlers/room_policy.py @@ -0,0 +1,92 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2016-2021 The Matrix.org Foundation C.I.C. +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import logging +from typing import TYPE_CHECKING + +from synapse.events import EventBase +from synapse.types.handlers.policy_server import RECOMMENDATION_OK +from synapse.util.stringutils import parse_and_validate_server_name + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class RoomPolicyHandler: + def __init__(self, hs: "HomeServer"): + self._hs = hs + self._store = hs.get_datastores().main + self._storage_controllers = hs.get_storage_controllers() + self._event_auth_handler = hs.get_event_auth_handler() + self._federation_client = hs.get_federation_client() + + async def is_event_allowed(self, event: EventBase) -> bool: + """Check if the given event is allowed in the room by the policy server. + + Note: This will *always* return True if the room's policy server is Synapse + itself. This is because Synapse can't be a policy server (currently). + + If no policy server is configured in the room, this returns True. Similarly, if + the policy server is invalid in any way (not joined, not a server, etc), this + returns True. + + If a valid and contactable policy server is configured in the room, this returns + True if that server suggests the event is not spammy, and False otherwise. + + Args: + event: The event to check. This should be a fully-formed PDU. + + Returns: + bool: True if the event is allowed in the room, False otherwise. + """ + if event.type == "org.matrix.msc4284.policy" and event.state_key is not None: + return True # always allow policy server change events + + policy_event = await self._storage_controllers.state.get_current_state_event( + event.room_id, "org.matrix.msc4284.policy", "" + ) + if not policy_event: + return True # no policy server == default allow + + policy_server = policy_event.content.get("via", "") + if policy_server is None or not isinstance(policy_server, str): + return True # no policy server == default allow + + if policy_server == self._hs.hostname: + return True # Synapse itself can't be a policy server (currently) + + try: + parse_and_validate_server_name(policy_server) + except ValueError: + return True # invalid policy server == default allow + + is_in_room = await self._event_auth_handler.is_host_in_room( + event.room_id, policy_server + ) + if not is_in_room: + return True # policy server not in room == default allow + + # At this point, the server appears valid and is in the room, so ask it to check + # the event. + recommendation = await self._federation_client.get_pdu_policy_recommendation( + policy_server, event + ) + if recommendation != RECOMMENDATION_OK: + return False + + return True # default allow diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 64f5bea014..838fee6a30 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -96,6 +96,7 @@ class RoomSummaryHandler: _PAGINATION_SESSION_VALIDITY_PERIOD_MS = 5 * 60 * 1000 def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._event_auth_handler = hs.get_event_auth_handler() self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -111,10 +112,19 @@ class RoomSummaryHandler: # If a user tries to fetch the same page multiple times in quick succession, # only process the first attempt and return its result to subsequent requests. self._pagination_response_cache: ResponseCache[ - Tuple[str, str, bool, Optional[int], Optional[int], Optional[str]] + Tuple[ + str, + str, + bool, + Optional[int], + Optional[int], + Optional[str], + Optional[Tuple[str, ...]], + ] ] = ResponseCache( - hs.get_clock(), - "get_room_hierarchy", + clock=hs.get_clock(), + name="get_room_hierarchy", + server_name=self.server_name, ) self._msc3266_enabled = hs.config.experimental.msc3266_enabled @@ -126,6 +136,7 @@ class RoomSummaryHandler: max_depth: Optional[int] = None, limit: Optional[int] = None, from_token: Optional[str] = None, + remote_room_hosts: Optional[Tuple[str, ...]] = None, ) -> JsonDict: """ Implementation of the room hierarchy C-S API. @@ -143,6 +154,9 @@ class RoomSummaryHandler: limit: An optional limit on the number of rooms to return per page. Must be a positive integer. from_token: An optional pagination token. + remote_room_hosts: An optional list of remote homeserver server names. If defined, + each host will be used to try and fetch the room hierarchy. Must be a tuple so + that it can be hashed by the `RoomSummaryHandler._pagination_response_cache`. Returns: The JSON hierarchy dictionary. @@ -162,6 +176,7 @@ class RoomSummaryHandler: max_depth, limit, from_token, + remote_room_hosts, ), self._get_room_hierarchy, requester.user.to_string(), @@ -170,6 +185,7 @@ class RoomSummaryHandler: max_depth, limit, from_token, + remote_room_hosts, ) async def _get_room_hierarchy( @@ -180,6 +196,7 @@ class RoomSummaryHandler: max_depth: Optional[int] = None, limit: Optional[int] = None, from_token: Optional[str] = None, + remote_room_hosts: Optional[Tuple[str, ...]] = None, ) -> JsonDict: """See docstring for SpaceSummaryHandler.get_room_hierarchy.""" @@ -199,7 +216,7 @@ class RoomSummaryHandler: if not local_room: room_hierarchy = await self._summarize_remote_room_hierarchy( - _RoomQueueEntry(requested_room_id, ()), + _RoomQueueEntry(requested_room_id, remote_room_hosts or ()), False, ) root_room_entry = room_hierarchy[0] @@ -240,7 +257,7 @@ class RoomSummaryHandler: processed_rooms = set(pagination_session["processed_rooms"]) else: # The queue of rooms to process, the next room is last on the stack. - room_queue = [_RoomQueueEntry(requested_room_id, ())] + room_queue = [_RoomQueueEntry(requested_room_id, remote_room_hosts or ())] # Rooms we have already processed. processed_rooms = set() @@ -701,7 +718,7 @@ class RoomSummaryHandler: # The API doesn't return the room version so assume that a # join rule of knock is valid. if ( - room.get("join_rule") + room.get("join_rule", JoinRules.PUBLIC) in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) or room.get("world_readable") is True ): diff --git a/synapse/handlers/saml.py b/synapse/handlers/saml.py index 8ebd3d4ff9..81bec7499c 100644 --- a/synapse/handlers/saml.py +++ b/synapse/handlers/saml.py @@ -124,7 +124,7 @@ class SamlHandler: ) # Since SAML sessions timeout it is useful to log when they were created. - logger.info("Initiating a new SAML session: %s" % (reqid,)) + logger.info("Initiating a new SAML session: %s", reqid) now = self.clock.time_msec() self._outstanding_requests_dict[reqid] = Saml2SessionData( diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py index 70cdb0721c..6469b182c8 100644 --- a/synapse/handlers/send_email.py +++ b/synapse/handlers/send_email.py @@ -24,16 +24,13 @@ import logging from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from io import BytesIO -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Dict, Optional -from pkg_resources import parse_version - -import twisted from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint -from twisted.internet.interfaces import IOpenSSLContextFactory, IProtocolFactory +from twisted.internet.interfaces import IProtocolFactory from twisted.internet.ssl import optionsForClientTLS -from twisted.mail.smtp import ESMTPSender, ESMTPSenderFactory +from twisted.mail.smtp import ESMTPSenderFactory from twisted.protocols.tls import TLSMemoryBIOFactory from synapse.logging.context import make_deferred_yieldable @@ -44,19 +41,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_is_old_twisted = parse_version(twisted.__version__) < parse_version("21") - - -class _NoTLSESMTPSender(ESMTPSender): - """Extend ESMTPSender to disable TLS - - Unfortunately, before Twisted 21.2, ESMTPSender doesn't give an easy way to disable - TLS, so we override its internal method which it uses to generate a context factory. - """ - - def _getContextFactory(self) -> Optional[IOpenSSLContextFactory]: - return None - async def _sendmail( reactor: ISynapseReactor, @@ -71,6 +55,7 @@ async def _sendmail( require_tls: bool = False, enable_tls: bool = True, force_tls: bool = False, + tlsname: Optional[str] = None, ) -> None: """A simple wrapper around ESMTPSenderFactory, to allow substitution in tests @@ -88,39 +73,31 @@ async def _sendmail( enable_tls: True to enable STARTTLS. If this is False and require_tls is True, the request will fail. force_tls: True to enable Implicit TLS. + tlsname: the domain name expected as the TLS certificate's commonname, + defaults to smtphost. """ msg = BytesIO(msg_bytes) d: "Deferred[object]" = Deferred() + if not enable_tls: + tlsname = None + elif tlsname is None: + tlsname = smtphost - def build_sender_factory(**kwargs: Any) -> ESMTPSenderFactory: - return ESMTPSenderFactory( - username, - password, - from_addr, - to_addr, - msg, - d, - heloFallback=True, - requireAuthentication=require_auth, - requireTransportSecurity=require_tls, - **kwargs, - ) - - factory: IProtocolFactory - if _is_old_twisted: - # before twisted 21.2, we have to override the ESMTPSender protocol to disable - # TLS - factory = build_sender_factory() - - if not enable_tls: - factory.protocol = _NoTLSESMTPSender - else: - # for twisted 21.2 and later, there is a 'hostname' parameter which we should - # set to enable TLS. - factory = build_sender_factory(hostname=smtphost if enable_tls else None) + factory: IProtocolFactory = ESMTPSenderFactory( + username, + password, + from_addr, + to_addr, + msg, + d, + heloFallback=True, + requireAuthentication=require_auth, + requireTransportSecurity=require_tls, + hostname=tlsname, + ) if force_tls: - factory = TLSMemoryBIOFactory(optionsForClientTLS(smtphost), True, factory) + factory = TLSMemoryBIOFactory(optionsForClientTLS(tlsname), True, factory) endpoint = HostnameEndpoint( reactor, smtphost, smtpport, timeout=30, bindAddress=None @@ -148,6 +125,7 @@ class SendEmailHandler: self._require_transport_security = hs.config.email.require_transport_security self._enable_tls = hs.config.email.enable_smtp_tls self._force_tls = hs.config.email.force_tls + self._tlsname = hs.config.email.email_tlsname self._sendmail = _sendmail @@ -171,7 +149,7 @@ class SendEmailHandler: additional_headers: A map of additional headers to include. """ try: - from_string = self._from % {"app": app_name} + from_string = self._from % {"app": app_name} # type: ignore[operator] except (KeyError, TypeError): from_string = self._from @@ -212,7 +190,7 @@ class SendEmailHandler: multipart_msg.attach(text_part) multipart_msg.attach(html_part) - logger.info("Sending email to %s" % email_address) + logger.info("Sending email to %s", email_address) await self._sendmail( self._reactor, @@ -227,4 +205,5 @@ class SendEmailHandler: require_tls=self._require_transport_security, enable_tls=self._enable_tls, force_tls=self._force_tls, + tlsname=self._tlsname, ) diff --git a/synapse/handlers/set_password.py b/synapse/handlers/set_password.py index 29cc03d71d..54116a9b72 100644 --- a/synapse/handlers/set_password.py +++ b/synapse/handlers/set_password.py @@ -21,7 +21,6 @@ import logging from typing import TYPE_CHECKING, Optional from synapse.api.errors import Codes, StoreError, SynapseError -from synapse.handlers.device import DeviceHandler from synapse.types import Requester if TYPE_CHECKING: @@ -36,10 +35,7 @@ class SetPasswordHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self._auth_handler = hs.get_auth_handler() - # This can only be instantiated on the main process. - device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) - self._device_handler = device_handler + self._device_handler = hs.get_device_handler() async def set_password( self, diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 85cfbc6dbf..255a041d0e 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -38,7 +38,9 @@ from synapse.logging.opentracing import ( tag_args, trace, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.storage.databases.main.roommember import extract_heroes_from_room_summary +from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.databases.main.stream import PaginateFunction from synapse.storage.roommember import ( MemberSummary, @@ -48,6 +50,7 @@ from synapse.types import ( MutableStateMap, PersistedEventPosition, Requester, + RoomStreamToken, SlidingSyncStreamToken, StateMap, StrCollection, @@ -77,7 +80,7 @@ logger = logging.getLogger(__name__) sync_processing_time = Histogram( "synapse_sliding_sync_processing_time", "Time taken to generate a sliding sync response, ignoring wait times.", - ["initial"], + labelnames=["initial", SERVER_NAME_LABEL], ) # Limit the number of state_keys we should remember sending down the connection for each @@ -92,6 +95,7 @@ MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER = 100 class SlidingSyncHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.clock = hs.get_clock() self.store = hs.get_datastores().main self.storage_controllers = hs.get_storage_controllers() @@ -112,7 +116,7 @@ class SlidingSyncHandler: sync_config: SlidingSyncConfig, from_token: Optional[SlidingSyncStreamToken] = None, timeout_ms: int = 0, - ) -> SlidingSyncResult: + ) -> Tuple[SlidingSyncResult, bool]: """ Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then @@ -124,9 +128,16 @@ class SlidingSyncHandler: from_token: The point in the stream to sync from. Token of the end of the previous batch. May be `None` if this is the initial sync request. timeout_ms: The time in milliseconds to wait for new data to arrive. If 0, - we will immediately but there might not be any new data so we just return an - empty response. + we will respond immediately but there might not be any new data so we just + return an empty response. + + Returns: + A tuple containing the `SlidingSyncResult` and whether we waited for new + activity before responding. Knowing whether we waited is useful in traces + to filter out long-running requests where we were just waiting. """ + did_wait = False + # If the user is not part of the mau group, then check that limits have # not been exceeded (if not part of the group by this point, almost certain # auth_blocking will occur) @@ -145,7 +156,7 @@ class SlidingSyncHandler: logger.warning( "Timed out waiting for worker to catch up. Returning empty response" ) - return SlidingSyncResult.empty(from_token) + return SlidingSyncResult.empty(from_token), did_wait # If we've spent significant time waiting to catch up, take it off # the timeout. @@ -181,8 +192,9 @@ class SlidingSyncHandler: current_sync_callback, from_token=from_token.stream_token, ) + did_wait = True - return result + return result, did_wait @trace async def current_sync_for_user( @@ -199,7 +211,7 @@ class SlidingSyncHandler: Args: sync_config: Sync configuration - to_token: The point in the stream to sync up to. + to_token: The latest point in the stream to sync up to. from_token: The point in the stream to sync from. Token of the end of the previous batch. May be `None` if this is the initial sync request. """ @@ -269,6 +281,7 @@ class SlidingSyncHandler: from_token=from_token, to_token=to_token, newly_joined=room_id in interested_rooms.newly_joined_rooms, + newly_left=room_id in interested_rooms.newly_left_rooms, is_dm=room_id in interested_rooms.dm_room_ids, ) @@ -365,9 +378,9 @@ class SlidingSyncHandler: set_tag(SynapseTags.FUNC_ARG_PREFIX + "sync_config.user", user_id) end_time_s = self.clock.time() - sync_processing_time.labels(from_token is not None).observe( - end_time_s - start_time_s - ) + sync_processing_time.labels( + initial=from_token is not None, **{SERVER_NAME_LABEL: self.server_name} + ).observe(end_time_s - start_time_s) return sliding_sync_result @@ -470,6 +483,64 @@ class SlidingSyncHandler: return state_map + @trace + async def get_current_state_deltas_for_room( + self, + room_id: str, + room_membership_for_user_at_to_token: RoomsForUserType, + from_token: RoomStreamToken, + to_token: RoomStreamToken, + ) -> List[StateDelta]: + """ + Get the state deltas between two tokens taking into account the user's + membership. If the user is LEAVE/BAN, we will only get the state deltas up to + their LEAVE/BAN event (inclusive). + + (> `from_token` and <= `to_token`) + """ + membership = room_membership_for_user_at_to_token.membership + # We don't know how to handle `membership` values other than these. The + # code below would need to be updated. + assert membership in ( + Membership.JOIN, + Membership.INVITE, + Membership.KNOCK, + Membership.LEAVE, + Membership.BAN, + ) + + # People shouldn't see past their leave/ban event + if membership in ( + Membership.LEAVE, + Membership.BAN, + ): + to_bound = ( + room_membership_for_user_at_to_token.event_pos.to_room_stream_token() + ) + # If we are participating in the room, we can get the latest current state in + # the room + elif membership == Membership.JOIN: + to_bound = to_token + # We can only rely on the stripped state included in the invite/knock event + # itself so there will never be any state deltas to send down. + elif membership in (Membership.INVITE, Membership.KNOCK): + return [] + else: + # We don't know how to handle this type of membership yet + # + # FIXME: We should use `assert_never` here but for some reason + # the exhaustive matching doesn't recognize the `Never` here. + # assert_never(membership) + raise AssertionError( + f"Unexpected membership {membership} that we don't know how to handle yet" + ) + + return await self.store.get_current_state_deltas_for_room( + room_id=room_id, + from_token=from_token, + to_token=to_bound, + ) + @trace async def get_room_sync_data( self, @@ -482,6 +553,7 @@ class SlidingSyncHandler: from_token: Optional[SlidingSyncStreamToken], to_token: StreamToken, newly_joined: bool, + newly_left: bool, is_dm: bool, ) -> SlidingSyncResult.RoomResult: """ @@ -499,6 +571,7 @@ class SlidingSyncHandler: from_token: The point in the stream to sync from. to_token: The point in the stream to sync up to. newly_joined: If the user has newly joined the room + newly_left: If the user has newly left the room is_dm: Whether the room is a DM room """ user = sync_config.user @@ -755,13 +828,19 @@ class SlidingSyncHandler: stripped_state = [] if invite_or_knock_event.membership == Membership.INVITE: - stripped_state.extend( - invite_or_knock_event.unsigned.get("invite_room_state", []) + invite_state = invite_or_knock_event.unsigned.get( + "invite_room_state", [] ) + if not isinstance(invite_state, list): + invite_state = [] + + stripped_state.extend(invite_state) elif invite_or_knock_event.membership == Membership.KNOCK: - stripped_state.extend( - invite_or_knock_event.unsigned.get("knock_room_state", []) - ) + knock_state = invite_or_knock_event.unsigned.get("knock_room_state", []) + if not isinstance(knock_state, list): + knock_state = [] + + stripped_state.extend(knock_state) stripped_state.append(strip_event(invite_or_knock_event)) @@ -790,8 +869,29 @@ class SlidingSyncHandler: # TODO: Limit the number of state events we're about to send down # the room, if its too many we should change this to an # `initial=True`? - deltas = await self.store.get_current_state_deltas_for_room( + + # For the case of rejecting remote invites, the leave event won't be + # returned by `get_current_state_deltas_for_room`. This is due to the current + # state only being filled out for rooms the server is in, and so doesn't pick + # up out-of-band leaves (including locally rejected invites) as these events + # are outliers and not added to the `current_state_delta_stream`. + # + # We rely on being explicitly told that the room has been `newly_left` to + # ensure we extract the out-of-band leave. + if newly_left and room_membership_for_user_at_to_token.event_id is not None: + membership_changed = True + leave_event = await self.store.get_event( + room_membership_for_user_at_to_token.event_id + ) + state_key = leave_event.get_state_key() + if state_key is not None: + room_state_delta_id_map[(leave_event.type, state_key)] = ( + room_membership_for_user_at_to_token.event_id + ) + + deltas = await self.get_current_state_deltas_for_room( room_id=room_id, + room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, from_token=from_bound, to_token=to_token.room_key, ) @@ -955,15 +1055,21 @@ class SlidingSyncHandler: and state_key == StateValues.LAZY ): lazy_load_room_members = True + # Everyone in the timeline is relevant - # - # FIXME: We probably also care about invite, ban, kick, targets, etc - # but the spec only mentions "senders". timeline_membership: Set[str] = set() if timeline_events is not None: for timeline_event in timeline_events: + # Anyone who sent a message is relevant timeline_membership.add(timeline_event.sender) + # We also care about invite, ban, kick, targets, + # etc. + if timeline_event.type == EventTypes.Member: + timeline_membership.add( + timeline_event.state_key + ) + # Update the required state filter so we pick up the new # membership for user_id in timeline_membership: diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 077887ec32..25ee954b7f 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -27,7 +27,7 @@ from typing import ( cast, ) -from typing_extensions import assert_never +from typing_extensions import TypeAlias, assert_never from synapse.api.constants import AccountDataTypes, EduTypes from synapse.handlers.receipts import ReceiptEventSource @@ -40,6 +40,7 @@ from synapse.types import ( SlidingSyncStreamToken, StrCollection, StreamToken, + ThreadSubscriptionsToken, ) from synapse.types.handlers.sliding_sync import ( HaveSentRoomFlag, @@ -54,6 +55,13 @@ from synapse.util.async_helpers import ( gather_optional_coroutines, ) +_ThreadSubscription: TypeAlias = ( + SlidingSyncResult.Extensions.ThreadSubscriptionsExtension.ThreadSubscription +) +_ThreadUnsubscription: TypeAlias = ( + SlidingSyncResult.Extensions.ThreadSubscriptionsExtension.ThreadUnsubscription +) + if TYPE_CHECKING: from synapse.server import HomeServer @@ -68,6 +76,7 @@ class SlidingSyncExtensionHandler: self.event_sources = hs.get_event_sources() self.device_handler = hs.get_device_handler() self.push_rules_handler = hs.get_push_rules_handler() + self._enable_thread_subscriptions = hs.config.experimental.msc4306_enabled @trace async def get_extensions_response( @@ -93,7 +102,7 @@ class SlidingSyncExtensionHandler: actual_room_ids: The actual room IDs in the the Sliding Sync response. actual_room_response_map: A map of room ID to room results in the the Sliding Sync response. - to_token: The point in the stream to sync up to. + to_token: The latest point in the stream to sync up to. from_token: The point in the stream to sync from. """ @@ -156,18 +165,32 @@ class SlidingSyncExtensionHandler: from_token=from_token, ) + thread_subs_coro = None + if ( + sync_config.extensions.thread_subscriptions is not None + and self._enable_thread_subscriptions + ): + thread_subs_coro = self.get_thread_subscriptions_extension_response( + sync_config=sync_config, + thread_subscriptions_request=sync_config.extensions.thread_subscriptions, + to_token=to_token, + from_token=from_token, + ) + ( to_device_response, e2ee_response, account_data_response, receipts_response, typing_response, + thread_subs_response, ) = await gather_optional_coroutines( to_device_coro, e2ee_coro, account_data_coro, receipts_coro, typing_coro, + thread_subs_coro, ) return SlidingSyncResult.Extensions( @@ -176,6 +199,7 @@ class SlidingSyncExtensionHandler: account_data=account_data_response, receipts=receipts_response, typing=typing_response, + thread_subscriptions=thread_subs_response, ) def find_relevant_room_ids_for_extension( @@ -877,3 +901,72 @@ class SlidingSyncExtensionHandler: return SlidingSyncResult.Extensions.TypingExtension( room_id_to_typing_map=room_id_to_typing_map, ) + + async def get_thread_subscriptions_extension_response( + self, + sync_config: SlidingSyncConfig, + thread_subscriptions_request: SlidingSyncConfig.Extensions.ThreadSubscriptionsExtension, + to_token: StreamToken, + from_token: Optional[SlidingSyncStreamToken], + ) -> Optional[SlidingSyncResult.Extensions.ThreadSubscriptionsExtension]: + """Handle Thread Subscriptions extension (MSC4308) + + Args: + sync_config: Sync configuration + thread_subscriptions_request: The thread_subscriptions extension from the request + to_token: The point in the stream to sync up to. + from_token: The point in the stream to sync from. + + Returns: + the response (None if empty or thread subscriptions are disabled) + """ + if not thread_subscriptions_request.enabled: + return None + + limit = thread_subscriptions_request.limit + + if from_token: + from_stream_id = from_token.stream_token.thread_subscriptions_key + else: + from_stream_id = StreamToken.START.thread_subscriptions_key + + to_stream_id = to_token.thread_subscriptions_key + + updates = await self.store.get_latest_updated_thread_subscriptions_for_user( + user_id=sync_config.user.to_string(), + from_id=from_stream_id, + to_id=to_stream_id, + limit=limit, + ) + + if len(updates) == 0: + return None + + subscribed_threads: Dict[str, Dict[str, _ThreadSubscription]] = {} + unsubscribed_threads: Dict[str, Dict[str, _ThreadUnsubscription]] = {} + for stream_id, room_id, thread_root_id, subscribed, automatic in updates: + if subscribed: + subscribed_threads.setdefault(room_id, {})[thread_root_id] = ( + _ThreadSubscription( + automatic=automatic, + bump_stamp=stream_id, + ) + ) + else: + unsubscribed_threads.setdefault(room_id, {})[thread_root_id] = ( + _ThreadUnsubscription(bump_stamp=stream_id) + ) + + prev_batch = None + if len(updates) == limit: + # Tell the client about a potential gap where there may be more + # thread subscriptions for it to backpaginate. + # We subtract one because the 'later in the stream' bound is inclusive, + # and we already saw the element at index 0. + prev_batch = ThreadSubscriptionsToken(updates[0][0] - 1) + + return SlidingSyncResult.Extensions.ThreadSubscriptionsExtension( + subscribed=subscribed_threads, + unsubscribed=unsubscribed_threads, + prev_batch=prev_batch, + ) diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index a1730b7e05..e196199f8a 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -23,6 +23,7 @@ from typing import ( List, Literal, Mapping, + MutableMapping, Optional, Set, Tuple, @@ -49,6 +50,7 @@ from synapse.storage.databases.main.state import ( Sentinel as StateSentinel, ) from synapse.storage.databases.main.stream import CurrentStateDeltaMembership +from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import ( RoomsForUser, RoomsForUserSlidingSync, @@ -72,6 +74,7 @@ from synapse.types.handlers.sliding_sync import ( SlidingSyncResult, ) from synapse.types.state import StateFilter +from synapse.util import MutableOverlayMapping if TYPE_CHECKING: from synapse.server import HomeServer @@ -244,31 +247,65 @@ class SlidingSyncRoomLists: # Note: this won't include rooms the user has left themselves. We add back # `newly_left` rooms below. This is more efficient than fetching all rooms and # then filtering out the old left rooms. - room_membership_for_user_map = await self.store.get_sliding_sync_rooms_for_user( - user_id + room_membership_for_user_map: MutableMapping[str, RoomsForUserSlidingSync] = ( + MutableOverlayMapping( + await self.store.get_sliding_sync_rooms_for_user_from_membership_snapshots( + user_id + ) + ) ) + # To play nice with the rewind logic below, we need to go fetch the rooms the + # user has left themselves but only if it changed after the `to_token`. + # + # If a leave happens *after* the token range, we may have still been joined (or + # any non-self-leave which is relevant to sync) to the room before so we need to + # include it in the list of potentially relevant rooms and apply our rewind + # logic (outside of this function) to see if it's actually relevant. + # + # We do this separately from + # `get_sliding_sync_rooms_for_user_from_membership_snapshots` as those results + # are cached and the `to_token` isn't very cache friendly (people are constantly + # requesting with new tokens) so we separate it out here. + self_leave_room_membership_for_user_map = ( + await self.store.get_sliding_sync_self_leave_rooms_after_to_token( + user_id, to_token + ) + ) + if self_leave_room_membership_for_user_map: + room_membership_for_user_map.update(self_leave_room_membership_for_user_map) # Remove invites from ignored users ignored_users = await self.store.ignored_users(user_id) + invite_config = await self.store.get_invite_config_for_user(user_id) if ignored_users: - # TODO: It would be nice to avoid these copies - room_membership_for_user_map = dict(room_membership_for_user_map) # Make a copy so we don't run into an error: `dictionary changed size during # iteration`, when we remove items for room_id in list(room_membership_for_user_map.keys()): room_for_user_sliding_sync = room_membership_for_user_map[room_id] if ( room_for_user_sliding_sync.membership == Membership.INVITE - and room_for_user_sliding_sync.sender in ignored_users + and room_for_user_sliding_sync.sender + and ( + room_for_user_sliding_sync.sender in ignored_users + or invite_config.get_invite_rule( + room_for_user_sliding_sync.sender + ) + == InviteRule.IGNORE + ) ): room_membership_for_user_map.pop(room_id, None) + ( + newly_joined_room_ids, + newly_left_room_map, + ) = await self._get_newly_joined_and_left_rooms( + user_id, from_token=from_token, to_token=to_token + ) + changes = await self._get_rewind_changes_to_current_membership_to_token( sync_config.user, room_membership_for_user_map, to_token=to_token ) if changes: - # TODO: It would be nice to avoid these copies - room_membership_for_user_map = dict(room_membership_for_user_map) for room_id, change in changes.items(): if change is None: # Remove rooms that the user joined after the `to_token` @@ -278,7 +315,7 @@ class SlidingSyncRoomLists: existing_room = room_membership_for_user_map.get(room_id) if existing_room is not None: # Update room membership events to the point in time of the `to_token` - room_membership_for_user_map[room_id] = RoomsForUserSlidingSync( + room_for_user = RoomsForUserSlidingSync( room_id=room_id, sender=change.sender, membership=change.membership, @@ -290,18 +327,18 @@ class SlidingSyncRoomLists: room_type=existing_room.room_type, is_encrypted=existing_room.is_encrypted, ) - - ( - newly_joined_room_ids, - newly_left_room_map, - ) = await self._get_newly_joined_and_left_rooms( - user_id, from_token=from_token, to_token=to_token - ) - dm_room_ids = await self._get_dm_rooms_for_user(user_id) + if filter_membership_for_sync( + user_id=user_id, + room_membership_for_user=room_for_user, + newly_left=room_id in newly_left_room_map, + ): + room_membership_for_user_map[room_id] = room_for_user + else: + room_membership_for_user_map.pop(room_id, None) # Add back `newly_left` rooms (rooms left in the from -> to token range). # - # We do this because `get_sliding_sync_rooms_for_user(...)` doesn't include + # We do this because `get_sliding_sync_rooms_for_user_from_membership_snapshots(...)` doesn't include # rooms that the user left themselves as it's more efficient to add them back # here than to fetch all rooms and then filter out the old left rooms. The user # only leaves a room once in a blue moon so this barely needs to run. @@ -310,8 +347,6 @@ class SlidingSyncRoomLists: newly_left_room_map.keys() - room_membership_for_user_map.keys() ) if missing_newly_left_rooms: - # TODO: It would be nice to avoid these copies - room_membership_for_user_map = dict(room_membership_for_user_map) for room_id in missing_newly_left_rooms: newly_left_room_for_user = newly_left_room_map[room_id] # This should be a given @@ -327,14 +362,21 @@ class SlidingSyncRoomLists: # If the membership exists, it's just a normal user left the room on # their own if newly_left_room_for_user_sliding_sync is not None: - room_membership_for_user_map[room_id] = ( - newly_left_room_for_user_sliding_sync - ) + if filter_membership_for_sync( + user_id=user_id, + room_membership_for_user=newly_left_room_for_user_sliding_sync, + newly_left=room_id in newly_left_room_map, + ): + room_membership_for_user_map[room_id] = ( + newly_left_room_for_user_sliding_sync + ) + else: + room_membership_for_user_map.pop(room_id, None) change = changes.get(room_id) if change is not None: # Update room membership events to the point in time of the `to_token` - room_membership_for_user_map[room_id] = RoomsForUserSlidingSync( + room_for_user = RoomsForUserSlidingSync( room_id=room_id, sender=change.sender, membership=change.membership, @@ -346,6 +388,14 @@ class SlidingSyncRoomLists: room_type=newly_left_room_for_user_sliding_sync.room_type, is_encrypted=newly_left_room_for_user_sliding_sync.is_encrypted, ) + if filter_membership_for_sync( + user_id=user_id, + room_membership_for_user=room_for_user, + newly_left=room_id in newly_left_room_map, + ): + room_membership_for_user_map[room_id] = room_for_user + else: + room_membership_for_user_map.pop(room_id, None) # If we are `newly_left` from the room but can't find any membership, # then we have been "state reset" out of the room @@ -367,7 +417,7 @@ class SlidingSyncRoomLists: newly_left_room_for_user.event_pos.to_room_stream_token(), ) - room_membership_for_user_map[room_id] = RoomsForUserSlidingSync( + room_for_user = RoomsForUserSlidingSync( room_id=room_id, sender=newly_left_room_for_user.sender, membership=newly_left_room_for_user.membership, @@ -378,6 +428,20 @@ class SlidingSyncRoomLists: room_type=room_type, is_encrypted=is_encrypted, ) + if filter_membership_for_sync( + user_id=user_id, + room_membership_for_user=room_for_user, + newly_left=room_id in newly_left_room_map, + ): + room_membership_for_user_map[room_id] = room_for_user + else: + room_membership_for_user_map.pop(room_id, None) + + # Remove any rooms that we globally exclude from sync. + for room_id in self.rooms_to_exclude_globally: + room_membership_for_user_map.pop(room_id, None) + + dm_room_ids = await self._get_dm_rooms_for_user(user_id) if sync_config.lists: sync_room_map = room_membership_for_user_map @@ -493,9 +557,6 @@ class SlidingSyncRoomLists: if sync_config.room_subscriptions: with start_active_span("assemble_room_subscriptions"): - # TODO: It would be nice to avoid these copies - room_membership_for_user_map = dict(room_membership_for_user_map) - # Find which rooms are partially stated and may need to be filtered out # depending on the `required_state` requested (see below). partial_state_rooms = await self.store.get_partial_rooms() @@ -1040,7 +1101,7 @@ class SlidingSyncRoomLists: ( newly_joined_room_ids, newly_left_room_map, - ) = await self._get_newly_joined_and_left_rooms( + ) = await self._get_newly_joined_and_left_rooms_fallback( user_id, to_token=to_token, from_token=from_token ) @@ -1096,6 +1157,53 @@ class SlidingSyncRoomLists: "state reset" out of the room, and so that room would not be part of the "current memberships" of the user. + Returns: + A 2-tuple of newly joined room IDs and a map of newly_left room + IDs to the `RoomsForUserStateReset` entry. + + We're using `RoomsForUserStateReset` but that doesn't necessarily mean the + user was state reset of the rooms. It's just that the `event_id`/`sender` + are optional and we can't tell the difference between the server leaving the + room when the user was the last person participating in the room and left or + was state reset out of the room. To actually check for a state reset, you + need to check if a membership still exists in the room. + """ + + newly_joined_room_ids: Set[str] = set() + newly_left_room_map: Dict[str, RoomsForUserStateReset] = {} + + if not from_token: + return newly_joined_room_ids, newly_left_room_map + + changes = await self.store.get_sliding_sync_membership_changes( + user_id, + from_key=from_token.room_key, + to_key=to_token.room_key, + excluded_room_ids=set(self.rooms_to_exclude_globally), + ) + + for room_id, entry in changes.items(): + if entry.membership == Membership.JOIN: + newly_joined_room_ids.add(room_id) + elif entry.membership == Membership.LEAVE: + newly_left_room_map[room_id] = entry + + return newly_joined_room_ids, newly_left_room_map + + @trace + async def _get_newly_joined_and_left_rooms_fallback( + self, + user_id: str, + to_token: StreamToken, + from_token: Optional[StreamToken], + ) -> Tuple[AbstractSet[str], Mapping[str, RoomsForUserStateReset]]: + """Fetch the sets of rooms that the user newly joined or left in the + given token range. + + Note: there may be rooms in the newly left rooms where the user was + "state reset" out of the room, and so that room would not be part of the + "current memberships" of the user. + Returns: A 2-tuple of newly joined room IDs and a map of newly_left room IDs to the `RoomsForUserStateReset` entry. diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index ee74289b6c..eec420cbb1 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -33,20 +33,19 @@ from typing import ( Mapping, NoReturn, Optional, + Protocol, Set, ) from urllib.parse import urlencode import attr -from typing_extensions import Protocol from twisted.web.iweb import IRequest from twisted.web.server import Request -from synapse.api.constants import LoginType +from synapse.api.constants import LoginType, ProfileFields from synapse.api.errors import Codes, NotFoundError, RedirectException, SynapseError from synapse.config.sso import SsoAttributeRequirement -from synapse.handlers.device import DeviceHandler from synapse.handlers.register import init_counters_for_auth_provider from synapse.handlers.ui_auth import UIAuthSessionDataConstants from synapse.http import get_request_user_agent @@ -203,7 +202,7 @@ class SsoHandler: def __init__(self, hs: "HomeServer"): self._clock = hs.get_clock() self._store = hs.get_datastores().main - self._server_name = hs.hostname + self.server_name = hs.hostname self._is_mine_server_name = hs.is_mine_server_name self._registration_handler = hs.get_registration_handler() self._auth_handler = hs.get_auth_handler() @@ -239,7 +238,9 @@ class SsoHandler: p_id = p.idp_id assert p_id not in self._identity_providers self._identity_providers[p_id] = p - init_counters_for_auth_provider(p_id) + init_counters_for_auth_provider( + auth_provider_id=p_id, server_name=self.server_name + ) def get_identity_providers(self) -> Mapping[str, SsoIdentityProvider]: """Get the configured identity providers""" @@ -570,7 +571,7 @@ class SsoHandler: return attributes # Check if this mxid already exists - user_id = UserID(attributes.localpart, self._server_name).to_string() + user_id = UserID(attributes.localpart, self.server_name).to_string() if not await self._store.get_users_by_id_case_insensitive(user_id): # This mxid is free break @@ -813,17 +814,18 @@ class SsoHandler: # bail if user already has the same avatar profile = await self._profile_handler.get_profile(user_id) - if profile["avatar_url"] is not None: - server_name = profile["avatar_url"].split("/")[-2] - media_id = profile["avatar_url"].split("/")[-1] + if ProfileFields.AVATAR_URL in profile: + avatar_url_parts = profile[ProfileFields.AVATAR_URL].split("/") + server_name = avatar_url_parts[-2] + media_id = avatar_url_parts[-1] if self._is_mine_server_name(server_name): - media = await self._media_repo.store.get_local_media(media_id) # type: ignore[has-type] + media = await self._media_repo.store.get_local_media(media_id) if media is not None and upload_name == media.upload_name: logger.info("skipping saving the user avatar") return True # store it in media repository - avatar_mxc_url = await self._media_repo.create_content( + avatar_mxc_url = await self._media_repo.create_or_update_content( media_type=headers[b"Content-Type"][0].decode("utf-8"), upload_name=upload_name, content=picture, @@ -907,7 +909,7 @@ class SsoHandler: # render an error page. html = self._bad_user_template.render( - server_name=self._server_name, + server_name=self.server_name, user_id_to_verify=user_id_to_verify, ) respond_with_html(request, 200, html) @@ -959,7 +961,7 @@ class SsoHandler: if contains_invalid_mxid_characters(localpart): raise SynapseError(400, "localpart is invalid: %s" % (localpart,)) - user_id = UserID(localpart, self._server_name).to_string() + user_id = UserID(localpart, self.server_name).to_string() user_infos = await self._store.get_users_by_id_case_insensitive(user_id) logger.info("[session %s] users: %s", session_id, user_infos) @@ -1180,8 +1182,6 @@ class SsoHandler: ) -> None: """Revoke any devices and in-flight logins tied to a provider session. - Can only be called from the main process. - Args: auth_provider_id: A unique identifier for this SSO provider, e.g. "oidc" or "saml". @@ -1190,11 +1190,6 @@ class SsoHandler: sessions belonging to other users and log an error. """ - # It is expected that this is the main process. - assert isinstance( - self._device_handler, DeviceHandler - ), "revoking SSO sessions can only be called on the main process" - # Invalidate any running user-mapping sessions to_delete = [] for session_id, session in self._username_mapping_sessions.items(): @@ -1229,12 +1224,16 @@ class SsoHandler: if expected_user_id is not None and user_id != expected_user_id: logger.error( "Received a logout notification from SSO provider " - f"{auth_provider_id!r} for the user {expected_user_id!r}, but with " - f"a session ID ({auth_provider_session_id!r}) which belongs to " - f"{user_id!r}. This may happen when the SSO provider user mapper " + "%r for the user %r, but with " + "a session ID (%r) which belongs to " + "%r. This may happen when the SSO provider user mapper " "uses something else than the standard attribute as mapping ID. " "For OIDC providers, set `backchannel_logout_ignore_sub` to `true` " - "in the provider config if that is the case." + "in the provider config if that is the case.", + auth_provider_id, + expected_user_id, + auth_provider_session_id, + user_id, ) continue @@ -1276,12 +1275,16 @@ def _check_attribute_requirement( return False # If the requirement is None, the attribute existing is enough. - if req.value is None: + if req.value is None and req.one_of is None: return True values = attributes[req.attribute] if req.value in values: return True + if req.one_of: + for value in req.one_of: + if value in values: + return True logger.info( "SSO attribute %s did not match required value '%s' (was '%s')", diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 8f90c17060..a2602ea818 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -32,10 +32,11 @@ from typing import ( ) from synapse.api.constants import EventContentFields, EventTypes, Membership -from synapse.metrics import event_processing_positions +from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main.state_deltas import StateDelta from synapse.types import JsonDict +from synapse.util.events import get_plain_text_topic_from_event_content if TYPE_CHECKING: from synapse.server import HomeServer @@ -53,6 +54,7 @@ class StatsHandler: def __init__(self, hs: "HomeServer"): self.hs = hs + self.server_name = hs.hostname self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.state = hs.get_state_handler() @@ -88,7 +90,7 @@ class StatsHandler: finally: self._is_processing = False - run_as_background_process("stats.notify_new_event", process) + run_as_background_process("stats.notify_new_event", self.server_name, process) async def _unsafe_process(self) -> None: # If self.pos is None then means we haven't fetched it from DB @@ -145,7 +147,9 @@ class StatsHandler: logger.debug("Handled room stats to %s -> %s", self.pos, max_pos) - event_processing_positions.labels("stats").set(max_pos) + event_processing_positions.labels( + name="stats", **{SERVER_NAME_LABEL: self.server_name} + ).set(max_pos) self.pos = max_pos @@ -299,7 +303,9 @@ class StatsHandler: elif delta.event_type == EventTypes.Name: room_state["name"] = event_content.get("name") elif delta.event_type == EventTypes.Topic: - room_state["topic"] = event_content.get("topic") + room_state["topic"] = get_plain_text_topic_from_event_content( + event_content + ) elif delta.event_type == EventTypes.RoomAvatar: room_state["avatar"] = event_content.get("url") elif delta.event_type == EventTypes.CanonicalAlias: diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 350c3fa09a..4a68fdcc76 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -20,7 +20,6 @@ # import itertools import logging -from enum import Enum from typing import ( TYPE_CHECKING, AbstractSet, @@ -28,14 +27,11 @@ from typing import ( Dict, FrozenSet, List, - Literal, Mapping, Optional, Sequence, Set, Tuple, - Union, - overload, ) import attr @@ -63,9 +59,11 @@ from synapse.logging.opentracing import ( start_active_span, trace, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.storage.databases.main.event_push_actions import RoomNotifCounts from synapse.storage.databases.main.roommember import extract_heroes_from_room_summary from synapse.storage.databases.main.stream import PaginateFunction +from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import MemberSummary from synapse.types import ( DeviceListUpdates, @@ -103,7 +101,7 @@ non_empty_sync_counter = Counter( "Count of non empty sync responses. type is initial_sync/full_state_sync" "/incremental_sync. lazy_loaded indicates if lazy loaded members were " "enabled for that request.", - ["type", "lazy_loaded"], + labelnames=["type", "lazy_loaded", SERVER_NAME_LABEL], ) # Store the cache that tracks which lazy-loaded members have been sent to a given @@ -118,25 +116,6 @@ LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE = 100 SyncRequestKey = Tuple[Any, ...] -class SyncVersion(Enum): - """ - Enum for specifying the version of sync request. This is used to key which type of - sync response that we are generating. - - This is different than the `sync_type` you might see used in other code below; which - specifies the sub-type sync request (e.g. initial_sync, full_state_sync, - incremental_sync) and is really only relevant for the `/sync` v2 endpoint. - """ - - # These string values are semantically significant because they are used in the the - # metrics - - # Traditional `/sync` endpoint - SYNC_V2 = "sync_v2" - # Part of MSC3575 Sliding Sync - E2EE_SYNC = "e2ee_sync" - - @attr.s(slots=True, frozen=True, auto_attribs=True) class SyncConfig: user: UserID @@ -306,28 +285,9 @@ class SyncResult: ) -@attr.s(slots=True, frozen=True, auto_attribs=True) -class E2eeSyncResult: - """ - Attributes: - next_batch: Token for the next sync - to_device: List of direct messages for the device. - device_lists: List of user_ids whose devices have changed - device_one_time_keys_count: Dict of algorithm to count for one time keys - for this device - device_unused_fallback_key_types: List of key types that have an unused fallback - key - """ - - next_batch: StreamToken - to_device: List[JsonDict] - device_lists: DeviceListUpdates - device_one_time_keys_count: JsonMapping - device_unused_fallback_key_types: List[str] - - class SyncHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.hs_config = hs.config self.store = hs.get_datastores().main self.notifier = hs.get_notifier() @@ -351,8 +311,9 @@ class SyncHandler: # cached result any more, and we could flush the entry from the cache to save # memory. self.response_cache: ResponseCache[SyncRequestKey] = ResponseCache( - hs.get_clock(), - "sync", + clock=hs.get_clock(), + name="sync", + server_name=self.server_name, timeout_ms=hs.config.caches.sync_response_cache_duration, ) @@ -360,60 +321,24 @@ class SyncHandler: self.lazy_loaded_members_cache: ExpiringCache[ Tuple[str, Optional[str]], LruCache[str, str] ] = ExpiringCache( - "lazy_loaded_members_cache", - self.clock, + cache_name="lazy_loaded_members_cache", + server_name=self.server_name, + clock=self.clock, max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE, ) self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync - @overload async def wait_for_sync_for_user( self, requester: Requester, sync_config: SyncConfig, - sync_version: Literal[SyncVersion.SYNC_V2], request_key: SyncRequestKey, since_token: Optional[StreamToken] = None, timeout: int = 0, full_state: bool = False, - ) -> SyncResult: ... - - @overload - async def wait_for_sync_for_user( - self, - requester: Requester, - sync_config: SyncConfig, - sync_version: Literal[SyncVersion.E2EE_SYNC], - request_key: SyncRequestKey, - since_token: Optional[StreamToken] = None, - timeout: int = 0, - full_state: bool = False, - ) -> E2eeSyncResult: ... - - @overload - async def wait_for_sync_for_user( - self, - requester: Requester, - sync_config: SyncConfig, - sync_version: SyncVersion, - request_key: SyncRequestKey, - since_token: Optional[StreamToken] = None, - timeout: int = 0, - full_state: bool = False, - ) -> Union[SyncResult, E2eeSyncResult]: ... - - async def wait_for_sync_for_user( - self, - requester: Requester, - sync_config: SyncConfig, - sync_version: SyncVersion, - request_key: SyncRequestKey, - since_token: Optional[StreamToken] = None, - timeout: int = 0, - full_state: bool = False, - ) -> Union[SyncResult, E2eeSyncResult]: + ) -> SyncResult: """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then return an empty sync result. @@ -428,8 +353,7 @@ class SyncHandler: full_state: Whether to return the full state for each room. Returns: - When `SyncVersion.SYNC_V2`, returns a full `SyncResult`. - When `SyncVersion.E2EE_SYNC`, returns a `E2eeSyncResult`. + returns a full `SyncResult`. """ # If the user is not part of the mau group, then check that limits have # not been exceeded (if not part of the group by this point, almost certain @@ -441,7 +365,6 @@ class SyncHandler: request_key, self._wait_for_sync_for_user, sync_config, - sync_version, since_token, timeout, full_state, @@ -450,48 +373,14 @@ class SyncHandler: logger.debug("Returning sync response for %s", user_id) return res - @overload async def _wait_for_sync_for_user( self, sync_config: SyncConfig, - sync_version: Literal[SyncVersion.SYNC_V2], since_token: Optional[StreamToken], timeout: int, full_state: bool, cache_context: ResponseCacheContext[SyncRequestKey], - ) -> SyncResult: ... - - @overload - async def _wait_for_sync_for_user( - self, - sync_config: SyncConfig, - sync_version: Literal[SyncVersion.E2EE_SYNC], - since_token: Optional[StreamToken], - timeout: int, - full_state: bool, - cache_context: ResponseCacheContext[SyncRequestKey], - ) -> E2eeSyncResult: ... - - @overload - async def _wait_for_sync_for_user( - self, - sync_config: SyncConfig, - sync_version: SyncVersion, - since_token: Optional[StreamToken], - timeout: int, - full_state: bool, - cache_context: ResponseCacheContext[SyncRequestKey], - ) -> Union[SyncResult, E2eeSyncResult]: ... - - async def _wait_for_sync_for_user( - self, - sync_config: SyncConfig, - sync_version: SyncVersion, - since_token: Optional[StreamToken], - timeout: int, - full_state: bool, - cache_context: ResponseCacheContext[SyncRequestKey], - ) -> Union[SyncResult, E2eeSyncResult]: + ) -> SyncResult: """The start of the machinery that produces a /sync response. See https://spec.matrix.org/v1.1/client-server-api/#syncing for full details. @@ -512,7 +401,7 @@ class SyncHandler: else: sync_type = "incremental_sync" - sync_label = f"{sync_version}:{sync_type}" + sync_label = f"sync_v2:{sync_type}" context = current_context() if context: @@ -573,19 +462,15 @@ class SyncHandler: if timeout == 0 or since_token is None or full_state: # we are going to return immediately, so don't bother calling # notifier.wait_for_events. - result: Union[ - SyncResult, E2eeSyncResult - ] = await self.current_sync_for_user( - sync_config, sync_version, since_token, full_state=full_state + result = await self.current_sync_for_user( + sync_config, since_token, full_state=full_state ) else: # Otherwise, we wait for something to happen and report it to the user. async def current_sync_callback( before_token: StreamToken, after_token: StreamToken - ) -> Union[SyncResult, E2eeSyncResult]: - return await self.current_sync_for_user( - sync_config, sync_version, since_token - ) + ) -> SyncResult: + return await self.current_sync_for_user(sync_config, since_token) result = await self.notifier.wait_for_events( sync_config.user.to_string(), @@ -610,47 +495,23 @@ class SyncHandler: lazy_loaded = "true" else: lazy_loaded = "false" - non_empty_sync_counter.labels(sync_label, lazy_loaded).inc() + non_empty_sync_counter.labels( + type=sync_label, + lazy_loaded=lazy_loaded, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() return result - @overload async def current_sync_for_user( self, sync_config: SyncConfig, - sync_version: Literal[SyncVersion.SYNC_V2], since_token: Optional[StreamToken] = None, full_state: bool = False, - ) -> SyncResult: ... - - @overload - async def current_sync_for_user( - self, - sync_config: SyncConfig, - sync_version: Literal[SyncVersion.E2EE_SYNC], - since_token: Optional[StreamToken] = None, - full_state: bool = False, - ) -> E2eeSyncResult: ... - - @overload - async def current_sync_for_user( - self, - sync_config: SyncConfig, - sync_version: SyncVersion, - since_token: Optional[StreamToken] = None, - full_state: bool = False, - ) -> Union[SyncResult, E2eeSyncResult]: ... - - async def current_sync_for_user( - self, - sync_config: SyncConfig, - sync_version: SyncVersion, - since_token: Optional[StreamToken] = None, - full_state: bool = False, - ) -> Union[SyncResult, E2eeSyncResult]: + ) -> SyncResult: """ Generates the response body of a sync result, represented as a - `SyncResult`/`E2eeSyncResult`. + `SyncResult`. This is a wrapper around `generate_sync_result` which starts an open tracing span to track the sync. See `generate_sync_result` for the next part of your @@ -663,28 +524,15 @@ class SyncHandler: full_state: Whether to return the full state for each room. Returns: - When `SyncVersion.SYNC_V2`, returns a full `SyncResult`. - When `SyncVersion.E2EE_SYNC`, returns a `E2eeSyncResult`. + returns a full `SyncResult`. """ with start_active_span("sync.current_sync_for_user"): log_kv({"since_token": since_token}) # Go through the `/sync` v2 path - if sync_version == SyncVersion.SYNC_V2: - sync_result: Union[ - SyncResult, E2eeSyncResult - ] = await self.generate_sync_result( - sync_config, since_token, full_state - ) - # Go through the MSC3575 Sliding Sync `/sync/e2ee` path - elif sync_version == SyncVersion.E2EE_SYNC: - sync_result = await self.generate_e2ee_sync_result( - sync_config, since_token - ) - else: - raise Exception( - f"Unknown sync_version (this is a Synapse problem): {sync_version}" - ) + sync_result = await self.generate_sync_result( + sync_config, since_token, full_state + ) set_tag(SynapseTags.SYNC_RESULT, bool(sync_result)) return sync_result @@ -709,7 +557,9 @@ class SyncHandler: sync_config = sync_result_builder.sync_config - with Measure(self.clock, "ephemeral_by_room"): + with Measure( + self.clock, name="ephemeral_by_room", server_name=self.server_name + ): typing_key = since_token.typing_key if since_token else 0 room_ids = sync_result_builder.joined_room_ids @@ -782,7 +632,9 @@ class SyncHandler: and current token to send down to clients. newly_joined_room """ - with Measure(self.clock, "load_filtered_recents"): + with Measure( + self.clock, name="load_filtered_recents", server_name=self.server_name + ): timeline_limit = sync_config.filter_collection.timeline_limit() block_all_timeline = ( sync_config.filter_collection.blocks_all_room_timeline() @@ -1128,7 +980,7 @@ class SyncHandler: ) if cache is None: logger.debug("creating LruCache for %r", cache_key) - cache = LruCache(LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE) + cache = LruCache(max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE) self.lazy_loaded_members_cache[cache_key] = cache else: logger.debug("found LruCache for %r", cache_key) @@ -1173,7 +1025,9 @@ class SyncHandler: # updates even if they occurred logically before the previous event. # TODO(mjark) Check for new redactions in the state events. - with Measure(self.clock, "compute_state_delta"): + with Measure( + self.clock, name="compute_state_delta", server_name=self.server_name + ): # The memberships needed for events in the timeline. # Only calculated when `lazy_load_members` is on. members_to_fetch: Optional[Set[str]] = None @@ -1790,7 +1644,9 @@ class SyncHandler: # the DB. return RoomNotifCounts.empty() - with Measure(self.clock, "unread_notifs_for_room_id"): + with Measure( + self.clock, name="unread_notifs_for_room_id", server_name=self.server_name + ): return await self.store.get_unread_event_push_actions_by_room_for_user( room_id, sync_config.user.to_string(), @@ -1951,102 +1807,6 @@ class SyncHandler: next_batch=sync_result_builder.now_token, ) - async def generate_e2ee_sync_result( - self, - sync_config: SyncConfig, - since_token: Optional[StreamToken] = None, - ) -> E2eeSyncResult: - """ - Generates the response body of a MSC3575 Sliding Sync `/sync/e2ee` result. - - This is represented by a `E2eeSyncResult` struct, which is built from small - pieces using a `SyncResultBuilder`. The `sync_result_builder` is passed as a - mutable ("inout") parameter to various helper functions. These retrieve and - process the data which forms the sync body, often writing to the - `sync_result_builder` to store their output. - - At the end, we transfer data from the `sync_result_builder` to a new `E2eeSyncResult` - instance to signify that the sync calculation is complete. - """ - user_id = sync_config.user.to_string() - app_service = self.store.get_app_service_by_user_id(user_id) - if app_service: - # We no longer support AS users using /sync directly. - # See https://github.com/matrix-org/matrix-doc/issues/1144 - raise NotImplementedError() - - sync_result_builder = await self.get_sync_result_builder( - sync_config, - since_token, - full_state=False, - ) - - # 1. Calculate `to_device` events - await self._generate_sync_entry_for_to_device(sync_result_builder) - - # 2. Calculate `device_lists` - # Device list updates are sent if a since token is provided. - device_lists = DeviceListUpdates() - include_device_list_updates = bool(since_token and since_token.device_list_key) - if include_device_list_updates: - # Note that _generate_sync_entry_for_rooms sets sync_result_builder.joined, which - # is used in calculate_user_changes below. - # - # TODO: Running `_generate_sync_entry_for_rooms()` is a lot of work just to - # figure out the membership changes/derived info needed for - # `_generate_sync_entry_for_device_list()`. In the future, we should try to - # refactor this away. - ( - newly_joined_rooms, - newly_left_rooms, - ) = await self._generate_sync_entry_for_rooms(sync_result_builder) - - # This uses the sync_result_builder.joined which is set in - # `_generate_sync_entry_for_rooms`, if that didn't find any joined - # rooms for some reason it is a no-op. - ( - newly_joined_or_invited_or_knocked_users, - newly_left_users, - ) = sync_result_builder.calculate_user_changes() - - # include_device_list_updates can only be True if we have a - # since token. - assert since_token is not None - device_lists = await self._device_handler.generate_sync_entry_for_device_list( - user_id=user_id, - since_token=since_token, - now_token=sync_result_builder.now_token, - joined_room_ids=sync_result_builder.joined_room_ids, - newly_joined_rooms=newly_joined_rooms, - newly_joined_or_invited_or_knocked_users=newly_joined_or_invited_or_knocked_users, - newly_left_rooms=newly_left_rooms, - newly_left_users=newly_left_users, - ) - - # 3. Calculate `device_one_time_keys_count` and `device_unused_fallback_key_types` - device_id = sync_config.device_id - one_time_keys_count: JsonMapping = {} - unused_fallback_key_types: List[str] = [] - if device_id: - # TODO: We should have a way to let clients differentiate between the states of: - # * no change in OTK count since the provided since token - # * the server has zero OTKs left for this device - # Spec issue: https://github.com/matrix-org/matrix-doc/issues/3298 - one_time_keys_count = await self.store.count_e2e_one_time_keys( - user_id, device_id - ) - unused_fallback_key_types = list( - await self.store.get_e2e_unused_fallback_key_types(user_id, device_id) - ) - - return E2eeSyncResult( - to_device=sync_result_builder.to_device, - device_lists=device_lists, - device_one_time_keys_count=one_time_keys_count, - device_unused_fallback_key_types=unused_fallback_key_types, - next_batch=sync_result_builder.now_token, - ) - async def get_sync_result_builder( self, sync_config: SyncConfig, @@ -2549,6 +2309,7 @@ class SyncHandler: room_entries: List[RoomSyncResultBuilder] = [] invited: List[InvitedSyncResult] = [] knocked: List[KnockedSyncResult] = [] + invite_config = await self.store.get_invite_config_for_user(user_id) for room_id, events in mem_change_events_by_room_id.items(): # The body of this loop will add this room to at least one of the five lists # above. Things get messy if you've e.g. joined, left, joined then left the @@ -2631,7 +2392,11 @@ class SyncHandler: # Only bother if we're still currently invited should_invite = last_non_join.membership == Membership.INVITE if should_invite: - if last_non_join.sender not in ignored_users: + if ( + last_non_join.sender not in ignored_users + and invite_config.get_invite_rule(last_non_join.sender) + != InviteRule.IGNORE + ): invite_room_sync = InvitedSyncResult(room_id, invite=last_non_join) if invite_room_sync: invited.append(invite_room_sync) @@ -2786,6 +2551,7 @@ class SyncHandler: membership_list=Membership.LIST, excluded_rooms=sync_result_builder.excluded_room_ids, ) + invite_config = await self.store.get_invite_config_for_user(user_id) room_entries = [] invited = [] @@ -2811,6 +2577,8 @@ class SyncHandler: elif event.membership == Membership.INVITE: if event.sender in ignored_users: continue + if invite_config.get_invite_rule(event.sender) == InviteRule.IGNORE: + continue invite = await self.store.get_event(event.event_id) invited.append(InvitedSyncResult(room_id=event.room_id, invite=invite)) elif event.membership == Membership.KNOCK: @@ -3065,8 +2833,10 @@ class SyncHandler: if batch.limited and since_token: user_id = sync_result_builder.sync_config.user.to_string() logger.debug( - "Incremental gappy sync of %s for user %s with %d state events" - % (room_id, user_id, len(state)) + "Incremental gappy sync of %s for user %s with %d state events", + room_id, + user_id, + len(state), ) elif room_builder.rtype == "archived": archived_room_sync = ArchivedSyncResult( diff --git a/synapse/handlers/thread_subscriptions.py b/synapse/handlers/thread_subscriptions.py new file mode 100644 index 0000000000..d56c915e0a --- /dev/null +++ b/synapse/handlers/thread_subscriptions.py @@ -0,0 +1,190 @@ +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING, Optional + +from synapse.api.constants import RelationTypes +from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError +from synapse.events import relation_from_event +from synapse.storage.databases.main.thread_subscriptions import ( + AutomaticSubscriptionConflicted, + ThreadSubscription, +) +from synapse.types import EventOrderings, StreamKeyType, UserID + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ThreadSubscriptionsHandler: + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + self.event_handler = hs.get_event_handler() + self.auth = hs.get_auth() + self._notifier = hs.get_notifier() + + async def get_thread_subscription_settings( + self, + user_id: UserID, + room_id: str, + thread_root_event_id: str, + ) -> Optional[ThreadSubscription]: + """Get thread subscription settings for a specific thread and user. + Checks that the thread root is both a real event and also that it is visible + to the user. + + Args: + user_id: The ID of the user + thread_root_event_id: The event ID of the thread root + + Returns: + A `ThreadSubscription` containing the active subscription settings or None if not set + """ + # First check that the user can access the thread root event + # and that it exists + try: + event = await self.event_handler.get_event( + user_id, room_id, thread_root_event_id + ) + if event is None: + raise NotFoundError("No such thread root") + except AuthError: + raise NotFoundError("No such thread root") + + return await self.store.get_subscription_for_thread( + user_id.to_string(), event.room_id, thread_root_event_id + ) + + async def subscribe_user_to_thread( + self, + user_id: UserID, + room_id: str, + thread_root_event_id: str, + *, + automatic_event_id: Optional[str], + ) -> Optional[int]: + """Sets or updates a user's subscription settings for a specific thread root. + + Args: + requester_user_id: The ID of the user whose settings are being updated. + thread_root_event_id: The event ID of the thread root. + automatic_event_id: if the user was subscribed by an automatic decision by + their client, the event ID that caused this. + + Returns: + The stream ID for this update, if the update isn't no-opped. + + Raises: + NotFoundError if the user cannot access the thread root event, or it isn't + known to this homeserver. Ditto for the automatic cause event if supplied. + + SynapseError(400, M_NOT_IN_THREAD): if client supplied an automatic cause event + but user cannot access the event. + + SynapseError(409, M_SKIPPED): if client requested an automatic subscription + but it was skipped because the cause event is logically later than an unsubscription. + """ + # First check that the user can access the thread root event + # and that it exists + try: + thread_root_event = await self.event_handler.get_event( + user_id, room_id, thread_root_event_id + ) + if thread_root_event is None: + raise NotFoundError("No such thread root") + except AuthError: + logger.info("rejecting thread subscriptions change (thread not accessible)") + raise NotFoundError("No such thread root") + + if automatic_event_id: + autosub_cause_event = await self.event_handler.get_event( + user_id, room_id, automatic_event_id + ) + if autosub_cause_event is None: + raise NotFoundError("Automatic subscription event not found") + relation = relation_from_event(autosub_cause_event) + if ( + relation is None + or relation.rel_type != RelationTypes.THREAD + or relation.parent_id != thread_root_event_id + ): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Automatic subscription must use an event in the thread", + errcode=Codes.MSC4306_NOT_IN_THREAD, + ) + + automatic_event_orderings = EventOrderings.from_event(autosub_cause_event) + else: + automatic_event_orderings = None + + outcome = await self.store.subscribe_user_to_thread( + user_id.to_string(), + room_id, + thread_root_event_id, + automatic_event_orderings=automatic_event_orderings, + ) + + if isinstance(outcome, AutomaticSubscriptionConflicted): + raise SynapseError( + HTTPStatus.CONFLICT, + "Automatic subscription obsoleted by an unsubscription request.", + errcode=Codes.MSC4306_CONFLICTING_UNSUBSCRIPTION, + ) + + if outcome is not None: + # wake up user streams (e.g. sliding sync) on the same worker + self._notifier.on_new_event( + StreamKeyType.THREAD_SUBSCRIPTIONS, + # outcome is a stream_id + outcome, + users=[user_id.to_string()], + ) + + return outcome + + async def unsubscribe_user_from_thread( + self, user_id: UserID, room_id: str, thread_root_event_id: str + ) -> Optional[int]: + """Clears a user's subscription settings for a specific thread root. + + Args: + requester_user_id: The ID of the user whose settings are being updated. + thread_root_event_id: The event ID of the thread root. + + Returns: + The stream ID for this update, if the update isn't no-opped. + + Raises: + NotFoundError if the user cannot access the thread root event, or it isn't + known to this homeserver. + """ + # First check that the user can access the thread root event + # and that it exists + try: + event = await self.event_handler.get_event( + user_id, room_id, thread_root_event_id + ) + if event is None: + raise NotFoundError("No such thread root") + except AuthError: + logger.info("rejecting thread subscriptions change (thread not accessible)") + raise NotFoundError("No such thread root") + + outcome = await self.store.unsubscribe_user_from_thread( + user_id.to_string(), + event.room_id, + thread_root_event_id, + ) + + if outcome is not None: + # wake up user streams (e.g. sliding sync) on the same worker + self._notifier.on_new_event( + StreamKeyType.THREAD_SUBSCRIPTIONS, + # outcome is a stream_id + outcome, + users=[user_id.to_string()], + ) + + return outcome diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 8d693fee30..6a7b36ea0c 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -80,7 +80,9 @@ class FollowerTypingHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() - self.server_name = hs.config.server.server_name + self.server_name = ( + hs.hostname + ) # nb must be called this for @wrap_as_background_process self.clock = hs.get_clock() self.is_mine_id = hs.is_mine_id self.is_mine_server_name = hs.is_mine_server_name @@ -143,7 +145,11 @@ class FollowerTypingHandler: last_fed_poke = self._member_last_federation_poke.get(member, None) if not last_fed_poke or last_fed_poke + FEDERATION_PING_INTERVAL <= now: run_as_background_process( - "typing._push_remote", self._push_remote, member=member, typing=True + "typing._push_remote", + self.server_name, + self._push_remote, + member=member, + typing=True, ) # Add a paranoia timer to ensure that we always have a timer for @@ -216,6 +222,7 @@ class FollowerTypingHandler: if self.federation: run_as_background_process( "_send_changes_in_typing_to_remotes", + self.server_name, self._send_changes_in_typing_to_remotes, row.room_id, prev_typing, @@ -263,6 +270,7 @@ class TypingWriterHandler(FollowerTypingHandler): assert hs.get_instance_name() in hs.config.worker.writers.typing + self.server_name = hs.hostname self.auth = hs.get_auth() self.notifier = hs.get_notifier() self.event_auth_handler = hs.get_event_auth_handler() @@ -280,7 +288,9 @@ class TypingWriterHandler(FollowerTypingHandler): # caches which room_ids changed at which serials self._typing_stream_change_cache = StreamChangeCache( - "TypingStreamChangeCache", self._latest_room_serial + name="TypingStreamChangeCache", + server_name=self.server_name, + current_stream_pos=self._latest_room_serial, ) def _handle_timeout_for_member(self, now: int, member: RoomMember) -> None: @@ -375,7 +385,11 @@ class TypingWriterHandler(FollowerTypingHandler): if self.hs.is_mine_id(member.user_id): # Only send updates for changes to our own users. run_as_background_process( - "typing._push_remote", self._push_remote, member, typing + "typing._push_remote", + self.server_name, + self._push_remote, + member, + typing, ) self._push_update_local(member=member, typing=typing) @@ -503,6 +517,7 @@ class TypingWriterHandler(FollowerTypingHandler): class TypingNotificationEventSource(EventSource[int, JsonMapping]): def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._main_store = hs.get_datastores().main self.clock = hs.get_clock() # We can't call get_typing_handler here because there's a cycle: @@ -535,7 +550,9 @@ class TypingNotificationEventSource(EventSource[int, JsonMapping]): appservice may be interested in. * The latest known room serial. """ - with Measure(self.clock, "typing.get_new_events_as"): + with Measure( + self.clock, name="typing.get_new_events_as", server_name=self.server_name + ): handler = self.get_typing_handler() events = [] @@ -571,7 +588,9 @@ class TypingNotificationEventSource(EventSource[int, JsonMapping]): Find typing notifications for given rooms (> `from_token` and <= `to_token`) """ - with Measure(self.clock, "typing.get_new_events"): + with Measure( + self.clock, name="typing.get_new_events", server_name=self.server_name + ): from_key = int(from_key) handler = self.get_typing_handler() diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index a343637b82..130099a239 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -26,9 +26,16 @@ from typing import TYPE_CHECKING, List, Optional, Set, Tuple from twisted.internet.interfaces import IDelayedCall import synapse.metrics -from synapse.api.constants import EventTypes, HistoryVisibility, JoinRules, Membership +from synapse.api.constants import ( + EventTypes, + HistoryVisibility, + JoinRules, + Membership, + ProfileFields, +) from synapse.api.errors import Codes, SynapseError from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.databases.main.user_directory import SearchResult @@ -102,6 +109,9 @@ class UserDirectoryHandler(StateDeltasHandler): self.is_mine_id = hs.is_mine_id self.update_user_directory = hs.config.worker.should_update_user_directory self.search_all_users = hs.config.userdirectory.user_directory_search_all_users + self.exclude_remote_users = ( + hs.config.userdirectory.user_directory_exclude_remote_users + ) self.show_locked_users = hs.config.userdirectory.show_locked_users self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self._hs = hs @@ -161,7 +171,7 @@ class UserDirectoryHandler(StateDeltasHandler): non_spammy_users = [] for user in results["results"]: if not await self._spam_checker_module_callbacks.check_username_for_spam( - user + user, user_id ): non_spammy_users.append(user) results["results"] = non_spammy_users @@ -183,7 +193,9 @@ class UserDirectoryHandler(StateDeltasHandler): self._is_processing = False self._is_processing = True - run_as_background_process("user_directory.notify_new_event", process) + run_as_background_process( + "user_directory.notify_new_event", self.server_name, process + ) async def handle_local_profile_change( self, user_id: str, profile: ProfileInfo @@ -228,7 +240,9 @@ class UserDirectoryHandler(StateDeltasHandler): # Loop round handling deltas until we're up to date while True: - with Measure(self.clock, "user_dir_delta"): + with Measure( + self.clock, name="user_dir_delta", server_name=self.server_name + ): room_max_stream_ordering = self.store.get_room_max_stream_ordering() if self.pos == room_max_stream_ordering: return @@ -249,9 +263,9 @@ class UserDirectoryHandler(StateDeltasHandler): self.pos = max_pos # Expose current event processing position to prometheus - synapse.metrics.event_processing_positions.labels("user_dir").set( - max_pos - ) + synapse.metrics.event_processing_positions.labels( + name="user_dir", **{SERVER_NAME_LABEL: self.server_name} + ).set(max_pos) await self.store.update_user_directory_stream_pos(max_pos) @@ -595,7 +609,9 @@ class UserDirectoryHandler(StateDeltasHandler): self._is_refreshing_remote_profiles = False self._is_refreshing_remote_profiles = True - run_as_background_process("user_directory.refresh_remote_profiles", process) + run_as_background_process( + "user_directory.refresh_remote_profiles", self.server_name, process + ) async def _unsafe_refresh_remote_profiles(self) -> None: limit = MAX_SERVERS_TO_REFRESH_PROFILES_FOR_IN_ONE_GO - len( @@ -677,7 +693,9 @@ class UserDirectoryHandler(StateDeltasHandler): self._is_refreshing_remote_profiles_for_servers.add(server_name) run_as_background_process( - "user_directory.refresh_remote_profiles_for_remote_server", process + "user_directory.refresh_remote_profiles_for_remote_server", + self.server_name, + process, ) async def _unsafe_refresh_remote_profiles_for_remote_server( @@ -740,10 +758,9 @@ class UserDirectoryHandler(StateDeltasHandler): ) continue except Exception: - logger.error( + logger.exception( "Failed to refresh profile for %r due to unhandled exception", user_id, - exc_info=True, ) await self.store.set_remote_user_profile_in_user_dir_stale( user_id, @@ -756,6 +773,10 @@ class UserDirectoryHandler(StateDeltasHandler): await self.store.update_profile_in_user_dir( user_id, - display_name=non_null_str_or_none(profile.get("displayname")), - avatar_url=non_null_str_or_none(profile.get("avatar_url")), + display_name=non_null_str_or_none( + profile.get(ProfileFields.DISPLAYNAME) + ), + avatar_url=non_null_str_or_none( + profile.get(ProfileFields.AVATAR_URL) + ), ) diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index db998f6701..0b375790dd 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -19,6 +19,7 @@ # # +import logging import random from types import TracebackType from typing import ( @@ -43,12 +44,15 @@ from synapse.logging.opentracing import start_active_span from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage.databases.main.lock import Lock, LockStore from synapse.util.async_helpers import timeout_deferred +from synapse.util.constants import ONE_MINUTE_SECONDS if TYPE_CHECKING: from synapse.logging.opentracing import opentracing from synapse.server import HomeServer +logger = logging.getLogger(__name__) + # This lock is used to avoid creating an event while we are purging the room. # We take a read lock when creating an event, and a write one when purging a room. # This is because it is fine to create several events concurrently, since referenced events @@ -62,6 +66,9 @@ class WorkerLocksHandler: """ def __init__(self, hs: "HomeServer") -> None: + self.server_name = ( + hs.hostname + ) # nb must be called this for @wrap_as_background_process self._reactor = hs.get_reactor() self._store = hs.get_datastores().main self._clock = hs.get_clock() @@ -269,6 +276,11 @@ class WaitingLock: def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) + if self._retry_interval > 10 * ONE_MINUTE_SECONDS: # >7 iterations + logger.warning( + "Lock timeout is getting excessive: %ss. There may be a deadlock.", + self._retry_interval, + ) return next * random.uniform(0.9, 1.1) @@ -344,4 +356,9 @@ class WaitingMultiLock: def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) + if self._retry_interval > 10 * ONE_MINUTE_SECONDS: # >7 iterations + logger.warning( + "Lock timeout is getting excessive: %ss. There may be a deadlock.", + self._retry_interval, + ) return next * random.uniform(0.9, 1.1) diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py index 2b9830b540..59eae841d5 100644 --- a/synapse/http/additional_resource.py +++ b/synapse/http/additional_resource.py @@ -53,7 +53,7 @@ class AdditionalResource(DirectServeJsonResource): hs: homeserver handler: function to be called to handle the request. """ - super().__init__() + super().__init__(clock=hs.get_clock()) self._handler = handler async def _async_render(self, request: Request) -> Optional[Tuple[int, Any]]: diff --git a/synapse/http/client.py b/synapse/http/client.py index 85923d956b..1f6d4dcd86 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -31,6 +31,7 @@ from typing import ( List, Mapping, Optional, + Protocol, Tuple, Union, ) @@ -40,8 +41,7 @@ import treq from canonicaljson import encode_canonical_json from netaddr import AddrFormatError, IPAddress, IPSet from prometheus_client import Counter -from typing_extensions import Protocol -from zope.interface import implementer, provider +from zope.interface import implementer from OpenSSL import SSL from OpenSSL.SSL import VERIFY_NONE @@ -85,6 +85,7 @@ from synapse.http.replicationagent import ReplicationAgent from synapse.http.types import QueryParams from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import set_tag, start_active_span, tags +from synapse.metrics import SERVER_NAME_LABEL from synapse.types import ISynapseReactor, StrSequence from synapse.util import json_decoder from synapse.util.async_helpers import timeout_deferred @@ -108,9 +109,13 @@ except ImportError: logger = logging.getLogger(__name__) -outgoing_requests_counter = Counter("synapse_http_client_requests", "", ["method"]) +outgoing_requests_counter = Counter( + "synapse_http_client_requests", "", labelnames=["method", SERVER_NAME_LABEL] +) incoming_responses_counter = Counter( - "synapse_http_client_responses", "", ["method", "code"] + "synapse_http_client_responses", + "", + labelnames=["method", "code", SERVER_NAME_LABEL], ) # the type of the headers map, to be passed to the t.w.h.Headers. @@ -213,7 +218,7 @@ class _IPBlockingResolver: if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist): logger.info( - "Blocked %s from DNS resolution to %s" % (ip_address, hostname) + "Blocked %s from DNS resolution to %s", ip_address, hostname ) has_bad_ip = True @@ -225,7 +230,7 @@ class _IPBlockingResolver: recv.addressResolved(address) recv.resolutionComplete() - @provider(IResolutionReceiver) + @implementer(IResolutionReceiver) class EndpointReceiver: @staticmethod def resolutionBegan(resolutionInProgress: IHostResolution) -> None: @@ -239,8 +244,9 @@ class _IPBlockingResolver: def resolutionComplete() -> None: _callback() + endpoint_receiver_wrapper = EndpointReceiver() self._reactor.nameResolver.resolveHostName( - EndpointReceiver, hostname, portNumber=portNumber + endpoint_receiver_wrapper, hostname, portNumber=portNumber ) return recv @@ -317,7 +323,7 @@ class BlocklistingAgentWrapper(Agent): pass else: if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist): - logger.info("Blocking access to %s" % (ip_address,)) + logger.info("Blocking access to %s", ip_address) e = SynapseError(HTTPStatus.FORBIDDEN, "IP address blocked") return defer.fail(Failure(e)) @@ -345,6 +351,7 @@ class BaseHttpClient: treq_args: Optional[Dict[str, Any]] = None, ): self.hs = hs + self.server_name = hs.hostname self.reactor = hs.get_reactor() self._extra_treq_args = treq_args or {} @@ -383,7 +390,9 @@ class BaseHttpClient: RequestTimedOutError if the request times out before the headers are read """ - outgoing_requests_counter.labels(method).inc() + outgoing_requests_counter.labels( + method=method, **{SERVER_NAME_LABEL: self.server_name} + ).inc() # log request but strip `access_token` (AS requests for example include this) logger.debug("Sending request %s %s", method, redact_uri(uri)) @@ -437,7 +446,11 @@ class BaseHttpClient: response = await make_deferred_yieldable(request_deferred) - incoming_responses_counter.labels(method, response.code).inc() + incoming_responses_counter.labels( + method=method, + code=response.code, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() logger.info( "Received response to %s %s: %s", method, @@ -446,7 +459,11 @@ class BaseHttpClient: ) return response except Exception as e: - incoming_responses_counter.labels(method, "ERR").inc() + incoming_responses_counter.labels( + method=method, + code="ERR", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() logger.info( "Error sending request to %s %s: %s %s", method, @@ -722,7 +739,7 @@ class BaseHttpClient: resp_headers = dict(response.headers.getAllRawHeaders()) if response.code > 299: - logger.warning("Got %d when downloading %s" % (response.code, url)) + logger.warning("Got %d when downloading %s", response.code, url) raise SynapseError( HTTPStatus.BAD_GATEWAY, "Got error %d" % (response.code,), Codes.UNKNOWN ) @@ -820,12 +837,12 @@ class SimpleHttpClient(BaseHttpClient): pool.cachedConnectionTimeout = 2 * 60 self.agent: IAgent = ProxyAgent( - self.reactor, - hs.get_reactor(), + reactor=self.reactor, + proxy_reactor=hs.get_reactor(), connectTimeout=15, contextFactory=self.hs.get_http_client_context_factory(), pool=pool, - use_proxy=use_proxy, + proxy_config=hs.config.server.proxy_config, ) if self._ip_blocklist: @@ -854,6 +871,7 @@ class ReplicationClient(BaseHttpClient): hs: The HomeServer instance to pass in """ super().__init__(hs) + self.server_name = hs.hostname # Use a pool, but a very small one. pool = HTTPConnectionPool(self.reactor) @@ -890,7 +908,9 @@ class ReplicationClient(BaseHttpClient): RequestTimedOutError if the request times out before the headers are read """ - outgoing_requests_counter.labels(method).inc() + outgoing_requests_counter.labels( + method=method, **{SERVER_NAME_LABEL: self.server_name} + ).inc() logger.debug("Sending request %s %s", method, uri) @@ -947,7 +967,11 @@ class ReplicationClient(BaseHttpClient): response = await make_deferred_yieldable(request_deferred) - incoming_responses_counter.labels(method, response.code).inc() + incoming_responses_counter.labels( + method=method, + code=response.code, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() logger.info( "Received response to %s %s: %s", method, @@ -956,7 +980,11 @@ class ReplicationClient(BaseHttpClient): ) return response except Exception as e: - incoming_responses_counter.labels(method, "ERR").inc() + incoming_responses_counter.labels( + method=method, + code="ERR", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() logger.info( "Error sending request to %s %s: %s %s", method, @@ -1105,7 +1133,7 @@ class _MultipartParserProtocol(protocol.Protocol): self.stream.write(data[start:end]) except Exception as e: logger.warning( - f"Exception encountered writing file data to stream: {e}" + "Exception encountered writing file data to stream: %s", e ) self.deferred.errback() self.file_length += end - start @@ -1128,7 +1156,7 @@ class _MultipartParserProtocol(protocol.Protocol): try: self.parser.write(incoming_data) except Exception as e: - logger.warning(f"Exception writing to multipart parser: {e}") + logger.warning("Exception writing to multipart parser: %s", e) self.deferred.errback() return diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py index 4e4d78cb88..db803bc75a 100644 --- a/synapse/http/connectproxyclient.py +++ b/synapse/http/connectproxyclient.py @@ -33,10 +33,11 @@ from twisted.internet.interfaces import ( IAddress, IConnector, IProtocol, + IProtocolFactory, IReactorCore, IStreamClientEndpoint, ) -from twisted.internet.protocol import ClientFactory, Protocol, connectionDone +from twisted.internet.protocol import ClientFactory, connectionDone from twisted.python.failure import Failure from twisted.web import http @@ -116,11 +117,7 @@ class HTTPConnectProxyEndpoint: def __repr__(self) -> str: return "" % (self._proxy_endpoint,) - # Mypy encounters a false positive here: it complains that ClientFactory - # is incompatible with IProtocolFactory. But ClientFactory inherits from - # Factory, which implements IProtocolFactory. So I think this is a bug - # in mypy-zope. - def connect(self, protocolFactory: ClientFactory) -> "defer.Deferred[IProtocol]": # type: ignore[override] + def connect(self, protocolFactory: IProtocolFactory) -> "defer.Deferred[IProtocol]": f = HTTPProxiedClientFactory( self._host, self._port, protocolFactory, self._proxy_creds ) @@ -148,7 +145,7 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): self, dst_host: bytes, dst_port: int, - wrapped_factory: ClientFactory, + wrapped_factory: IProtocolFactory, proxy_creds: Optional[ProxyCredentials], ): self.dst_host = dst_host @@ -158,7 +155,10 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): self.on_connection: "defer.Deferred[None]" = defer.Deferred() def startedConnecting(self, connector: IConnector) -> None: - return self.wrapped_factory.startedConnecting(connector) + # We expect the wrapped factory to be a ClientFactory, but the generic + # interfaces only guarantee that it implements IProtocolFactory. + if isinstance(self.wrapped_factory, ClientFactory): + return self.wrapped_factory.startedConnecting(connector) def buildProtocol(self, addr: IAddress) -> "HTTPConnectProtocol": wrapped_protocol = self.wrapped_factory.buildProtocol(addr) @@ -177,13 +177,15 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): logger.debug("Connection to proxy failed: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) - return self.wrapped_factory.clientConnectionFailed(connector, reason) + if isinstance(self.wrapped_factory, ClientFactory): + return self.wrapped_factory.clientConnectionFailed(connector, reason) def clientConnectionLost(self, connector: IConnector, reason: Failure) -> None: logger.debug("Connection to proxy lost: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) - return self.wrapped_factory.clientConnectionLost(connector, reason) + if isinstance(self.wrapped_factory, ClientFactory): + return self.wrapped_factory.clientConnectionLost(connector, reason) class HTTPConnectProtocol(protocol.Protocol): @@ -208,7 +210,7 @@ class HTTPConnectProtocol(protocol.Protocol): self, host: bytes, port: int, - wrapped_protocol: Protocol, + wrapped_protocol: IProtocol, connected_deferred: defer.Deferred, proxy_creds: Optional[ProxyCredentials], ): @@ -223,11 +225,14 @@ class HTTPConnectProtocol(protocol.Protocol): ) self.http_setup_client.on_connected.addCallback(self.proxyConnected) + # Set once we start connecting to the wrapped protocol + self.wrapped_connection_started = False + def connectionMade(self) -> None: self.http_setup_client.makeConnection(self.transport) def connectionLost(self, reason: Failure = connectionDone) -> None: - if self.wrapped_protocol.connected: + if self.wrapped_connection_started: self.wrapped_protocol.connectionLost(reason) self.http_setup_client.connectionLost(reason) @@ -236,6 +241,8 @@ class HTTPConnectProtocol(protocol.Protocol): self.connected_deferred.errback(reason) def proxyConnected(self, _: Union[None, "defer.Deferred[None]"]) -> None: + self.wrapped_connection_started = True + assert self.transport is not None self.wrapped_protocol.makeConnection(self.transport) self.connected_deferred.callback(self.wrapped_protocol) @@ -247,7 +254,7 @@ class HTTPConnectProtocol(protocol.Protocol): def dataReceived(self, data: bytes) -> None: # if we've set up the HTTP protocol, we can send the data there - if self.wrapped_protocol.connected: + if self.wrapped_connection_started: return self.wrapped_protocol.dataReceived(data) # otherwise, we must still be setting up the connection: send the data to the diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index a7742fcea8..6ebadf0dbf 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -21,7 +21,6 @@ import logging import urllib.parse from typing import Any, Generator, List, Optional from urllib.request import ( # type: ignore[attr-defined] - getproxies_environment, proxy_bypass_environment, ) @@ -40,6 +39,7 @@ from twisted.web.client import URI, Agent, HTTPConnectionPool from twisted.web.http_headers import Headers from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer, IResponse +from synapse.config.server import ProxyConfig from synapse.crypto.context_factory import FederationPolicyForHTTPS from synapse.http import proxyagent from synapse.http.client import BlocklistingAgentWrapper, BlocklistingReactorWrapper @@ -77,6 +77,8 @@ class MatrixFederationAgent: ip_blocklist: Disallowed IP addresses. + proxy_config: Proxy configuration to use for this agent. + proxy_reactor: twisted reactor to use for connections to the proxy server reactor might have some blocking applied (i.e. for DNS queries), but we need unblocked access to the proxy. @@ -92,14 +94,29 @@ class MatrixFederationAgent: def __init__( self, + *, + server_name: str, reactor: ISynapseReactor, tls_client_options_factory: Optional[FederationPolicyForHTTPS], user_agent: bytes, ip_allowlist: Optional[IPSet], ip_blocklist: IPSet, + proxy_config: Optional[ProxyConfig] = None, _srv_resolver: Optional[SrvResolver] = None, _well_known_resolver: Optional[WellKnownResolver] = None, ): + """ + Args: + server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + reactor + tls_client_options_factory + user_agent + ip_allowlist + ip_blocklist + _srv_resolver + _well_known_resolver + """ + # proxy_reactor is not blocklisting reactor proxy_reactor = reactor @@ -116,10 +133,11 @@ class MatrixFederationAgent: self._agent = Agent.usingEndpointFactory( reactor, MatrixHostnameEndpointFactory( - reactor, - proxy_reactor, - tls_client_options_factory, - _srv_resolver, + reactor=reactor, + proxy_reactor=proxy_reactor, + tls_client_options_factory=tls_client_options_factory, + srv_resolver=_srv_resolver, + proxy_config=proxy_config, ), pool=self._pool, ) @@ -127,14 +145,15 @@ class MatrixFederationAgent: if _well_known_resolver is None: _well_known_resolver = WellKnownResolver( - reactor, + server_name=server_name, + reactor=reactor, agent=BlocklistingAgentWrapper( ProxyAgent( - reactor, - proxy_reactor, + reactor=reactor, + proxy_reactor=proxy_reactor, pool=self._pool, contextFactory=tls_client_options_factory, - use_proxy=True, + proxy_config=proxy_config, ), ip_blocklist=ip_blocklist, ), @@ -232,14 +251,17 @@ class MatrixHostnameEndpointFactory: def __init__( self, + *, reactor: IReactorCore, proxy_reactor: IReactorCore, tls_client_options_factory: Optional[FederationPolicyForHTTPS], srv_resolver: Optional[SrvResolver], + proxy_config: Optional[ProxyConfig], ): self._reactor = reactor self._proxy_reactor = proxy_reactor self._tls_client_options_factory = tls_client_options_factory + self._proxy_config = proxy_config if srv_resolver is None: srv_resolver = SrvResolver() @@ -248,11 +270,12 @@ class MatrixHostnameEndpointFactory: def endpointForURI(self, parsed_uri: URI) -> "MatrixHostnameEndpoint": return MatrixHostnameEndpoint( - self._reactor, - self._proxy_reactor, - self._tls_client_options_factory, - self._srv_resolver, - parsed_uri, + reactor=self._reactor, + proxy_reactor=self._proxy_reactor, + tls_client_options_factory=self._tls_client_options_factory, + srv_resolver=self._srv_resolver, + proxy_config=self._proxy_config, + parsed_uri=parsed_uri, ) @@ -269,6 +292,7 @@ class MatrixHostnameEndpoint: tls_client_options_factory: factory to use for fetching client tls options, or none to disable TLS. srv_resolver: The SRV resolver to use + proxy_config: Proxy configuration to use for this agent. parsed_uri: The parsed URI that we're wanting to connect to. Raises: @@ -278,26 +302,28 @@ class MatrixHostnameEndpoint: def __init__( self, + *, reactor: IReactorCore, proxy_reactor: IReactorCore, tls_client_options_factory: Optional[FederationPolicyForHTTPS], srv_resolver: SrvResolver, + proxy_config: Optional[ProxyConfig], parsed_uri: URI, ): self._reactor = reactor self._parsed_uri = parsed_uri + self.proxy_config = proxy_config # http_proxy is not needed because federation is always over TLS - proxies = getproxies_environment() - https_proxy = proxies["https"].encode() if "https" in proxies else None - self.no_proxy = proxies["no"] if "no" in proxies else None # endpoint and credentials to use to connect to the outbound https proxy, if any. ( self._https_proxy_endpoint, self._https_proxy_creds, ) = proxyagent.http_proxy_endpoint( - https_proxy, + self.proxy_config.https_proxy.encode() + if self.proxy_config and self.proxy_config.https_proxy + else None, proxy_reactor, tls_client_options_factory, ) @@ -334,10 +360,10 @@ class MatrixHostnameEndpoint: port = server.port should_skip_proxy = False - if self.no_proxy is not None: + if self.proxy_config is not None: should_skip_proxy = proxy_bypass_environment( host.decode(), - proxies={"no": self.no_proxy}, + proxies=self.proxy_config.get_proxies_dictionary(), ) endpoint: IStreamClientEndpoint diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 9a6bac7281..70242ad0ae 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -77,10 +77,6 @@ WELL_KNOWN_RETRY_ATTEMPTS = 3 logger = logging.getLogger(__name__) -_well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache("well-known") -_had_valid_well_known_cache: TTLCache[bytes, bool] = TTLCache("had-valid-well-known") - - @attr.s(slots=True, frozen=True, auto_attribs=True) class WellKnownLookupResult: delegated_server: Optional[bytes] @@ -91,20 +87,36 @@ class WellKnownResolver: def __init__( self, + server_name: str, reactor: IReactorTime, agent: IAgent, user_agent: bytes, well_known_cache: Optional[TTLCache[bytes, Optional[bytes]]] = None, had_well_known_cache: Optional[TTLCache[bytes, bool]] = None, ): + """ + Args: + server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + reactor + agent + user_agent + well_known_cache + had_well_known_cache + """ + + self.server_name = server_name self._reactor = reactor self._clock = Clock(reactor) if well_known_cache is None: - well_known_cache = _well_known_cache + well_known_cache = TTLCache( + cache_name="well-known", server_name=server_name + ) if had_well_known_cache is None: - had_well_known_cache = _had_valid_well_known_cache + had_well_known_cache = TTLCache( + cache_name="had-valid-well-known", server_name=server_name + ) self._well_known_cache = well_known_cache self._had_valid_well_known_cache = had_well_known_cache @@ -134,7 +146,13 @@ class WellKnownResolver: # TODO: should we linearise so that we don't end up doing two .well-known # requests for the same server in parallel? try: - with Measure(self._clock, "get_well_known"): + with Measure( + self._clock, + name="get_well_known", + # This should be our homeserver where the the code is running (used to + # label metrics) + server_name=self.server_name, + ): result: Optional[bytes] cache_period: float diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index e658c68e23..15f8e147ab 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -34,6 +34,7 @@ from typing import ( Dict, Generic, List, + Literal, Optional, TextIO, Tuple, @@ -48,7 +49,6 @@ import treq from canonicaljson import encode_canonical_json from prometheus_client import Counter from signedjson.sign import sign_json -from typing_extensions import Literal from twisted.internet import defer from twisted.internet.error import DNSLookupError @@ -87,6 +87,7 @@ from synapse.http.types import QueryParams from synapse.logging import opentracing from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import set_tag, start_active_span, tags +from synapse.metrics import SERVER_NAME_LABEL from synapse.types import JsonDict from synapse.util import json_decoder from synapse.util.async_helpers import AwakenableSleeper, Linearizer, timeout_deferred @@ -99,10 +100,14 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) outgoing_requests_counter = Counter( - "synapse_http_matrixfederationclient_requests", "", ["method"] + "synapse_http_matrixfederationclient_requests", + "", + labelnames=["method", SERVER_NAME_LABEL], ) incoming_responses_counter = Counter( - "synapse_http_matrixfederationclient_responses", "", ["method", "code"] + "synapse_http_matrixfederationclient_responses", + "", + labelnames=["method", "code", SERVER_NAME_LABEL], ) @@ -417,17 +422,19 @@ class MatrixFederationHttpClient: if hs.get_instance_name() in outbound_federation_restricted_to: # Talk to federation directly federation_agent: IAgent = MatrixFederationAgent( - self.reactor, - tls_client_options_factory, - user_agent.encode("ascii"), - hs.config.server.federation_ip_range_allowlist, - hs.config.server.federation_ip_range_blocklist, + server_name=self.server_name, + reactor=self.reactor, + tls_client_options_factory=tls_client_options_factory, + user_agent=user_agent.encode("ascii"), + ip_allowlist=hs.config.server.federation_ip_range_allowlist, + ip_blocklist=hs.config.server.federation_ip_range_blocklist, + proxy_config=hs.config.server.proxy_config, ) else: proxy_authorization_secret = hs.config.worker.worker_replication_secret - assert ( - proxy_authorization_secret is not None - ), "`worker_replication_secret` must be set when using `outbound_federation_restricted_to` (used to authenticate requests across workers)" + assert proxy_authorization_secret is not None, ( + "`worker_replication_secret` must be set when using `outbound_federation_restricted_to` (used to authenticate requests across workers)" + ) federation_proxy_credentials = BearerProxyCredentials( proxy_authorization_secret.encode("ascii") ) @@ -436,9 +443,9 @@ class MatrixFederationHttpClient: # locations federation_proxy_locations = outbound_federation_restricted_to.locations federation_agent = ProxyAgent( - self.reactor, - self.reactor, - tls_client_options_factory, + reactor=self.reactor, + proxy_reactor=self.reactor, + contextFactory=tls_client_options_factory, federation_proxy_locations=federation_proxy_locations, federation_proxy_credentials=federation_proxy_credentials, ) @@ -602,7 +609,7 @@ class MatrixFederationHttpClient: try: parse_and_validate_server_name(request.destination) except ValueError: - logger.exception(f"Invalid destination: {request.destination}.") + logger.exception("Invalid destination: %s.", request.destination) raise FederationDeniedError(request.destination) if timeout is not None: @@ -618,9 +625,10 @@ class MatrixFederationHttpClient: raise FederationDeniedError(request.destination) limiter = await synapse.util.retryutils.get_retry_limiter( - request.destination, - self.clock, - self._store, + destination=request.destination, + our_server_name=self.server_name, + clock=self.clock, + store=self._store, backoff_on_404=backoff_on_404, ignore_backoff=ignore_backoff, notifier=self.hs.get_notifier(), @@ -694,10 +702,16 @@ class MatrixFederationHttpClient: _sec_timeout, ) - outgoing_requests_counter.labels(request.method).inc() + outgoing_requests_counter.labels( + method=request.method, **{SERVER_NAME_LABEL: self.server_name} + ).inc() try: - with Measure(self.clock, "outbound_request"): + with Measure( + self.clock, + name="outbound_request", + server_name=self.server_name, + ): # we don't want all the fancy cookie and redirect handling # that treq.request gives: just use the raw Agent. @@ -729,7 +743,9 @@ class MatrixFederationHttpClient: raise RequestSendFailed(e, can_retry=True) from e incoming_responses_counter.labels( - request.method, response.code + method=request.method, + code=response.code, + **{SERVER_NAME_LABEL: self.server_name}, ).inc() set_tag(tags.HTTP_STATUS_CODE, response.code) diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index 5cd990b0d0..9b044f3b0a 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -106,7 +106,7 @@ class ProxyResource(_AsyncResource): isLeaf = True def __init__(self, reactor: ISynapseReactor, hs: "HomeServer"): - super().__init__(True) + super().__init__(hs.get_clock(), True) self.reactor = reactor self.agent = hs.get_federation_http_client().agent diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index f80f67acc6..ab413990c5 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -21,10 +21,9 @@ import logging import random import re -from typing import Any, Collection, Dict, List, Optional, Sequence, Tuple +from typing import Any, Collection, Dict, List, Optional, Sequence, Tuple, Union, cast from urllib.parse import urlparse from urllib.request import ( # type: ignore[attr-defined] - getproxies_environment, proxy_bypass_environment, ) @@ -40,6 +39,7 @@ from twisted.internet.interfaces import ( IProtocol, IProtocolFactory, IReactorCore, + IReactorTime, IStreamClientEndpoint, ) from twisted.python.failure import Failure @@ -53,6 +53,7 @@ from twisted.web.error import SchemeNotSupported from twisted.web.http_headers import Headers from twisted.web.iweb import IAgent, IBodyProducer, IPolicyForHTTPS, IResponse +from synapse.config.server import ProxyConfig from synapse.config.workers import ( InstanceLocationConfig, InstanceTcpLocationConfig, @@ -98,8 +99,7 @@ class ProxyAgent(_AgentBase): pool: connection pool to be used. If None, a non-persistent pool instance will be created. - use_proxy: Whether proxy settings should be discovered and used - from conventional environment variables. + proxy_config: Proxy configuration to use for this agent. federation_proxy_locations: An optional list of locations to proxy outbound federation traffic through (only requests that use the `matrix-federation://` scheme @@ -117,19 +117,22 @@ class ProxyAgent(_AgentBase): def __init__( self, + *, reactor: IReactorCore, proxy_reactor: Optional[IReactorCore] = None, contextFactory: Optional[IPolicyForHTTPS] = None, connectTimeout: Optional[float] = None, bindAddress: Optional[bytes] = None, pool: Optional[HTTPConnectionPool] = None, - use_proxy: bool = False, + proxy_config: Optional[ProxyConfig] = None, federation_proxy_locations: Collection[InstanceLocationConfig] = (), federation_proxy_credentials: Optional[ProxyCredentials] = None, ): contextFactory = contextFactory or BrowserLikePolicyForHTTPS() - _AgentBase.__init__(self, reactor, pool) + # `_AgentBase` expects an `IReactorTime` provider. `IReactorCore` + # extends `IReactorTime`, so this cast is safe. + _AgentBase.__init__(self, cast(IReactorTime, reactor), pool) if proxy_reactor is None: self.proxy_reactor = reactor @@ -142,34 +145,42 @@ class ProxyAgent(_AgentBase): if bindAddress is not None: self._endpoint_kwargs["bindAddress"] = bindAddress - http_proxy = None - https_proxy = None - no_proxy = None - if use_proxy: - proxies = getproxies_environment() - http_proxy = proxies["http"].encode() if "http" in proxies else None - https_proxy = proxies["https"].encode() if "https" in proxies else None - no_proxy = proxies["no"] if "no" in proxies else None + self.proxy_config = proxy_config + if self.proxy_config is not None: + logger.debug( + "Using proxy settings: http_proxy=%s, https_proxy=%s, no_proxy=%s", + self.proxy_config.http_proxy, + self.proxy_config.https_proxy, + self.proxy_config.no_proxy_hosts, + ) self.http_proxy_endpoint, self.http_proxy_creds = http_proxy_endpoint( - http_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs + self.proxy_config.http_proxy.encode() + if self.proxy_config and self.proxy_config.http_proxy + else None, + self.proxy_reactor, + contextFactory, + **self._endpoint_kwargs, ) self.https_proxy_endpoint, self.https_proxy_creds = http_proxy_endpoint( - https_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs + self.proxy_config.https_proxy.encode() + if self.proxy_config and self.proxy_config.https_proxy + else None, + self.proxy_reactor, + contextFactory, + **self._endpoint_kwargs, ) - self.no_proxy = no_proxy - self._policy_for_https = contextFactory - self._reactor = reactor + self._reactor = cast(IReactorTime, reactor) self._federation_proxy_endpoint: Optional[IStreamClientEndpoint] = None self._federation_proxy_credentials: Optional[ProxyCredentials] = None if federation_proxy_locations: - assert ( - federation_proxy_credentials is not None - ), "`federation_proxy_credentials` are required when using `federation_proxy_locations`" + assert federation_proxy_credentials is not None, ( + "`federation_proxy_credentials` are required when using `federation_proxy_locations`" + ) endpoints: List[IStreamClientEndpoint] = [] for federation_proxy_location in federation_proxy_locations: @@ -251,14 +262,18 @@ class ProxyAgent(_AgentBase): raise ValueError(f"Invalid URI {uri!r}") parsed_uri = URI.fromBytes(uri) - pool_key = f"{parsed_uri.scheme!r}{parsed_uri.host!r}{parsed_uri.port}" + pool_key: tuple[bytes, bytes, int] = ( + parsed_uri.scheme, + parsed_uri.host, + parsed_uri.port, + ) request_path = parsed_uri.originForm should_skip_proxy = False - if self.no_proxy is not None: + if self.proxy_config is not None: should_skip_proxy = proxy_bypass_environment( parsed_uri.host.decode(), - proxies={"no": self.no_proxy}, + proxies=self.proxy_config.get_proxies_dictionary(), ) if ( @@ -277,7 +292,7 @@ class ProxyAgent(_AgentBase): ) # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: - pool_key = "http-proxy" + pool_key = (b"http-proxy", b"", 0) endpoint = self.http_proxy_endpoint request_path = uri elif ( @@ -296,9 +311,9 @@ class ProxyAgent(_AgentBase): parsed_uri.scheme == b"matrix-federation" and self._federation_proxy_endpoint ): - assert ( - self._federation_proxy_credentials is not None - ), "`federation_proxy_credentials` are required when using `federation_proxy_locations`" + assert self._federation_proxy_credentials is not None, ( + "`federation_proxy_credentials` are required when using `federation_proxy_locations`" + ) # Set a Proxy-Authorization header if headers is None: @@ -351,7 +366,9 @@ def http_proxy_endpoint( proxy: Optional[bytes], reactor: IReactorCore, tls_options_factory: Optional[IPolicyForHTTPS], - **kwargs: object, + timeout: float = 30, + bindAddress: Optional[Union[bytes, str, tuple[Union[bytes, str], int]]] = None, + attemptDelay: Optional[float] = None, ) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]: """Parses an http proxy setting and returns an endpoint for the proxy @@ -382,12 +399,15 @@ def http_proxy_endpoint( # 3.9+) on scheme-less proxies, e.g. host:port. scheme, host, port, credentials = parse_proxy(proxy) - proxy_endpoint = HostnameEndpoint(reactor, host, port, **kwargs) + proxy_endpoint = HostnameEndpoint( + reactor, host, port, timeout, bindAddress, attemptDelay + ) if scheme == b"https": if tls_options_factory: tls_options = tls_options_factory.creatorForNetloc(host, port) - proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint) + wrapped_proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint) + return wrapped_proxy_endpoint, credentials else: raise RuntimeError( f"No TLS options for a https connection via proxy {proxy!s}" diff --git a/synapse/http/replicationagent.py b/synapse/http/replicationagent.py index ee8c707062..d70575dbd5 100644 --- a/synapse/http/replicationagent.py +++ b/synapse/http/replicationagent.py @@ -89,7 +89,7 @@ class ReplicationEndpointFactory: location_config.port, ) if scheme == "https": - endpoint = wrapClientTLS( + wrapped_endpoint = wrapClientTLS( # The 'port' argument below isn't actually used by the function self.context_factory.creatorForNetloc( location_config.host.encode("utf-8"), @@ -97,6 +97,8 @@ class ReplicationEndpointFactory: ), endpoint, ) + return wrapped_endpoint + return endpoint elif isinstance(location_config, InstanceUnixLocationConfig): return UNIXClientEndpoint(self.reactor, location_config.path) @@ -178,9 +180,16 @@ class ReplicationAgent(_AgentBase): worker_name = parsedURI.netloc.decode("utf-8") key_scheme = self._endpointFactory.instance_map[worker_name].scheme() key_netloc = self._endpointFactory.instance_map[worker_name].netloc() - # This sets the Pool key to be: - # (http(s), ) or (unix, ) - key = (key_scheme, key_netloc) + # Build a connection pool key. + # + # `_AgentBase` expects this to be a three-tuple of `(scheme, host, + # port)` of type `bytes`. We don't have a real port when connecting via + # a Unix socket, so use `0`. + key = ( + key_scheme.encode("ascii"), + key_netloc.encode("utf-8"), + 0, + ) # _requestWithEndpoint comes from _AgentBase class return self._requestWithEndpoint( diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 366f06eb80..83f52edb7c 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -27,40 +27,52 @@ from typing import Dict, Mapping, Set, Tuple from prometheus_client.core import Counter, Histogram from synapse.logging.context import current_context -from synapse.metrics import LaterGauge +from synapse.metrics import SERVER_NAME_LABEL, LaterGauge logger = logging.getLogger(__name__) # total number of responses served, split by method/servlet/tag response_count = Counter( - "synapse_http_server_response_count", "", ["method", "servlet", "tag"] + "synapse_http_server_response_count", + "", + labelnames=["method", "servlet", "tag", SERVER_NAME_LABEL], ) requests_counter = Counter( - "synapse_http_server_requests_received", "", ["method", "servlet"] + "synapse_http_server_requests_received", + "", + labelnames=["method", "servlet", SERVER_NAME_LABEL], ) outgoing_responses_counter = Counter( - "synapse_http_server_responses", "", ["method", "code"] + "synapse_http_server_responses", + "", + labelnames=["method", "code", SERVER_NAME_LABEL], ) response_timer = Histogram( "synapse_http_server_response_time_seconds", "sec", - ["method", "servlet", "tag", "code"], + labelnames=["method", "servlet", "tag", "code", SERVER_NAME_LABEL], ) response_ru_utime = Counter( - "synapse_http_server_response_ru_utime_seconds", "sec", ["method", "servlet", "tag"] + "synapse_http_server_response_ru_utime_seconds", + "sec", + labelnames=["method", "servlet", "tag", SERVER_NAME_LABEL], ) response_ru_stime = Counter( - "synapse_http_server_response_ru_stime_seconds", "sec", ["method", "servlet", "tag"] + "synapse_http_server_response_ru_stime_seconds", + "sec", + labelnames=["method", "servlet", "tag", SERVER_NAME_LABEL], ) response_db_txn_count = Counter( - "synapse_http_server_response_db_txn_count", "", ["method", "servlet", "tag"] + "synapse_http_server_response_db_txn_count", + "", + labelnames=["method", "servlet", "tag", SERVER_NAME_LABEL], ) # seconds spent waiting for db txns, excluding scheduling time, when processing @@ -68,34 +80,42 @@ response_db_txn_count = Counter( response_db_txn_duration = Counter( "synapse_http_server_response_db_txn_duration_seconds", "", - ["method", "servlet", "tag"], + labelnames=["method", "servlet", "tag", SERVER_NAME_LABEL], ) # seconds spent waiting for a db connection, when processing this request response_db_sched_duration = Counter( "synapse_http_server_response_db_sched_duration_seconds", "", - ["method", "servlet", "tag"], + labelnames=["method", "servlet", "tag", SERVER_NAME_LABEL], ) # size in bytes of the response written response_size = Counter( - "synapse_http_server_response_size", "", ["method", "servlet", "tag"] + "synapse_http_server_response_size", + "", + labelnames=["method", "servlet", "tag", SERVER_NAME_LABEL], ) # In flight metrics are incremented while the requests are in flight, rather # than when the response was written. in_flight_requests_ru_utime = Counter( - "synapse_http_server_in_flight_requests_ru_utime_seconds", "", ["method", "servlet"] + "synapse_http_server_in_flight_requests_ru_utime_seconds", + "", + labelnames=["method", "servlet", SERVER_NAME_LABEL], ) in_flight_requests_ru_stime = Counter( - "synapse_http_server_in_flight_requests_ru_stime_seconds", "", ["method", "servlet"] + "synapse_http_server_in_flight_requests_ru_stime_seconds", + "", + labelnames=["method", "servlet", SERVER_NAME_LABEL], ) in_flight_requests_db_txn_count = Counter( - "synapse_http_server_in_flight_requests_db_txn_count", "", ["method", "servlet"] + "synapse_http_server_in_flight_requests_db_txn_count", + "", + labelnames=["method", "servlet", SERVER_NAME_LABEL], ) # seconds spent waiting for db txns, excluding scheduling time, when processing @@ -103,14 +123,14 @@ in_flight_requests_db_txn_count = Counter( in_flight_requests_db_txn_duration = Counter( "synapse_http_server_in_flight_requests_db_txn_duration_seconds", "", - ["method", "servlet"], + labelnames=["method", "servlet", SERVER_NAME_LABEL], ) # seconds spent waiting for a db connection, when processing this request in_flight_requests_db_sched_duration = Counter( "synapse_http_server_in_flight_requests_db_sched_duration_seconds", "", - ["method", "servlet"], + labelnames=["method", "servlet", SERVER_NAME_LABEL], ) _in_flight_requests: Set["RequestMetrics"] = set() @@ -124,31 +144,44 @@ def _get_in_flight_counts() -> Mapping[Tuple[str, ...], int]: # Cast to a list to prevent it changing while the Prometheus # thread is collecting metrics with _in_flight_requests_lock: - reqs = list(_in_flight_requests) + request_metrics = list(_in_flight_requests) - for rm in reqs: - rm.update_metrics() + for request_metric in request_metrics: + request_metric.update_metrics() # Map from (method, name) -> int, the number of in flight requests of that # type. The key type is Tuple[str, str], but we leave the length unspecified # for compatability with LaterGauge's annotations. counts: Dict[Tuple[str, ...], int] = {} - for rm in reqs: - key = (rm.method, rm.name) + for request_metric in request_metrics: + key = ( + request_metric.method, + request_metric.name, + request_metric.our_server_name, + ) counts[key] = counts.get(key, 0) + 1 return counts -LaterGauge( - "synapse_http_server_in_flight_requests_count", - "", - ["method", "servlet"], - _get_in_flight_counts, +in_flight_requests = LaterGauge( + name="synapse_http_server_in_flight_requests_count", + desc="", + labelnames=["method", "servlet", SERVER_NAME_LABEL], +) +in_flight_requests.register_hook( + homeserver_instance_id=None, hook=_get_in_flight_counts ) class RequestMetrics: + def __init__(self, our_server_name: str) -> None: + """ + Args: + our_server_name: Our homeserver name (used to label metrics) (`hs.hostname`) + """ + self.our_server_name = our_server_name + def start(self, time_sec: float, name: str, method: str) -> None: self.start_ts = time_sec self.start_context = current_context() @@ -194,33 +227,40 @@ class RequestMetrics: response_code_str = str(response_code) - outgoing_responses_counter.labels(self.method, response_code_str).inc() + outgoing_responses_counter.labels( + method=self.method, + code=response_code_str, + **{SERVER_NAME_LABEL: self.our_server_name}, + ).inc() - response_count.labels(self.method, self.name, tag).inc() + response_base_labels = { + "method": self.method, + "servlet": self.name, + "tag": tag, + SERVER_NAME_LABEL: self.our_server_name, + } - response_timer.labels(self.method, self.name, tag, response_code_str).observe( - time_sec - self.start_ts - ) + response_count.labels(**response_base_labels).inc() + + response_timer.labels( + code=response_code_str, + **response_base_labels, + ).observe(time_sec - self.start_ts) resource_usage = context.get_resource_usage() - response_ru_utime.labels(self.method, self.name, tag).inc( - resource_usage.ru_utime - ) - response_ru_stime.labels(self.method, self.name, tag).inc( - resource_usage.ru_stime - ) - response_db_txn_count.labels(self.method, self.name, tag).inc( + response_ru_utime.labels(**response_base_labels).inc(resource_usage.ru_utime) + response_ru_stime.labels(**response_base_labels).inc(resource_usage.ru_stime) + response_db_txn_count.labels(**response_base_labels).inc( resource_usage.db_txn_count ) - response_db_txn_duration.labels(self.method, self.name, tag).inc( + response_db_txn_duration.labels(**response_base_labels).inc( resource_usage.db_txn_duration_sec ) - response_db_sched_duration.labels(self.method, self.name, tag).inc( + response_db_sched_duration.labels(**response_base_labels).inc( resource_usage.db_sched_duration_sec ) - - response_size.labels(self.method, self.name, tag).inc(sent_bytes) + response_size.labels(**response_base_labels).inc(sent_bytes) # We always call this at the end to ensure that we update the metrics # regardless of whether a call to /metrics while the request was in @@ -240,24 +280,30 @@ class RequestMetrics: diff = new_stats - self._request_stats self._request_stats = new_stats + in_flight_labels = { + "method": self.method, + "servlet": self.name, + SERVER_NAME_LABEL: self.our_server_name, + } + # max() is used since rapid use of ru_stime/ru_utime can end up with the # count going backwards due to NTP, time smearing, fine-grained # correction, or floating points. Who knows, really? - in_flight_requests_ru_utime.labels(self.method, self.name).inc( + in_flight_requests_ru_utime.labels(**in_flight_labels).inc( max(diff.ru_utime, 0) ) - in_flight_requests_ru_stime.labels(self.method, self.name).inc( + in_flight_requests_ru_stime.labels(**in_flight_labels).inc( max(diff.ru_stime, 0) ) - in_flight_requests_db_txn_count.labels(self.method, self.name).inc( + in_flight_requests_db_txn_count.labels(**in_flight_labels).inc( diff.db_txn_count ) - in_flight_requests_db_txn_duration.labels(self.method, self.name).inc( + in_flight_requests_db_txn_duration.labels(**in_flight_labels).inc( diff.db_txn_duration_sec ) - in_flight_requests_db_sched_duration.labels(self.method, self.name).inc( + in_flight_requests_db_sched_duration.labels(**in_flight_labels).inc( diff.db_sched_duration_sec ) diff --git a/synapse/http/server.py b/synapse/http/server.py index 792961a147..e395f79894 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -39,18 +39,20 @@ from typing import ( List, Optional, Pattern, + Protocol, Tuple, Union, + cast, ) import attr import jinja2 from canonicaljson import encode_canonical_json -from typing_extensions import Protocol from zope.interface import implementer -from twisted.internet import defer, interfaces +from twisted.internet import defer, interfaces, reactor from twisted.internet.defer import CancelledError +from twisted.internet.interfaces import IReactorTime from twisted.python import failure from twisted.web import resource @@ -67,6 +69,7 @@ from twisted.web.util import redirectTo from synapse.api.errors import ( CodeMessageException, Codes, + LimitExceededError, RedirectException, SynapseError, UnrecognizedRequestError, @@ -74,7 +77,7 @@ from synapse.api.errors import ( from synapse.config.homeserver import HomeServerConfig from synapse.logging.context import defer_to_thread, preserve_fn, run_in_background from synapse.logging.opentracing import active_span, start_active_span, trace_servlet -from synapse.util import json_encoder +from synapse.util import Clock, json_encoder from synapse.util.caches import intern_dict from synapse.util.cancellation import is_function_cancellable from synapse.util.iterutils import chunk_seq @@ -308,9 +311,10 @@ class _AsyncResource(resource.Resource, metaclass=abc.ABCMeta): context from the request the servlet is handling. """ - def __init__(self, extract_context: bool = False): + def __init__(self, clock: Clock, extract_context: bool = False): super().__init__() + self._clock = clock self._extract_context = extract_context def render(self, request: "SynapseRequest") -> int: @@ -329,7 +333,12 @@ class _AsyncResource(resource.Resource, metaclass=abc.ABCMeta): request.request_metrics.name = self.__class__.__name__ with trace_servlet(request, self._extract_context): - callback_return = await self._async_render(request) + try: + callback_return = await self._async_render(request) + except LimitExceededError as e: + if e.pause: + await self._clock.sleep(e.pause) + raise if callback_return is not None: code, response = callback_return @@ -393,8 +402,17 @@ class DirectServeJsonResource(_AsyncResource): formatting responses and errors as JSON. """ - def __init__(self, canonical_json: bool = False, extract_context: bool = False): - super().__init__(extract_context) + def __init__( + self, + canonical_json: bool = False, + extract_context: bool = False, + # Clock is optional as this class is exposed to the module API. + clock: Optional[Clock] = None, + ): + if clock is None: + clock = Clock(cast(IReactorTime, reactor)) + + super().__init__(clock, extract_context) self.canonical_json = canonical_json def _send_response( @@ -450,8 +468,8 @@ class JsonResource(DirectServeJsonResource): canonical_json: bool = True, extract_context: bool = False, ): - super().__init__(canonical_json, extract_context) self.clock = hs.get_clock() + super().__init__(canonical_json, extract_context, clock=self.clock) # Map of path regex -> method -> callback. self._routes: Dict[Pattern[str], Dict[bytes, _PathEntry]] = {} self.hs = hs @@ -497,7 +515,7 @@ class JsonResource(DirectServeJsonResource): key word arguments to pass to the callback """ # At this point the path must be bytes. - request_path_bytes: bytes = request.path # type: ignore + request_path_bytes: bytes = request.path request_path = request_path_bytes.decode("ascii") # Treat HEAD requests as GET requests. request_method = request.method @@ -564,6 +582,17 @@ class DirectServeHtmlResource(_AsyncResource): # The error template to use for this resource ERROR_TEMPLATE = HTML_ERROR_TEMPLATE + def __init__( + self, + extract_context: bool = False, + # Clock is optional as this class is exposed to the module API. + clock: Optional[Clock] = None, + ): + if clock is None: + clock = Clock(cast(IReactorTime, reactor)) + + super().__init__(clock, extract_context) + def _send_response( self, request: "SynapseRequest", @@ -673,6 +702,10 @@ class _ByteProducer: self._request: Optional[Request] = request self._iterator = iterator self._paused = False + self.tracing_scope = start_active_span( + "write_bytes_to_request", + ) + self.tracing_scope.__enter__() try: self._request.registerProducer(self, True) @@ -683,8 +716,8 @@ class _ByteProducer: logger.info("Connection disconnected before response was written: %r", e) # We drop our references to data we'll not use. - self._request = None self._iterator = iter(()) + self.tracing_scope.__exit__(type(e), None, e.__traceback__) else: # Start producing if `registerProducer` was successful self.resumeProducing() @@ -698,6 +731,9 @@ class _ByteProducer: self._request.write(b"".join(data)) def pauseProducing(self) -> None: + opentracing_span = active_span() + if opentracing_span is not None: + opentracing_span.log_kv({"event": "producer_paused"}) self._paused = True def resumeProducing(self) -> None: @@ -708,6 +744,10 @@ class _ByteProducer: self._paused = False + opentracing_span = active_span() + if opentracing_span is not None: + opentracing_span.log_kv({"event": "producer_resumed"}) + # Write until there's backpressure telling us to stop. while not self._paused: # Get the next chunk and write it to the request. @@ -742,6 +782,7 @@ class _ByteProducer: def stopProducing(self) -> None: # Clear a circular reference. self._request = None + self.tracing_scope.__exit__(None, None, None) def _encode_json_bytes(json_object: object) -> bytes: @@ -884,8 +925,9 @@ def _write_bytes_to_request(request: Request, bytes_to_write: bytes) -> None: # once (via `Request.write`) is that doing so starts the timeout for the # next request to be received: so if it takes longer than 60s to stream back # the response to the client, the client never gets it. + # c.f https://github.com/twisted/twisted/issues/12498 # - # The correct solution is to use a Producer; then the timeout is only + # One workaround is to use a `Producer`; then the timeout is only # started once all of the content is sent over the TCP connection. # To make sure we don't write all of the bytes at once we split it up into diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 0330f1c878..69bdce2b83 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -28,6 +28,7 @@ from http import HTTPStatus from typing import ( TYPE_CHECKING, List, + Literal, Mapping, Optional, Sequence, @@ -37,8 +38,6 @@ from typing import ( overload, ) -from typing_extensions import Literal - from twisted.web.server import Request from synapse._pydantic_compat import ( @@ -131,6 +130,16 @@ def parse_integer( return parse_integer_from_args(args, name, default, required, negative) +@overload +def parse_integer_from_args( + args: Mapping[bytes, Sequence[bytes]], + name: str, + default: int, + required: Literal[False] = False, + negative: bool = False, +) -> int: ... + + @overload def parse_integer_from_args( args: Mapping[bytes, Sequence[bytes]], @@ -583,9 +592,9 @@ def parse_enum( is not one of those allowed values. """ # Assert the enum values are strings. - assert all( - isinstance(e.value, str) for e in E - ), "parse_enum only works with string values" + assert all(isinstance(e.value, str) for e in E), ( + "parse_enum only works with string values" + ) str_value = parse_string( request, name, diff --git a/synapse/http/site.py b/synapse/http/site.py index 1cd90cb9b7..55088fc190 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -21,6 +21,7 @@ import contextlib import logging import time +from http import HTTPStatus from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple, Union import attr @@ -43,6 +44,7 @@ from synapse.logging.context import ( LoggingContext, PreserveLoggingContext, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.types import ISynapseReactor, Requester if TYPE_CHECKING: @@ -82,12 +84,14 @@ class SynapseRequest(Request): self, channel: HTTPChannel, site: "SynapseSite", + our_server_name: str, *args: Any, max_request_body_size: int = 1024, request_id_header: Optional[str] = None, **kw: Any, ): super().__init__(channel, *args, **kw) + self.our_server_name = our_server_name self._max_request_body_size = max_request_body_size self.request_id_header = request_id_header self.synapse_site = site @@ -139,6 +143,41 @@ class SynapseRequest(Request): self.synapse_site.site_tag, ) + # Twisted machinery: this method is called by the Channel once the full request has + # been received, to dispatch the request to a resource. + # + # We're patching Twisted to bail/abort early when we see someone trying to upload + # `multipart/form-data` so we can avoid Twisted parsing the entire request body into + # in-memory (specific problem of this specific `Content-Type`). This protects us + # from an attacker uploading something bigger than the available RAM and crashing + # the server with a `MemoryError`, or carefully block just enough resources to cause + # all other requests to fail. + # + # FIXME: This can be removed once we Twisted releases a fix and we update to a + # version that is patched + def requestReceived(self, command: bytes, path: bytes, version: bytes) -> None: + if command == b"POST": + ctype = self.requestHeaders.getRawHeaders(b"content-type") + if ctype and b"multipart/form-data" in ctype[0]: + self.method, self.uri = command, path + self.clientproto = version + self.code = HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value + self.code_message = bytes( + HTTPStatus.UNSUPPORTED_MEDIA_TYPE.phrase, "ascii" + ) + self.responseHeaders.setRawHeaders(b"content-length", [b"0"]) + + logger.warning( + "Aborting connection from %s because `content-type: multipart/form-data` is unsupported: %s %s", + self.client, + command, + path, + ) + self.write(b"") + self.loseConnection() + return + return super().requestReceived(command, path, version) + def handleContentChunk(self, data: bytes) -> None: # we should have a `content` by now. assert self.content, "handleContentChunk() called before gotLength()" @@ -298,7 +337,11 @@ class SynapseRequest(Request): # dispatching to the handler, so that the handler # can update the servlet name in the request # metrics - requests_counter.labels(self.get_method(), self.request_metrics.name).inc() + requests_counter.labels( + method=self.get_method(), + servlet=self.request_metrics.name, + **{SERVER_NAME_LABEL: self.our_server_name}, + ).inc() @contextlib.contextmanager def processing(self) -> Generator[None, None, None]: @@ -419,7 +462,7 @@ class SynapseRequest(Request): self.request_metrics.name. """ self.start_time = time.time() - self.request_metrics = RequestMetrics() + self.request_metrics = RequestMetrics(our_server_name=self.our_server_name) self.request_metrics.start( self.start_time, name=servlet_name, method=self.get_method() ) @@ -658,6 +701,7 @@ class SynapseSite(ProxySite): self.site_tag = site_tag self.reactor: ISynapseReactor = reactor + self.server_name = hs.hostname assert config.http_options is not None proxied = config.http_options.x_forwarded @@ -669,6 +713,7 @@ class SynapseSite(ProxySite): return request_class( channel, self, + our_server_name=self.server_name, max_request_body_size=max_request_body_size, queued=queued, request_id_header=request_id_header, diff --git a/synapse/logging/context.py b/synapse/logging/context.py index 8a2dfeba13..aa4b98e7c7 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -40,6 +40,7 @@ from typing import ( Any, Awaitable, Callable, + Literal, Optional, Tuple, Type, @@ -49,13 +50,12 @@ from typing import ( ) import attr -from typing_extensions import Literal, ParamSpec +from typing_extensions import ParamSpec from twisted.internet import defer, threads from twisted.python.threadpool import ThreadPool if TYPE_CHECKING: - from synapse.logging.scopecontextmanager import _LogContextScope from synapse.types import ISynapseReactor logger = logging.getLogger(__name__) @@ -227,16 +227,24 @@ LoggingContextOrSentinel = Union["LoggingContext", "_Sentinel"] class _Sentinel: - """Sentinel to represent the root context""" + """ + Sentinel to represent the root context - __slots__ = ["previous_context", "finished", "request", "scope", "tag"] + This should only be used for tasks outside of Synapse like when we yield control + back to the Twisted reactor (event loop) so we don't leak the current logging + context to other tasks that are scheduled next in the event loop. + + Nothing from the Synapse homeserver should be logged with the sentinel context. i.e. + we should always know which server the logs are coming from. + """ + + __slots__ = ["previous_context", "finished", "request", "tag"] def __init__(self) -> None: # Minimal set for compatibility with LoggingContext self.previous_context = None self.finished = False self.request = None - self.scope = None self.tag = None def __str__(self) -> str: @@ -289,7 +297,6 @@ class LoggingContext: "finished", "request", "tag", - "scope", ] def __init__( @@ -310,7 +317,6 @@ class LoggingContext: self.main_thread = get_thread_id() self.request = None self.tag = "" - self.scope: Optional["_LogContextScope"] = None # keep track of whether we have hit the __exit__ block for this context # (suggesting that the the thing that created the context thinks it should @@ -323,9 +329,6 @@ class LoggingContext: # we track the current request_id self.request = self.parent_context.request - # we also track the current scope: - self.scope = self.parent_context.scope - if request is not None: # the request param overrides the request from the parent context self.request = request @@ -622,9 +625,17 @@ class LoggingContextFilter(logging.Filter): class PreserveLoggingContext: - """Context manager which replaces the logging context + """ + Context manager which replaces the logging context - The previous logging context is restored on exit.""" + The previous logging context is restored on exit. + + `make_deferred_yieldable` is pretty equivalent to using `with + PreserveLoggingContext():` (using the default sentinel context), i.e. it clears the + logcontext before awaiting (and so before execution passes back to the reactor) and + restores the old context once the awaitable completes (execution passes from the + reactor back to the code). + """ __slots__ = ["_old_context", "_new_context"] @@ -790,6 +801,15 @@ def run_in_background( return from the function, and that the sentinel context is set once the deferred returned by the function completes. + To explain how the log contexts work here: + - When `run_in_background` is called, the current context is stored ("original"), + we kick off the background task in the current context, and we restore that + original context before returning + - When the background task finishes, we don't want to leak our context into the + reactor which would erroneously get attached to the next operation picked up by + the event loop. We add a callback to the deferred which will clear the logging + context after it finishes and yields control back to the reactor. + Useful for wrapping functions that return a deferred or coroutine, which you don't yield or await on (for instance because you want to pass it to deferred.gatherResults()). @@ -801,9 +821,15 @@ def run_in_background( `f` doesn't raise any deferred exceptions, otherwise a scary-looking CRITICAL error about an unhandled error will be logged without much indication about where it came from. + + Returns: + Deferred which returns the result of func, or `None` if func raises. + Note that the returned Deferred does not follow the synapse logcontext + rules. """ - current = current_context() + calling_context = current_context() try: + # (kick off the task in the current context) res = f(*args, **kwargs) except Exception: # the assumption here is that the caller doesn't want to be disturbed @@ -812,6 +838,9 @@ def run_in_background( # `res` may be a coroutine, `Deferred`, some other kind of awaitable, or a plain # value. Convert it to a `Deferred`. + # + # Wrapping the value in a deferred has the side effect of executing the coroutine, + # if it is one. If it's already a deferred, then we can just use that. d: "defer.Deferred[R]" if isinstance(res, typing.Coroutine): # Wrap the coroutine in a `Deferred`. @@ -826,20 +855,24 @@ def run_in_background( # `res` is a plain value. Wrap it in a `Deferred`. d = defer.succeed(res) + # The deferred has already completed if d.called and not d.paused: # The function should have maintained the logcontext, so we can # optimise out the messing about return d - # The function may have reset the context before returning, so - # we need to restore it now. - ctx = set_current_context(current) + # The function may have reset the context before returning, so we need to restore it + # now. + # + # Our goal is to have the caller logcontext unchanged after firing off the + # background task and returning. + set_current_context(calling_context) - # The original context will be restored when the deferred - # completes, but there is nothing waiting for it, so it will - # get leaked into the reactor or some other function which - # wasn't expecting it. We therefore need to reset the context - # here. + # The original logcontext will be restored when the deferred completes, but + # there is nothing waiting for it, so it will get leaked into the reactor (which + # would then get picked up by the next thing the reactor does). We therefore + # need to reset the logcontext here (set the `sentinel` logcontext) before + # yielding control back to the reactor. # # (If this feels asymmetric, consider it this way: we are # effectively forking a new thread of execution. We are @@ -847,7 +880,7 @@ def run_in_background( # which is supposed to have a single entry and exit point. But # by spawning off another deferred, we are effectively # adding a new exit point.) - d.addBoth(_set_context_cb, ctx) + d.addBoth(_set_context_cb, SENTINEL_CONTEXT) return d @@ -865,20 +898,34 @@ def run_coroutine_in_background( coroutine directly rather than a function. We can do this because coroutines do not run until called, and so calling an async function without awaiting cannot change the log contexts. - """ - current = current_context() + This is an ergonomic helper so we can do this: + ```python + run_coroutine_in_background(func1(arg1)) + ``` + Rather than having to do this: + ```python + run_in_background(lambda: func1(arg1)) + ``` + """ + calling_context = current_context() + + # Wrap the coroutine in a deferred, which will have the side effect of executing the + # coroutine in the background. d = defer.ensureDeferred(coroutine) - # The function may have reset the context before returning, so - # we need to restore it now. - ctx = set_current_context(current) + # The function may have reset the context before returning, so we need to restore it + # now. + # + # Our goal is to have the caller logcontext unchanged after firing off the + # background task and returning. + set_current_context(calling_context) - # The original context will be restored when the deferred - # completes, but there is nothing waiting for it, so it will - # get leaked into the reactor or some other function which - # wasn't expecting it. We therefore need to reset the context - # here. + # The original logcontext will be restored when the deferred completes, but + # there is nothing waiting for it, so it will get leaked into the reactor (which + # would then get picked up by the next thing the reactor does). We therefore + # need to reset the logcontext here (set the `sentinel` logcontext) before + # yielding control back to the reactor. # # (If this feels asymmetric, consider it this way: we are # effectively forking a new thread of execution. We are @@ -886,7 +933,7 @@ def run_coroutine_in_background( # which is supposed to have a single entry and exit point. But # by spawning off another deferred, we are effectively # adding a new exit point.) - d.addBoth(_set_context_cb, ctx) + d.addBoth(_set_context_cb, SENTINEL_CONTEXT) return d @@ -894,24 +941,43 @@ T = TypeVar("T") def make_deferred_yieldable(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]": - """Given a deferred, make it follow the Synapse logcontext rules: - - If the deferred has completed, essentially does nothing (just returns another - completed deferred with the result/failure). - - If the deferred has not yet completed, resets the logcontext before - returning a deferred. Then, when the deferred completes, restores the - current logcontext before running callbacks/errbacks. - - (This is more-or-less the opposite operation to run_in_background.) """ + Given a deferred, make it follow the Synapse logcontext rules: + + - If the deferred has completed, essentially does nothing (just returns another + completed deferred with the result/failure). + - If the deferred has not yet completed, resets the logcontext before returning a + incomplete deferred. Then, when the deferred completes, restores the current + logcontext before running callbacks/errbacks. + + This means the resultant deferred can be awaited without leaking the current + logcontext to the reactor (which would then get erroneously picked up by the next + thing the reactor does), and also means that the logcontext is preserved when the + deferred completes. + + (This is more-or-less the opposite operation to run_in_background in terms of how it + handles log contexts.) + + Pretty much equivalent to using `with PreserveLoggingContext():`, i.e. it clears the + logcontext before awaiting (and so before execution passes back to the reactor) and + restores the old context once the awaitable completes (execution passes from the + reactor back to the code). + """ + # The deferred has already completed if deferred.called and not deferred.paused: # it looks like this deferred is ready to run any callbacks we give it # immediately. We may as well optimise out the logcontext faffery. return deferred - # ok, we can't be sure that a yield won't block, so let's reset the - # logcontext, and add a callback to the deferred to restore it. + # Our goal is to have the caller logcontext unchanged after they yield/await the + # returned deferred. + # + # When the caller yield/await's the returned deferred, it may yield + # control back to the reactor. To avoid leaking the current logcontext to the + # reactor (which would then get erroneously picked up by the next thing the reactor + # does) while the deferred runs in the reactor event loop, we reset the logcontext + # and add a callback to the deferred to restore it so the caller's logcontext is + # active when the deferred completes. prev_context = set_current_context(SENTINEL_CONTEXT) deferred.addBoth(_set_context_cb, prev_context) return deferred diff --git a/synapse/logging/filter.py b/synapse/logging/filter.py index 11c27c63f2..16de488dbc 100644 --- a/synapse/logging/filter.py +++ b/synapse/logging/filter.py @@ -19,8 +19,7 @@ # # import logging - -from typing_extensions import Literal +from typing import Literal class MetadataFilter(logging.Filter): diff --git a/synapse/logging/loggers.py b/synapse/logging/loggers.py new file mode 100644 index 0000000000..7f7bfef5d4 --- /dev/null +++ b/synapse/logging/loggers.py @@ -0,0 +1,25 @@ +import logging + +root_logger = logging.getLogger() + + +class ExplicitlyConfiguredLogger(logging.Logger): + """ + A custom logger class that only allows logging if the logger is explicitly + configured (does not inherit log level from parent). + """ + + def isEnabledFor(self, level: int) -> bool: + # Check if the logger is explicitly configured + explicitly_configured_logger = self.manager.loggerDict.get(self.name) + + log_level = logging.NOTSET + if isinstance(explicitly_configured_logger, logging.Logger): + log_level = explicitly_configured_logger.level + + # If the logger is not configured, we don't log anything + if log_level == logging.NOTSET: + return False + + # Otherwise, follow the normal logging behavior + return level >= log_level diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index d976e58e49..b596b1abdb 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -251,18 +251,17 @@ class _DummyTagNames: try: import opentracing import opentracing.tags + from opentracing.scope_managers.contextvars import ContextVarsScopeManager tags = opentracing.tags except ImportError: opentracing = None # type: ignore[assignment] tags = _DummyTagNames # type: ignore[assignment] + ContextVarsScopeManager = None # type: ignore try: from jaeger_client import Config as JaegerConfig - - from synapse.logging.scopecontextmanager import LogContextScopeManager except ImportError: JaegerConfig = None # type: ignore - LogContextScopeManager = None # type: ignore try: @@ -484,7 +483,7 @@ def init_tracer(hs: "HomeServer") -> None: config = JaegerConfig( config=jaeger_config, service_name=f"{hs.config.server.server_name} {instance_name_by_type}", - scope_manager=LogContextScopeManager(), + scope_manager=ContextVarsScopeManager(), metrics_factory=PrometheusMetricsFactory(), ) @@ -796,6 +795,13 @@ def inject_response_headers(response_headers: Headers) -> None: response_headers.addRawHeader("Synapse-Trace-Id", f"{trace_id:x}") +@ensure_active_span("inject the span into a header dict") +def inject_request_headers(headers: Dict[str, str]) -> None: + span = opentracing.tracer.active_span + assert span is not None + opentracing.tracer.inject(span.context, opentracing.Format.HTTP_HEADERS, headers) + + @ensure_active_span( "get the active span context as a dict", ret=cast(Dict[str, str], {}) ) diff --git a/synapse/logging/scopecontextmanager.py b/synapse/logging/scopecontextmanager.py deleted file mode 100644 index 581e6d6411..0000000000 --- a/synapse/logging/scopecontextmanager.py +++ /dev/null @@ -1,178 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2019 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -import logging -from types import TracebackType -from typing import Optional, Type - -from opentracing import Scope, ScopeManager, Span - -import twisted - -from synapse.logging.context import ( - LoggingContext, - current_context, - nested_logging_context, -) - -logger = logging.getLogger(__name__) - - -class LogContextScopeManager(ScopeManager): - """ - The LogContextScopeManager tracks the active scope in opentracing - by using the log contexts which are native to synapse. This is so - that the basic opentracing api can be used across twisted defereds. - - It would be nice just to use opentracing's ContextVarsScopeManager, - but currently that doesn't work due to https://twistedmatrix.com/trac/ticket/10301. - """ - - def __init__(self) -> None: - pass - - @property - def active(self) -> Optional[Scope]: - """ - Returns the currently active Scope which can be used to access the - currently active Scope.span. - If there is a non-null Scope, its wrapped Span - becomes an implicit parent of any newly-created Span at - Tracer.start_active_span() time. - - Return: - The Scope that is active, or None if not available. - """ - ctx = current_context() - return ctx.scope - - def activate(self, span: Span, finish_on_close: bool) -> Scope: - """ - Makes a Span active. - Args - span: the span that should become active. - finish_on_close: whether Span should be automatically finished when - Scope.close() is called. - - Returns: - Scope to control the end of the active period for - *span*. It is a programming error to neglect to call - Scope.close() on the returned instance. - """ - - ctx = current_context() - - if not ctx: - logger.error("Tried to activate scope outside of loggingcontext") - return Scope(None, span) # type: ignore[arg-type] - - if ctx.scope is not None: - # start a new logging context as a child of the existing one. - # Doing so -- rather than updating the existing logcontext -- means that - # creating several concurrent spans under the same logcontext works - # correctly. - ctx = nested_logging_context("") - enter_logcontext = True - else: - # if there is no span currently associated with the current logcontext, we - # just store the scope in it. - # - # This feels a bit dubious, but it does hack around a problem where a - # span outlasts its parent logcontext (which would otherwise lead to - # "Re-starting finished log context" errors). - enter_logcontext = False - - scope = _LogContextScope(self, span, ctx, enter_logcontext, finish_on_close) - ctx.scope = scope - if enter_logcontext: - ctx.__enter__() - - return scope - - -class _LogContextScope(Scope): - """ - A custom opentracing scope, associated with a LogContext - - * filters out _DefGen_Return exceptions which arise from calling - `defer.returnValue` in Twisted code - - * When the scope is closed, the logcontext's active scope is reset to None. - and - if enter_logcontext was set - the logcontext is finished too. - """ - - def __init__( - self, - manager: LogContextScopeManager, - span: Span, - logcontext: LoggingContext, - enter_logcontext: bool, - finish_on_close: bool, - ): - """ - Args: - manager: - the manager that is responsible for this scope. - span: - the opentracing span which this scope represents the local - lifetime for. - logcontext: - the log context to which this scope is attached. - enter_logcontext: - if True the log context will be exited when the scope is finished - finish_on_close: - if True finish the span when the scope is closed - """ - super().__init__(manager, span) - self.logcontext = logcontext - self._finish_on_close = finish_on_close - self._enter_logcontext = enter_logcontext - - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> None: - if exc_type == twisted.internet.defer._DefGen_Return: - # filter out defer.returnValue() calls - exc_type = value = traceback = None - super().__exit__(exc_type, value, traceback) - - def __str__(self) -> str: - return f"Scope<{self.span}>" - - def close(self) -> None: - active_scope = self.manager.active - if active_scope is not self: - logger.error( - "Closing scope %s which is not the currently-active one %s", - self, - active_scope, - ) - - if self._finish_on_close: - self.span.finish() - - self.logcontext.scope = None - - if self._enter_logcontext: - self.logcontext.__exit__(None, None, None) diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 7877df62fa..29911dab77 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -118,6 +118,9 @@ DEFAULT_MAX_TIMEOUT_MS = 20_000 # Maximum allowed timeout_ms for download and thumbnail requests MAXIMUM_ALLOWED_MAX_TIMEOUT_MS = 60_000 +# The ETag header value to use for immutable media. This can be anything. +_IMMUTABLE_ETAG = "1" + def respond_404(request: SynapseRequest) -> None: assert request.path is not None @@ -224,12 +227,7 @@ def add_file_headers( request.setHeader(b"Content-Disposition", disposition.encode("ascii")) - # cache for at least a day. - # XXX: we might want to turn this off for data we don't want to - # recommend caching as it's sensitive or private - or at least - # select private. don't bother setting Expires as all our - # clients are smart enough to be happy with Cache-Control - request.setHeader(b"Cache-Control", b"public,max-age=86400,s-maxage=86400") + _add_cache_headers(request) if file_size is not None: request.setHeader(b"Content-Length", b"%d" % (file_size,)) @@ -240,6 +238,26 @@ def add_file_headers( request.setHeader(b"X-Robots-Tag", "noindex, nofollow, noarchive, noimageindex") +def _add_cache_headers(request: Request) -> None: + """Adds the appropriate cache headers to the response""" + + # Cache on the client for at least a day. + # + # We set this to "public,s-maxage=0,proxy-revalidate" to allow CDNs to cache + # the media, so long as they "revalidate" the media on every request. By + # revalidate, we mean send the request to Synapse with a `If-None-Match` + # header, to which Synapse can either respond with a 304 if the user is + # authenticated/authorized, or a 401/403 if they're not. + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=0,proxy-revalidate" + ) + + # Set an ETag header to allow requesters to use it in requests to check if + # the cache is still valid. Since media is immutable (though may be + # deleted), we just set this to a constant. + request.setHeader(b"ETag", _IMMUTABLE_ETAG) + + # separators as defined in RFC2616. SP and HT are handled separately. # see _can_encode_filename_as_token. _FILENAME_SEPARATOR_CHARS = { @@ -336,13 +354,15 @@ async def respond_with_multipart_responder( from synapse.media.media_storage import MultipartFileConsumer + _add_cache_headers(request) + # note that currently the json_object is just {}, this will change when linked media # is implemented multipart_consumer = MultipartFileConsumer( clock, request, media_type, - {}, + {}, # Note: if we change this we need to change the returned ETag. disposition, media_length, ) @@ -360,12 +380,13 @@ async def respond_with_multipart_responder( try: await responder.write_to_consumer(multipart_consumer) + except ConsumerRequestedStopError as e: + logger.debug("Failed to write to consumer: %s %s", type(e), e) + # Unregister the producer, if it has one, so Twisted doesn't complain + if request.producer: + request.unregisterProducer() except Exception as e: - # The majority of the time this will be due to the client having gone - # away. Unfortunately, Twisted simply throws a generic exception at us - # in that case. logger.warning("Failed to write to consumer: %s %s", type(e), e) - # Unregister the producer, if it has one, so Twisted doesn't complain if request.producer: request.unregisterProducer() @@ -406,12 +427,13 @@ async def respond_with_responder( add_file_headers(request, media_type, file_size, upload_name) try: await responder.write_to_consumer(request) + except ConsumerRequestedStopError as e: + logger.debug("Failed to write to consumer: %s %s", type(e), e) + # Unregister the producer, if it has one, so Twisted doesn't complain + if request.producer: + request.unregisterProducer() except Exception as e: - # The majority of the time this will be due to the client having gone - # away. Unfortunately, Twisted simply throws a generic exception at us - # in that case. logger.warning("Failed to write to consumer: %s %s", type(e), e) - # Unregister the producer, if it has one, so Twisted doesn't complain if request.producer: request.unregisterProducer() @@ -419,6 +441,46 @@ async def respond_with_responder( finish_request(request) +def respond_with_304(request: SynapseRequest) -> None: + request.setResponseCode(304) + + # could alternatively use request.notifyFinish() and flip a flag when + # the Deferred fires, but since the flag is RIGHT THERE it seems like + # a waste. + if request._disconnected: + logger.warning( + "Not sending response to request %s, already disconnected.", request + ) + return None + + _add_cache_headers(request) + + request.finish() + + +def check_for_cached_entry_and_respond(request: SynapseRequest) -> bool: + """Check if the request has a conditional header that allows us to return a + 304 Not Modified response, and if it does, return a 304 response. + + This handles clients and intermediary proxies caching media. + This method assumes that the user has already been + authorised to request the media. + + Returns True if we have responded.""" + + # We've checked the user has access to the media, so we now check if it + # is a "conditional request" and we can just return a `304 Not Modified` + # response. Since media is immutable (though may be deleted), we just + # check this is the expected constant. + etag = request.getHeader("If-None-Match") + if etag == _IMMUTABLE_ETAG: + # Return a `304 Not modified`. + respond_with_304(request) + return True + + return False + + class Responder(ABC): """Represents a response that can be streamed to the requester. @@ -614,6 +676,10 @@ def _parseparam(s: bytes) -> Generator[bytes, None, None]: s = s[end:] +class ConsumerRequestedStopError(Exception): + """A consumer asked us to stop producing""" + + @implementer(interfaces.IPushProducer) class ThreadedFileSender: """ @@ -691,7 +757,9 @@ class ThreadedFileSender: self.wakeup_event.set() if not self.deferred.called: - self.deferred.errback(Exception("Consumer asked us to stop producing")) + self.deferred.errback( + ConsumerRequestedStopError("Consumer asked us to stop producing") + ) async def start_read_loop(self) -> None: """This is the loop that drives reading/writing""" diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index f4d25a7b8b..54791f43a7 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -52,13 +52,18 @@ from synapse.media._base import ( FileInfo, Responder, ThumbnailInfo, + check_for_cached_entry_and_respond, get_filename_from_headers, respond_404, respond_with_multipart_responder, respond_with_responder, ) from synapse.media.filepath import MediaFilePaths -from synapse.media.media_storage import MediaStorage +from synapse.media.media_storage import ( + MediaStorage, + SHA256TransparentIOReader, + SHA256TransparentIOWriter, +) from synapse.media.storage_provider import StorageProviderWrapper from synapse.media.thumbnailer import Thumbnailer, ThumbnailError from synapse.media.url_previewer import UrlPreviewer @@ -172,14 +177,27 @@ class MediaRepository: else: self.url_previewer = None + # We get the media upload limits and sort them in descending order of + # time period, so that we can apply some optimizations. + self.default_media_upload_limits = hs.config.media.media_upload_limits + self.default_media_upload_limits.sort( + key=lambda limit: limit.time_period_ms, reverse=True + ) + + self.media_repository_callbacks = hs.get_module_api_callbacks().media_repository + def _start_update_recently_accessed(self) -> Deferred: return run_as_background_process( - "update_recently_accessed_media", self._update_recently_accessed + "update_recently_accessed_media", + self.server_name, + self._update_recently_accessed, ) def _start_apply_media_retention_rules(self) -> Deferred: return run_as_background_process( - "apply_media_retention_rules", self._apply_media_retention_rules + "apply_media_retention_rules", + self.server_name, + self._apply_media_retention_rules, ) async def _update_recently_accessed(self) -> None: @@ -280,52 +298,16 @@ class MediaRepository: raise NotFoundError("Media ID has expired") @trace - async def update_content( - self, - media_id: str, - media_type: str, - upload_name: Optional[str], - content: IO, - content_length: int, - auth_user: UserID, - ) -> None: - """Update the content of the given media ID. - - Args: - media_id: The media ID to replace. - media_type: The content type of the file. - upload_name: The name of the file, if provided. - content: A file like object that is the content to store - content_length: The length of the content - auth_user: The user_id of the uploader - """ - file_info = FileInfo(server_name=None, file_id=media_id) - fname = await self.media_storage.store_file(content, file_info) - logger.info("Stored local media in file %r", fname) - - await self.store.update_local_media( - media_id=media_id, - media_type=media_type, - upload_name=upload_name, - media_length=content_length, - user_id=auth_user, - ) - - try: - await self._generate_thumbnails(None, media_id, media_id, media_type) - except Exception as e: - logger.info("Failed to generate thumbnails: %s", e) - - @trace - async def create_content( + async def create_or_update_content( self, media_type: str, upload_name: Optional[str], content: IO, content_length: int, auth_user: UserID, + media_id: Optional[str] = None, ) -> MXCUri: - """Store uploaded content for a local user and return the mxc URL + """Create or update the content of the given media ID. Args: media_type: The content type of the file. @@ -333,28 +315,99 @@ class MediaRepository: content: A file like object that is the content to store content_length: The length of the content auth_user: The user_id of the uploader + media_id: The media ID to update if provided, otherwise creates + new media ID. Returns: The mxc url of the stored content """ - media_id = random_string(24) + is_new_media = media_id is None + if media_id is None: + media_id = random_string(24) file_info = FileInfo(server_name=None, file_id=media_id) - - fname = await self.media_storage.store_file(content, file_info) + sha256reader = SHA256TransparentIOReader(content) + # This implements all of IO as it has a passthrough + fname = await self.media_storage.store_file(sha256reader.wrap(), file_info) + sha256 = sha256reader.hexdigest() + should_quarantine = await self.store.get_is_hash_quarantined(sha256) logger.info("Stored local media in file %r", fname) - await self.store.store_local_media( - media_id=media_id, - media_type=media_type, - time_now_ms=self.clock.time_msec(), - upload_name=upload_name, - media_length=content_length, - user_id=auth_user, + if should_quarantine: + logger.warning( + "Media has been automatically quarantined as it matched existing quarantined media" + ) + + # Check that the user has not exceeded any of the media upload limits. + + # Use limits from module API if provided + media_upload_limits = ( + await self.media_repository_callbacks.get_media_upload_limits_for_user( + auth_user.to_string() + ) ) + # Otherwise use the default limits from config + if media_upload_limits is None: + # Note: the media upload limits are sorted so larger time periods are + # first. + media_upload_limits = self.default_media_upload_limits + + # This is the total size of media uploaded by the user in the last + # `time_period_ms` milliseconds, or None if we haven't checked yet. + uploaded_media_size: Optional[int] = None + + for limit in media_upload_limits: + # We only need to check the amount of media uploaded by the user in + # this latest (smaller) time period if the amount of media uploaded + # in a previous (larger) time period is below the limit. + # + # This optimization means that in the common case where the user + # hasn't uploaded much media, we only need to query the database + # once. + if ( + uploaded_media_size is None + or uploaded_media_size + content_length > limit.max_bytes + ): + uploaded_media_size = await self.store.get_media_uploaded_size_for_user( + user_id=auth_user.to_string(), time_period_ms=limit.time_period_ms + ) + + if uploaded_media_size + content_length > limit.max_bytes: + await self.media_repository_callbacks.on_media_upload_limit_exceeded( + user_id=auth_user.to_string(), + limit=limit, + sent_bytes=uploaded_media_size, + attempted_bytes=content_length, + ) + raise SynapseError( + 400, "Media upload limit exceeded", Codes.RESOURCE_LIMIT_EXCEEDED + ) + + if is_new_media: + await self.store.store_local_media( + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=upload_name, + media_length=content_length, + user_id=auth_user, + sha256=sha256, + quarantined_by="system" if should_quarantine else None, + ) + else: + await self.store.update_local_media( + media_id=media_id, + media_type=media_type, + upload_name=upload_name, + media_length=content_length, + user_id=auth_user, + sha256=sha256, + quarantined_by="system" if should_quarantine else None, + ) + try: await self._generate_thumbnails(None, media_id, media_id, media_type) except Exception as e: @@ -459,6 +512,11 @@ class MediaRepository: self.mark_recently_accessed(None, media_id) + # Once we've checked auth we can return early if the media is cached on + # the client + if check_for_cached_entry_and_respond(request): + return + media_type = media_info.media_type if not media_type: media_type = "application/octet-stream" @@ -538,6 +596,17 @@ class MediaRepository: allow_authenticated, ) + # Check if the media is cached on the client, if so return 304. We need + # to do this after we have fetched remote media, as we need it to do the + # auth. + if check_for_cached_entry_and_respond(request): + # We always need to use the responder. + if responder: + with responder: + pass + + return + # We deliberately stream the file outside the lock if responder and media_info: upload_name = name if name else media_info.upload_name @@ -739,11 +808,13 @@ class MediaRepository: file_info = FileInfo(server_name=server_name, file_id=file_id) async with self.media_storage.store_into_file(file_info) as (f, fname): + sha256writer = SHA256TransparentIOWriter(f) try: length, headers = await self.client.download_media( server_name, media_id, - output_stream=f, + # This implements all of BinaryIO as it has a passthrough + output_stream=sha256writer.wrap(), max_size=self.max_upload_size, max_timeout_ms=max_timeout_ms, download_ratelimiter=download_ratelimiter, @@ -808,6 +879,7 @@ class MediaRepository: upload_name=upload_name, media_length=length, filesystem_id=file_id, + sha256=sha256writer.hexdigest(), ) logger.info("Stored remote media in file %r", fname) @@ -828,6 +900,7 @@ class MediaRepository: last_access_ts=time_now_ms, quarantined_by=None, authenticated=authenticated, + sha256=sha256writer.hexdigest(), ) async def _federation_download_remote_file( @@ -862,11 +935,13 @@ class MediaRepository: file_info = FileInfo(server_name=server_name, file_id=file_id) async with self.media_storage.store_into_file(file_info) as (f, fname): + sha256writer = SHA256TransparentIOWriter(f) try: res = await self.client.federation_download_media( server_name, media_id, - output_stream=f, + # This implements all of BinaryIO as it has a passthrough + output_stream=sha256writer.wrap(), max_size=self.max_upload_size, max_timeout_ms=max_timeout_ms, download_ratelimiter=download_ratelimiter, @@ -937,6 +1012,7 @@ class MediaRepository: upload_name=upload_name, media_length=length, filesystem_id=file_id, + sha256=sha256writer.hexdigest(), ) logger.debug("Stored remote media in file %r", fname) @@ -957,6 +1033,7 @@ class MediaRepository: last_access_ts=time_now_ms, quarantined_by=None, authenticated=authenticated, + sha256=sha256writer.hexdigest(), ) def _get_thumbnail_requirements( @@ -1343,8 +1420,8 @@ class MediaRepository: ) logger.info( - "Purging remote media last accessed before" - f" {remote_media_threshold_timestamp_ms}" + "Purging remote media last accessed before %s", + remote_media_threshold_timestamp_ms, ) await self.delete_old_remote_media( @@ -1359,8 +1436,8 @@ class MediaRepository: ) logger.info( - "Purging local media last accessed before" - f" {local_media_threshold_timestamp_ms}" + "Purging local media last accessed before %s", + local_media_threshold_timestamp_ms, ) await self.delete_old_local_media( diff --git a/synapse/media/media_storage.py b/synapse/media/media_storage.py index c25d1a9ba3..afd33c02a1 100644 --- a/synapse/media/media_storage.py +++ b/synapse/media/media_storage.py @@ -19,6 +19,7 @@ # # import contextlib +import hashlib import json import logging import os @@ -70,6 +71,88 @@ logger = logging.getLogger(__name__) CRLF = b"\r\n" +class SHA256TransparentIOWriter: + """Will generate a SHA256 hash from a source stream transparently. + + Args: + source: Source stream. + """ + + def __init__(self, source: BinaryIO): + self._hash = hashlib.sha256() + self._source = source + + def write(self, buffer: Union[bytes, bytearray]) -> int: + """Wrapper for source.write() + + Args: + buffer + + Returns: + the value of source.write() + """ + res = self._source.write(buffer) + self._hash.update(buffer) + return res + + def hexdigest(self) -> str: + """The digest of the written or read value. + + Returns: + The digest in hex formaat. + """ + return self._hash.hexdigest() + + def wrap(self) -> BinaryIO: + # This class implements a subset the IO interface and passes through everything else via __getattr__ + return cast(BinaryIO, self) + + # Passthrough any other calls + def __getattr__(self, attr_name: str) -> Any: + return getattr(self._source, attr_name) + + +class SHA256TransparentIOReader: + """Will generate a SHA256 hash from a source stream transparently. + + Args: + source: Source IO stream. + """ + + def __init__(self, source: IO): + self._hash = hashlib.sha256() + self._source = source + + def read(self, n: int = -1) -> bytes: + """Wrapper for source.read() + + Args: + n + + Returns: + the value of source.read() + """ + bytes = self._source.read(n) + self._hash.update(bytes) + return bytes + + def hexdigest(self) -> str: + """The digest of the written or read value. + + Returns: + The digest in hex formaat. + """ + return self._hash.hexdigest() + + def wrap(self) -> IO: + # This class implements a subset the IO interface and passes through everything else via __getattr__ + return cast(IO, self) + + # Passthrough any other calls + def __getattr__(self, attr_name: str) -> Any: + return getattr(self._source, attr_name) + + class MediaStorage: """Responsible for storing/fetching files from local sources. @@ -107,7 +190,6 @@ class MediaStorage: Returns: the file path written to in the primary media store """ - async with self.store_into_file(file_info) as (f, fname): # Write to the main media repository await self.write_to_file(source, f) diff --git a/synapse/media/preview_html.py b/synapse/media/preview_html.py index 62ce7789be..38ae126a23 100644 --- a/synapse/media/preview_html.py +++ b/synapse/media/preview_html.py @@ -133,7 +133,7 @@ def decode_body( content_type: The Content-Type header. Returns: - The parsed HTML body, or None if an error occurred during processed. + The parsed HTML body, or None if an error occurred during processing. """ # If there's no body, nothing useful is going to be found. if not body: @@ -158,9 +158,31 @@ def decode_body( # Create an HTML parser. parser = etree.HTMLParser(recover=True, encoding=encoding) - # Attempt to parse the body. Returns None if the body was successfully - # parsed, but no tree was found. - return etree.fromstring(body, parser) + # Attempt to parse the body. With `lxml` 6.0.0+, this will be an empty HTML + # tree if the body was successfully parsed, but no tree was found. In + # previous `lxml` versions, `etree.fromstring` would return `None` in that + # case. + html_tree = etree.fromstring(body, parser) + + # Account for the above referenced case where `html_tree` is an HTML tree + # with an empty body. If so, return None. + if html_tree is not None and html_tree.tag == "html": + # If the tree has only a single element and it's empty, then + # return None. + body_el = html_tree.find("body") + if body_el is not None and len(html_tree) == 1: + # Extract the content of the body tag as text. + body_text = "".join(cast(Iterable[str], body_el.itertext())) + + # Strip any undecodable Unicode characters and whitespace. + body_text = body_text.strip("\ufffd").strip() + + # If there's no text left, and there were no child tags, + # then we consider the tag empty. + if not body_text and len(body_el) == 0: + return None + + return html_tree def _get_meta_tags( diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index 3845067835..5d9afda322 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -34,6 +34,7 @@ from synapse.logging.opentracing import trace from synapse.media._base import ( FileInfo, ThumbnailInfo, + check_for_cached_entry_and_respond, respond_404, respond_with_file, respond_with_multipart_responder, @@ -67,6 +68,11 @@ class ThumbnailError(Exception): class Thumbnailer: FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"} + # Which image formats we allow Pillow to open. + # This should intentionally be kept restrictive, because the decoder of any + # format in this list becomes part of our trusted computing base. + PILLOW_FORMATS = ("jpeg", "png", "webp", "gif") + @staticmethod def set_limits(max_image_pixels: int) -> None: Image.MAX_IMAGE_PIXELS = max_image_pixels @@ -76,7 +82,7 @@ class Thumbnailer: self._closed = False try: - self.image = Image.open(input_path) + self.image = Image.open(input_path, formats=self.PILLOW_FORMATS) except OSError as e: # If an error occurs opening the image, a thumbnail won't be able to # be generated. @@ -289,6 +295,11 @@ class ThumbnailProvider: if media_info.authenticated: raise NotFoundError() + # Once we've checked auth we can return early if the media is cached on + # the client + if check_for_cached_entry_and_respond(request): + return + thumbnail_infos = await self.store.get_local_media_thumbnails(media_id) await self._select_and_respond_with_thumbnail( request, @@ -329,6 +340,11 @@ class ThumbnailProvider: if media_info.authenticated: raise NotFoundError() + # Once we've checked auth we can return early if the media is cached on + # the client + if check_for_cached_entry_and_respond(request): + return + thumbnail_infos = await self.store.get_local_media_thumbnails(media_id) for info in thumbnail_infos: t_w = info.width == desired_width @@ -426,6 +442,10 @@ class ThumbnailProvider: respond_404(request) return + # Check if the media is cached on the client, if so return 304. + if check_for_cached_entry_and_respond(request): + return + thumbnail_infos = await self.store.get_remote_media_thumbnails( server_name, media_id ) @@ -505,6 +525,10 @@ class ThumbnailProvider: if media_info.authenticated: raise NotFoundError() + # Check if the media is cached on the client, if so return 304. + if check_for_cached_entry_and_respond(request): + return + thumbnail_infos = await self.store.get_remote_media_thumbnails( server_name, media_id ) diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 2e65a04789..8f106a3d5f 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -41,7 +41,7 @@ from synapse.api.errors import Codes, SynapseError from synapse.http.client import SimpleHttpClient from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.media._base import FileInfo, get_filename_from_headers -from synapse.media.media_storage import MediaStorage +from synapse.media.media_storage import MediaStorage, SHA256TransparentIOWriter from synapse.media.oembed import OEmbedProvider from synapse.media.preview_html import decode_body, parse_html_to_open_graph from synapse.metrics.background_process_metrics import run_as_background_process @@ -200,6 +200,7 @@ class UrlPreviewer: # JSON-encoded OG metadata self._cache: ExpiringCache[str, ObservableDeferred] = ExpiringCache( cache_name="url_previews", + server_name=self.server_name, clock=self.clock, # don't spider URLs more often than once an hour expiry_ms=ONE_HOUR, @@ -287,7 +288,7 @@ class UrlPreviewer: og["og:image:width"] = dims["width"] og["og:image:height"] = dims["height"] else: - logger.warning("Couldn't get dims for %s" % url) + logger.warning("Couldn't get dims for %s", url) # define our OG response for this media elif _is_html(media_info.media_type): @@ -593,17 +594,26 @@ class UrlPreviewer: file_info = FileInfo(server_name=None, file_id=file_id, url_cache=True) async with self.media_storage.store_into_file(file_info) as (f, fname): + sha256writer = SHA256TransparentIOWriter(f) if url.startswith("data:"): if not allow_data_urls: raise SynapseError( 500, "Previewing of data: URLs is forbidden", Codes.UNKNOWN ) - download_result = await self._parse_data_url(url, f) + download_result = await self._parse_data_url(url, sha256writer.wrap()) else: - download_result = await self._download_url(url, f) + download_result = await self._download_url(url, sha256writer.wrap()) try: + sha256 = sha256writer.hexdigest() + should_quarantine = await self.store.get_is_hash_quarantined(sha256) + + if should_quarantine: + logger.warning( + "Media has been automatically quarantined as it matched existing quarantined media" + ) + time_now_ms = self.clock.time_msec() await self.store.store_local_media( @@ -614,6 +624,8 @@ class UrlPreviewer: media_length=download_result.length, user_id=user, url_cache=url, + sha256=sha256, + quarantined_by="system" if should_quarantine else None, ) except Exception as e: @@ -728,7 +740,7 @@ class UrlPreviewer: def _start_expire_url_cache_data(self) -> Deferred: return run_as_background_process( - "expire_url_cache_data", self._expire_url_cache_data + "expire_url_cache_data", self.server_name, self._expire_url_cache_data ) async def _expire_url_cache_data(self) -> None: diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 3051b623d0..2ffb14070b 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -25,6 +25,7 @@ import logging import os import platform import threading +from importlib import metadata from typing import ( Callable, Dict, @@ -32,6 +33,7 @@ from typing import ( Iterable, Mapping, Optional, + Sequence, Set, Tuple, Type, @@ -41,7 +43,15 @@ from typing import ( ) import attr -from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram, Metric +from packaging.version import parse as parse_version +from prometheus_client import ( + CollectorRegistry, + Counter, + Gauge, + Histogram, + Metric, + generate_latest, +) from prometheus_client.core import ( REGISTRY, GaugeHistogramMetricFamily, @@ -49,11 +59,12 @@ from prometheus_client.core import ( ) from twisted.python.threadpool import ThreadPool +from twisted.web.resource import Resource +from twisted.web.server import Request # This module is imported for its side effects; flake8 needn't warn that it's unused. import synapse.metrics._reactor_metrics # noqa: F401 from synapse.metrics._gc import MIN_TIME_BETWEEN_GCS, install_gc_manager -from synapse.metrics._twisted_exposition import MetricsResource, generate_latest from synapse.metrics._types import Collector from synapse.types import StrSequence from synapse.util import SYNAPSE_VERSION @@ -62,10 +73,71 @@ logger = logging.getLogger(__name__) METRICS_PREFIX = "/_synapse/metrics" -all_gauges: Dict[str, Collector] = {} - HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat") +SERVER_NAME_LABEL = "server_name" +""" +The `server_name` label is used to identify the homeserver that the metrics correspond +to. Because we support multiple instances of Synapse running in the same process and all +metrics are in a single global `REGISTRY`, we need to manually label any metrics. + +In the case of a Synapse homeserver, this should be set to the homeserver name +(`hs.hostname`). + +We're purposely not using the `instance` label for this purpose as that should be "The +: part of the target's URL that was scraped.". Also: "In Prometheus +terms, an endpoint you can scrape is called an *instance*, usually corresponding to a +single process." (source: https://prometheus.io/docs/concepts/jobs_instances/) +""" + + +CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" +""" +Content type of the latest text format for Prometheus metrics. + +Pulled directly from the prometheus_client library. +""" + + +def _set_prometheus_client_use_created_metrics(new_value: bool) -> None: + """ + Sets whether prometheus_client should expose `_created`-suffixed metrics for + all gauges, histograms and summaries. + + There is no programmatic way in the old versions of `prometheus_client` to disable + this without poking at internals; the proper way in the old `prometheus_client` + versions (> `0.14.0` < `0.18.0`) is to use an environment variable which + prometheus_client loads at import time. For versions > `0.18.0`, we can use the + dedicated `disable_created_metrics()`/`enable_created_metrics()`. + + The motivation for disabling these `_created` metrics is that they're a waste of + space as they're not useful but they take up space in Prometheus. It's not the end + of the world if this doesn't work. + """ + import prometheus_client.metrics + + if hasattr(prometheus_client.metrics, "_use_created"): + prometheus_client.metrics._use_created = new_value + # Just log an error for old versions that don't support disabling the unecessary + # metrics. It's not the end of the world if this doesn't work as it just means extra + # wasted space taken up in Prometheus but things keep working. + elif parse_version(metadata.version("prometheus_client")) < parse_version("0.14.0"): + logger.error( + "Can't disable `_created` metrics in prometheus_client (unsupported `prometheus_client` version, too old)" + ) + # If the attribute doesn't exist on a newer version, this is a sign that the brittle + # hack is broken. We should consider updating the minimum version of + # `prometheus_client` to a version (> `0.18.0`) where we can use dedicated + # `disable_created_metrics()`/`enable_created_metrics()` functions. + else: + raise Exception( + "Can't disable `_created` metrics in prometheus_client (brittle hack broken?)" + ) + + +# Set this globally so it applies wherever we generate/collect metrics +_set_prometheus_client_use_created_metrics(False) + class _RegistryProxy: @staticmethod @@ -82,47 +154,117 @@ class _RegistryProxy: RegistryProxy = cast(CollectorRegistry, _RegistryProxy) -@attr.s(slots=True, hash=True, auto_attribs=True) +@attr.s(slots=True, hash=True, auto_attribs=True, kw_only=True) class LaterGauge(Collector): """A Gauge which periodically calls a user-provided callback to produce metrics.""" name: str desc: str - labels: Optional[StrSequence] = attr.ib(hash=False) - # callback: should either return a value (if there are no labels for this metric), - # or dict mapping from a label tuple to a value - caller: Callable[ - [], Union[Mapping[Tuple[str, ...], Union[int, float]], Union[int, float]] - ] + labelnames: Optional[StrSequence] = attr.ib(hash=False) + _instance_id_to_hook_map: Dict[ + Optional[str], # instance_id + Callable[ + [], Union[Mapping[Tuple[str, ...], Union[int, float]], Union[int, float]] + ], + ] = attr.ib(factory=dict, hash=False) + """ + Map from homeserver instance_id to a callback. Each callback should either return a + value (if there are no labels for this metric), or dict mapping from a label tuple + to a value. + + We use `instance_id` instead of `server_name` because it's possible to have multiple + workers running in the same process with the same `server_name`. + """ def collect(self) -> Iterable[Metric]: - g = GaugeMetricFamily(self.name, self.desc, labels=self.labels) + # The decision to add `SERVER_NAME_LABEL` is from the `LaterGauge` usage itself + # (we don't enforce it here, one level up). + g = GaugeMetricFamily(self.name, self.desc, labels=self.labelnames) # type: ignore[missing-server-name-label] - try: - calls = self.caller() - except Exception: - logger.exception("Exception running callback for LaterGauge(%s)", self.name) - yield g - return + for homeserver_instance_id, hook in self._instance_id_to_hook_map.items(): + try: + hook_result = hook() + except Exception: + logger.exception( + "Exception running callback for LaterGauge(%s) for homeserver_instance_id=%s", + self.name, + homeserver_instance_id, + ) + # Continue to return the rest of the metrics that aren't broken + continue - if isinstance(calls, (int, float)): - g.add_metric([], calls) - else: - for k, v in calls.items(): - g.add_metric(k, v) + if isinstance(hook_result, (int, float)): + g.add_metric([], hook_result) + else: + for k, v in hook_result.items(): + g.add_metric(k, v) yield g + def register_hook( + self, + *, + homeserver_instance_id: Optional[str], + hook: Callable[ + [], Union[Mapping[Tuple[str, ...], Union[int, float]], Union[int, float]] + ], + ) -> None: + """ + Register a callback/hook that will be called to generate a metric samples for + the gauge. + + Args: + homeserver_instance_id: The unique ID for this Synapse process instance + (`hs.get_instance_id()`) that this hook is associated with. This can be used + later to lookup all hooks associated with a given server name in order to + unregister them. This should only be omitted for global hooks that work + across all homeservers. + hook: A callback that should either return a value (if there are no + labels for this metric), or dict mapping from a label tuple to a value + """ + # We shouldn't have multiple hooks registered for the same homeserver `instance_id`. + existing_hook = self._instance_id_to_hook_map.get(homeserver_instance_id) + assert existing_hook is None, ( + f"LaterGauge(name={self.name}) hook already registered for homeserver_instance_id={homeserver_instance_id}. " + "This is likely a Synapse bug and you forgot to unregister the previous hooks for " + "the server (especially in tests)." + ) + + self._instance_id_to_hook_map[homeserver_instance_id] = hook + + def unregister_hooks_for_homeserver_instance_id( + self, homeserver_instance_id: str + ) -> None: + """ + Unregister all hooks associated with the given homeserver `instance_id`. This should be + called when a homeserver is shutdown to avoid extra hooks sitting around. + + Args: + homeserver_instance_id: The unique ID for this Synapse process instance to + unregister hooks for (`hs.get_instance_id()`). + """ + self._instance_id_to_hook_map.pop(homeserver_instance_id, None) + def __attrs_post_init__(self) -> None: - self._register() - - def _register(self) -> None: - if self.name in all_gauges.keys(): - logger.warning("%s already registered, reregistering" % (self.name,)) - REGISTRY.unregister(all_gauges.pop(self.name)) - REGISTRY.register(self) - all_gauges[self.name] = self + + # We shouldn't have multiple metrics with the same name. Typically, metrics + # should be created globally so you shouldn't be running into this and this will + # catch any stupid mistakes. The `REGISTRY.register(self)` call above will also + # raise an error if the metric already exists but to make things explicit, we'll + # also check here. + existing_gauge = all_later_gauges_to_clean_up_on_shutdown.get(self.name) + assert existing_gauge is None, f"LaterGauge(name={self.name}) already exists. " + + # Keep track of the gauge so we can clean it up later. + all_later_gauges_to_clean_up_on_shutdown[self.name] = self + + +all_later_gauges_to_clean_up_on_shutdown: Dict[str, LaterGauge] = {} +""" +Track all `LaterGauge` instances so we can remove any associated hooks during homeserver +shutdown. +""" # `MetricsEntry` only makes sense when it is a `Protocol`, @@ -174,7 +316,7 @@ class InFlightGauge(Generic[MetricsEntry], Collector): # Protects access to _registrations self._lock = threading.Lock() - self._register_with_collector() + REGISTRY.register(self) def register( self, @@ -192,7 +334,16 @@ class InFlightGauge(Generic[MetricsEntry], Collector): same key. Note that `callback` may be called on a separate thread. + + Args: + key: A tuple of label values, which must match the order of the + `labels` given to the constructor. + callback """ + assert len(key) == len(self.labels), ( + f"Expected {len(self.labels)} labels in `key`, got {len(key)}: {key}" + ) + with self._lock: self._registrations.setdefault(key, set()).add(callback) @@ -201,7 +352,17 @@ class InFlightGauge(Generic[MetricsEntry], Collector): key: Tuple[str, ...], callback: Callable[[MetricsEntry], None], ) -> None: - """Registers that we've exited a block with labels `key`.""" + """ + Registers that we've exited a block with labels `key`. + + Args: + key: A tuple of label values, which must match the order of the + `labels` given to the constructor. + callback + """ + assert len(key) == len(self.labels), ( + f"Expected {len(self.labels)} labels in `key`, got {len(key)}: {key}" + ) with self._lock: self._registrations.setdefault(key, set()).discard(callback) @@ -211,7 +372,9 @@ class InFlightGauge(Generic[MetricsEntry], Collector): Note: may be called by a separate thread. """ - in_flight = GaugeMetricFamily( + # The decision to add `SERVER_NAME_LABEL` is from the `GaugeBucketCollector` + # usage itself (we don't enforce it here, one level up). + in_flight = GaugeMetricFamily( # type: ignore[missing-server-name-label] self.name + "_total", self.desc, labels=self.labels ) @@ -225,7 +388,7 @@ class InFlightGauge(Generic[MetricsEntry], Collector): with self._lock: callbacks = set(self._registrations[key]) - in_flight.add_metric(key, len(callbacks)) + in_flight.add_metric(labels=key, value=len(callbacks)) metrics = self._metrics_class() metrics_by_key[key] = metrics @@ -235,20 +398,59 @@ class InFlightGauge(Generic[MetricsEntry], Collector): yield in_flight for name in self.sub_metrics: - gauge = GaugeMetricFamily( + # The decision to add `SERVER_NAME_LABEL` is from the `InFlightGauge` usage + # itself (we don't enforce it here, one level up). + gauge = GaugeMetricFamily( # type: ignore[missing-server-name-label] "_".join([self.name, name]), "", labels=self.labels ) for key, metrics in metrics_by_key.items(): - gauge.add_metric(key, getattr(metrics, name)) + gauge.add_metric(labels=key, value=getattr(metrics, name)) yield gauge - def _register_with_collector(self) -> None: - if self.name in all_gauges.keys(): - logger.warning("%s already registered, reregistering" % (self.name,)) - REGISTRY.unregister(all_gauges.pop(self.name)) - REGISTRY.register(self) - all_gauges[self.name] = self +class GaugeHistogramMetricFamilyWithLabels(GaugeHistogramMetricFamily): + """ + Custom version of `GaugeHistogramMetricFamily` from `prometheus_client` that allows + specifying labels and label values. + + A single gauge histogram and its samples. + + For use by custom collectors. + """ + + def __init__( + self, + *, + name: str, + documentation: str, + gsum_value: float, + buckets: Optional[Sequence[Tuple[str, float]]] = None, + labelnames: StrSequence = (), + labelvalues: StrSequence = (), + unit: str = "", + ): + # Sanity check the number of label values matches the number of label names. + if len(labelvalues) != len(labelnames): + raise ValueError( + "The number of label values must match the number of label names" + ) + + # Call the super to validate and set the labelnames. We use this stable API + # instead of setting the internal `_labelnames` field directly. + super().__init__( + name=name, + documentation=documentation, + labels=labelnames, + # Since `GaugeHistogramMetricFamily` doesn't support supplying `labels` and + # `buckets` at the same time (artificial limitation), we will just set these + # as `None` and set up the buckets ourselves just below. + buckets=None, + gsum_value=None, + ) + + # Create a gauge for each bucket. + if buckets is not None: + self.add_metric(labels=labelvalues, buckets=buckets, gsum_value=gsum_value) class GaugeBucketCollector(Collector): @@ -263,14 +465,17 @@ class GaugeBucketCollector(Collector): __slots__ = ( "_name", "_documentation", + "_labelnames", "_bucket_bounds", "_metric", ) def __init__( self, + *, name: str, documentation: str, + labelnames: Optional[StrSequence], buckets: Iterable[float], registry: CollectorRegistry = REGISTRY, ): @@ -284,6 +489,7 @@ class GaugeBucketCollector(Collector): """ self._name = name self._documentation = documentation + self._labelnames = labelnames if labelnames else () # the tops of the buckets self._bucket_bounds = [float(b) for b in buckets] @@ -295,7 +501,7 @@ class GaugeBucketCollector(Collector): # We initially set this to None. We won't report metrics until # this has been initialised after a successful data update - self._metric: Optional[GaugeHistogramMetricFamily] = None + self._metric: Optional[GaugeHistogramMetricFamilyWithLabels] = None registry.register(self) @@ -304,15 +510,26 @@ class GaugeBucketCollector(Collector): if self._metric is not None: yield self._metric - def update_data(self, values: Iterable[float]) -> None: + def update_data(self, values: Iterable[float], labels: StrSequence = ()) -> None: """Update the data to be reported by the metric The existing data is cleared, and each measurement in the input is assigned to the relevant bucket. - """ - self._metric = self._values_to_metric(values) - def _values_to_metric(self, values: Iterable[float]) -> GaugeHistogramMetricFamily: + Args: + values + labels + """ + self._metric = self._values_to_metric(values, labels) + + def _values_to_metric( + self, values: Iterable[float], labels: StrSequence = () + ) -> GaugeHistogramMetricFamilyWithLabels: + """ + Args: + values + labels + """ total = 0.0 bucket_values = [0 for _ in self._bucket_bounds] @@ -330,9 +547,13 @@ class GaugeBucketCollector(Collector): # that bucket or below. accumulated_values = itertools.accumulate(bucket_values) - return GaugeHistogramMetricFamily( - self._name, - self._documentation, + # The decision to add `SERVER_NAME_LABEL` is from the `GaugeBucketCollector` + # usage itself (we don't enforce it here, one level up). + return GaugeHistogramMetricFamilyWithLabels( # type: ignore[missing-server-name-label] + name=self._name, + documentation=self._documentation, + labelnames=self._labelnames, + labelvalues=labels, buckets=list( zip((str(b) for b in self._bucket_bounds), accumulated_values) ), @@ -364,61 +585,82 @@ class CPUMetrics(Collector): line = s.read() raw_stats = line.split(") ", 1)[1].split(" ") - user = GaugeMetricFamily("process_cpu_user_seconds_total", "") + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + user = GaugeMetricFamily("process_cpu_user_seconds_total", "") # type: ignore[missing-server-name-label] user.add_metric([], float(raw_stats[11]) / self.ticks_per_sec) yield user - sys = GaugeMetricFamily("process_cpu_system_seconds_total", "") + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + sys = GaugeMetricFamily("process_cpu_system_seconds_total", "") # type: ignore[missing-server-name-label] sys.add_metric([], float(raw_stats[12]) / self.ticks_per_sec) yield sys -REGISTRY.register(CPUMetrics()) +# This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. +REGISTRY.register(CPUMetrics()) # type: ignore[missing-server-name-label] # # Federation Metrics # -sent_transactions_counter = Counter("synapse_federation_client_sent_transactions", "") +sent_transactions_counter = Counter( + "synapse_federation_client_sent_transactions", "", labelnames=[SERVER_NAME_LABEL] +) -events_processed_counter = Counter("synapse_federation_client_events_processed", "") +events_processed_counter = Counter( + "synapse_federation_client_events_processed", "", labelnames=[SERVER_NAME_LABEL] +) event_processing_loop_counter = Counter( - "synapse_event_processing_loop_count", "Event processing loop iterations", ["name"] + "synapse_event_processing_loop_count", + "Event processing loop iterations", + labelnames=["name", SERVER_NAME_LABEL], ) event_processing_loop_room_count = Counter( "synapse_event_processing_loop_room_count", "Rooms seen per event processing loop iteration", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], ) # Used to track where various components have processed in the event stream, # e.g. federation sending, appservice sending, etc. -event_processing_positions = Gauge("synapse_event_processing_positions", "", ["name"]) +event_processing_positions = Gauge( + "synapse_event_processing_positions", "", labelnames=["name", SERVER_NAME_LABEL] +) # Used to track the current max events stream position -event_persisted_position = Gauge("synapse_event_persisted_position", "") +event_persisted_position = Gauge( + "synapse_event_persisted_position", "", labelnames=[SERVER_NAME_LABEL] +) # Used to track the received_ts of the last event processed by various # components -event_processing_last_ts = Gauge("synapse_event_processing_last_ts", "", ["name"]) +event_processing_last_ts = Gauge( + "synapse_event_processing_last_ts", "", labelnames=["name", SERVER_NAME_LABEL] +) # Used to track the lag processing events. This is the time difference # between the last processed event's received_ts and the time it was # finished being processed. -event_processing_lag = Gauge("synapse_event_processing_lag", "", ["name"]) +event_processing_lag = Gauge( + "synapse_event_processing_lag", "", labelnames=["name", SERVER_NAME_LABEL] +) event_processing_lag_by_event = Histogram( "synapse_event_processing_lag_by_event", "Time between an event being persisted and it being queued up to be sent to the relevant remote servers", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], ) # Build info of the running server. -build_info = Gauge( +# +# This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. We +# consider this process-level because all Synapse homeservers running in the process +# will use the same Synapse version. +build_info = Gauge( # type: ignore[missing-server-name-label] "synapse_build_info", "Build information", ["pythonversion", "version", "osversion"] ) build_info.labels( @@ -434,44 +676,74 @@ threepid_send_requests = Histogram( " there is a request with try count of 4, then there would have been one" " each for 1, 2 and 3", buckets=(1, 2, 3, 4, 5, 10), - labelnames=("type", "reason"), + labelnames=("type", "reason", SERVER_NAME_LABEL), ) threadpool_total_threads = Gauge( "synapse_threadpool_total_threads", "Total number of threads currently in the threadpool", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], ) threadpool_total_working_threads = Gauge( "synapse_threadpool_working_threads", "Number of threads currently working in the threadpool", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], ) threadpool_total_min_threads = Gauge( "synapse_threadpool_min_threads", "Minimum number of threads configured in the threadpool", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], ) threadpool_total_max_threads = Gauge( "synapse_threadpool_max_threads", "Maximum number of threads configured in the threadpool", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], ) -def register_threadpool(name: str, threadpool: ThreadPool) -> None: - """Add metrics for the threadpool.""" +def register_threadpool(*, name: str, server_name: str, threadpool: ThreadPool) -> None: + """ + Add metrics for the threadpool. - threadpool_total_min_threads.labels(name).set(threadpool.min) - threadpool_total_max_threads.labels(name).set(threadpool.max) + Args: + name: The name of the threadpool, used to identify it in the metrics. + server_name: The homeserver name (used to label metrics) (this should be `hs.hostname`). + threadpool: The threadpool to register metrics for. + """ - threadpool_total_threads.labels(name).set_function(lambda: len(threadpool.threads)) - threadpool_total_working_threads.labels(name).set_function( - lambda: len(threadpool.working) - ) + threadpool_total_min_threads.labels( + name=name, **{SERVER_NAME_LABEL: server_name} + ).set(threadpool.min) + threadpool_total_max_threads.labels( + name=name, **{SERVER_NAME_LABEL: server_name} + ).set(threadpool.max) + + threadpool_total_threads.labels( + name=name, **{SERVER_NAME_LABEL: server_name} + ).set_function(lambda: len(threadpool.threads)) + threadpool_total_working_threads.labels( + name=name, **{SERVER_NAME_LABEL: server_name} + ).set_function(lambda: len(threadpool.working)) + + +class MetricsResource(Resource): + """ + Twisted ``Resource`` that serves prometheus metrics. + """ + + isLeaf = True + + def __init__(self, registry: CollectorRegistry = REGISTRY): + self.registry = registry + + def render_GET(self, request: Request) -> bytes: + request.setHeader(b"Content-Type", CONTENT_TYPE_LATEST.encode("ascii")) + response = generate_latest(self.registry) + request.setHeader(b"Content-Length", str(len(response))) + return response __all__ = [ diff --git a/synapse/metrics/_gc.py b/synapse/metrics/_gc.py index d16481a0f6..e7783b05e6 100644 --- a/synapse/metrics/_gc.py +++ b/synapse/metrics/_gc.py @@ -54,8 +54,9 @@ running_on_pypy = platform.python_implementation() == "PyPy" # Python GC metrics # -gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects", ["gen"]) -gc_time = Histogram( +# These are process-level metrics, so they do not have the `SERVER_NAME_LABEL`. +gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects", ["gen"]) # type: ignore[missing-server-name-label] +gc_time = Histogram( # type: ignore[missing-server-name-label] "python_gc_time", "Time taken to GC (sec)", ["gen"], @@ -82,7 +83,8 @@ gc_time = Histogram( class GCCounts(Collector): def collect(self) -> Iterable[Metric]: - cm = GaugeMetricFamily("python_gc_counts", "GC object counts", labels=["gen"]) + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + cm = GaugeMetricFamily("python_gc_counts", "GC object counts", labels=["gen"]) # type: ignore[missing-server-name-label] for n, m in enumerate(gc.get_count()): cm.add_metric([str(n)], m) @@ -101,7 +103,8 @@ def install_gc_manager() -> None: if running_on_pypy: return - REGISTRY.register(GCCounts()) + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + REGISTRY.register(GCCounts()) # type: ignore[missing-server-name-label] gc.disable() @@ -176,7 +179,8 @@ class PyPyGCStats(Collector): # # Total time spent in GC: 0.073 # s.total_gc_time - pypy_gc_time = CounterMetricFamily( + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + pypy_gc_time = CounterMetricFamily( # type: ignore[missing-server-name-label] "pypy_gc_time_seconds_total", "Total time spent in PyPy GC", labels=[], @@ -184,7 +188,8 @@ class PyPyGCStats(Collector): pypy_gc_time.add_metric([], s.total_gc_time / 1000) yield pypy_gc_time - pypy_mem = GaugeMetricFamily( + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + pypy_mem = GaugeMetricFamily( # type: ignore[missing-server-name-label] "pypy_memory_bytes", "Memory tracked by PyPy allocator", labels=["state", "class", "kind"], @@ -208,4 +213,5 @@ class PyPyGCStats(Collector): if running_on_pypy: - REGISTRY.register(PyPyGCStats()) + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + REGISTRY.register(PyPyGCStats()) # type: ignore[missing-server-name-label] diff --git a/synapse/metrics/_reactor_metrics.py b/synapse/metrics/_reactor_metrics.py index c0a4ee16ee..9852d0b932 100644 --- a/synapse/metrics/_reactor_metrics.py +++ b/synapse/metrics/_reactor_metrics.py @@ -33,7 +33,7 @@ from twisted.internet.asyncioreactor import AsyncioSelectorReactor from synapse.metrics._types import Collector try: - from selectors import KqueueSelector + from selectors import KqueueSelector # type: ignore[attr-defined] except ImportError: class KqueueSelector: # type: ignore[no-redef] @@ -62,7 +62,8 @@ logger = logging.getLogger(__name__) # Twisted reactor metrics # -tick_time = Histogram( +# This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. +tick_time = Histogram( # type: ignore[missing-server-name-label] "python_twisted_reactor_tick_time", "Tick time of the Twisted reactor (sec)", buckets=[0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2, 5], @@ -114,7 +115,8 @@ class ReactorLastSeenMetric(Collector): self._call_wrapper = call_wrapper def collect(self) -> Iterable[Metric]: - cm = GaugeMetricFamily( + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + cm = GaugeMetricFamily( # type: ignore[missing-server-name-label] "python_twisted_reactor_last_seen", "Seconds since the Twisted reactor was last seen", ) @@ -165,4 +167,5 @@ except Exception as e: if wrapper: - REGISTRY.register(ReactorLastSeenMetric(wrapper)) + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + REGISTRY.register(ReactorLastSeenMetric(wrapper)) # type: ignore[missing-server-name-label] diff --git a/synapse/metrics/_twisted_exposition.py b/synapse/metrics/_twisted_exposition.py deleted file mode 100644 index 9652ca83fb..0000000000 --- a/synapse/metrics/_twisted_exposition.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2019 Matrix.org Foundation C.I.C. -# Copyright 2015-2019 Prometheus Python Client Developers -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -from prometheus_client import REGISTRY, CollectorRegistry, generate_latest - -from twisted.web.resource import Resource -from twisted.web.server import Request - -CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" - - -class MetricsResource(Resource): - """ - Twisted ``Resource`` that serves prometheus metrics. - """ - - isLeaf = True - - def __init__(self, registry: CollectorRegistry = REGISTRY): - self.registry = registry - - def render_GET(self, request: Request) -> bytes: - request.setHeader(b"Content-Type", CONTENT_TYPE_LATEST.encode("ascii")) - response = generate_latest(self.registry) - request.setHeader(b"Content-Length", str(len(response))) - return response diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 49d0ff9fc1..633705b02a 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -31,6 +31,7 @@ from typing import ( Dict, Iterable, Optional, + Protocol, Set, Type, TypeVar, @@ -39,7 +40,7 @@ from typing import ( from prometheus_client import Metric from prometheus_client.core import REGISTRY, Counter, Gauge -from typing_extensions import ParamSpec +from typing_extensions import Concatenate, ParamSpec from twisted.internet import defer @@ -49,6 +50,7 @@ from synapse.logging.context import ( PreserveLoggingContext, ) from synapse.logging.opentracing import SynapseTags, start_active_span +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics._types import Collector if TYPE_CHECKING: @@ -64,13 +66,13 @@ logger = logging.getLogger(__name__) _background_process_start_count = Counter( "synapse_background_process_start_count", "Number of background processes started", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], ) _background_process_in_flight_count = Gauge( "synapse_background_process_in_flight_count", "Number of background processes in flight", - labelnames=["name"], + labelnames=["name", SERVER_NAME_LABEL], ) # we set registry=None in all of these to stop them getting registered with @@ -80,21 +82,21 @@ _background_process_in_flight_count = Gauge( _background_process_ru_utime = Counter( "synapse_background_process_ru_utime_seconds", "User CPU time used by background processes, in seconds", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=None, ) _background_process_ru_stime = Counter( "synapse_background_process_ru_stime_seconds", "System CPU time used by background processes, in seconds", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=None, ) _background_process_db_txn_count = Counter( "synapse_background_process_db_txn_count", "Number of database transactions done by background processes", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=None, ) @@ -104,14 +106,14 @@ _background_process_db_txn_duration = Counter( "Seconds spent by background processes waiting for database " "transactions, excluding scheduling time" ), - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=None, ) _background_process_db_sched_duration = Counter( "synapse_background_process_db_sched_duration_seconds", "Seconds spent by background processes waiting for database connections", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=None, ) @@ -165,12 +167,15 @@ class _Collector(Collector): yield from m.collect() -REGISTRY.register(_Collector()) +# The `SERVER_NAME_LABEL` is included in the individual metrics added to this registry, +# so we don't need to worry about it on the collector itself. +REGISTRY.register(_Collector()) # type: ignore[missing-server-name-label] class _BackgroundProcess: - def __init__(self, desc: str, ctx: LoggingContext): + def __init__(self, *, desc: str, server_name: str, ctx: LoggingContext): self.desc = desc + self.server_name = server_name self._context = ctx self._reported_stats: Optional[ContextResourceUsage] = None @@ -185,15 +190,21 @@ class _BackgroundProcess: # For unknown reasons, the difference in times can be negative. See comment in # synapse.http.request_metrics.RequestMetrics.update_metrics. - _background_process_ru_utime.labels(self.desc).inc(max(diff.ru_utime, 0)) - _background_process_ru_stime.labels(self.desc).inc(max(diff.ru_stime, 0)) - _background_process_db_txn_count.labels(self.desc).inc(diff.db_txn_count) - _background_process_db_txn_duration.labels(self.desc).inc( - diff.db_txn_duration_sec - ) - _background_process_db_sched_duration.labels(self.desc).inc( - diff.db_sched_duration_sec - ) + _background_process_ru_utime.labels( + name=self.desc, **{SERVER_NAME_LABEL: self.server_name} + ).inc(max(diff.ru_utime, 0)) + _background_process_ru_stime.labels( + name=self.desc, **{SERVER_NAME_LABEL: self.server_name} + ).inc(max(diff.ru_stime, 0)) + _background_process_db_txn_count.labels( + name=self.desc, **{SERVER_NAME_LABEL: self.server_name} + ).inc(diff.db_txn_count) + _background_process_db_txn_duration.labels( + name=self.desc, **{SERVER_NAME_LABEL: self.server_name} + ).inc(diff.db_txn_duration_sec) + _background_process_db_sched_duration.labels( + name=self.desc, **{SERVER_NAME_LABEL: self.server_name} + ).inc(diff.db_sched_duration_sec) R = TypeVar("R") @@ -201,6 +212,7 @@ R = TypeVar("R") def run_as_background_process( desc: "LiteralString", + server_name: str, func: Callable[..., Awaitable[Optional[R]]], *args: Any, bg_start_span: bool = True, @@ -216,8 +228,15 @@ def run_as_background_process( clock.looping_call and friends (or for firing-and-forgetting in the middle of a normal synapse async function). + Because the returned Deferred does not follow the synapse logcontext rules, awaiting + the result of this function will result in the log context being cleared (bad). In + order to properly await the result of this function and maintain the current log + context, use `make_deferred_yieldable`. + Args: desc: a description for this background process type + server_name: The homeserver name that this background process is being run for + (this should be `hs.hostname`). func: a function, which may return a Deferred or a coroutine bg_start_span: Whether to start an opentracing span. Defaults to True. Should only be disabled for processes that will not log to or tag @@ -236,10 +255,16 @@ def run_as_background_process( count = _background_process_counts.get(desc, 0) _background_process_counts[desc] = count + 1 - _background_process_start_count.labels(desc).inc() - _background_process_in_flight_count.labels(desc).inc() + _background_process_start_count.labels( + name=desc, **{SERVER_NAME_LABEL: server_name} + ).inc() + _background_process_in_flight_count.labels( + name=desc, **{SERVER_NAME_LABEL: server_name} + ).inc() - with BackgroundProcessLoggingContext(desc, count) as context: + with BackgroundProcessLoggingContext( + name=desc, server_name=server_name, instance_id=count + ) as context: try: if bg_start_span: ctx = start_active_span( @@ -256,8 +281,24 @@ def run_as_background_process( ) return None finally: - _background_process_in_flight_count.labels(desc).dec() + _background_process_in_flight_count.labels( + name=desc, **{SERVER_NAME_LABEL: server_name} + ).dec() + # To explain how the log contexts work here: + # - When `run_as_background_process` is called, the current context is stored + # (using `PreserveLoggingContext`), we kick off the background task, and we + # restore the original context before returning (also part of + # `PreserveLoggingContext`). + # - The background task runs in its own new logcontext named after `desc` + # - When the background task finishes, we don't want to leak our background context + # into the reactor which would erroneously get attached to the next operation + # picked up by the event loop. We use `PreserveLoggingContext` to set the + # `sentinel` context and means the new `BackgroundProcessLoggingContext` will + # remember the `sentinel` context as its previous context to return to when it + # exits and yields control back to the reactor. + # + # TODO: Why can't we simplify to using `return run_in_background(run)`? with PreserveLoggingContext(): # Note that we return a Deferred here so that it can be used in a # looping_call and other places that expect a Deferred. @@ -267,6 +308,14 @@ def run_as_background_process( P = ParamSpec("P") +class HasServerName(Protocol): + server_name: str + """ + The homeserver name that this cache is associated with (used to label the metric) + (`hs.hostname`). + """ + + def wrap_as_background_process( desc: "LiteralString", ) -> Callable[ @@ -292,22 +341,37 @@ def wrap_as_background_process( multiple places. """ - def wrap_as_background_process_inner( - func: Callable[P, Awaitable[Optional[R]]], + def wrapper( + func: Callable[Concatenate[HasServerName, P], Awaitable[Optional[R]]], ) -> Callable[P, "defer.Deferred[Optional[R]]"]: @wraps(func) - def wrap_as_background_process_inner_2( - *args: P.args, **kwargs: P.kwargs + def wrapped_func( + self: HasServerName, *args: P.args, **kwargs: P.kwargs ) -> "defer.Deferred[Optional[R]]": - # type-ignore: mypy is confusing kwargs with the bg_start_span kwarg. - # Argument 4 to "run_as_background_process" has incompatible type - # "**P.kwargs"; expected "bool" - # See https://github.com/python/mypy/issues/8862 - return run_as_background_process(desc, func, *args, **kwargs) # type: ignore[arg-type] + assert self.server_name is not None, ( + "The `server_name` attribute must be set on the object where `@wrap_as_background_process` decorator is used." + ) - return wrap_as_background_process_inner_2 + return run_as_background_process( + desc, + self.server_name, + func, + self, + *args, + # type-ignore: mypy is confusing kwargs with the bg_start_span kwarg. + # Argument 4 to "run_as_background_process" has incompatible type + # "**P.kwargs"; expected "bool" + # See https://github.com/python/mypy/issues/8862 + **kwargs, # type: ignore[arg-type] + ) - return wrap_as_background_process_inner + # There are some shenanigans here, because we're decorating a method but + # explicitly making use of the `self` parameter. The key thing here is that the + # return type within the return type for `measure_func` itself describes how the + # decorated function will be called. + return wrapped_func # type: ignore[return-value] + + return wrapper # type: ignore[return-value] class BackgroundProcessLoggingContext(LoggingContext): @@ -317,13 +381,20 @@ class BackgroundProcessLoggingContext(LoggingContext): __slots__ = ["_proc"] - def __init__(self, name: str, instance_id: Optional[Union[int, str]] = None): + def __init__( + self, + *, + name: str, + server_name: str, + instance_id: Optional[Union[int, str]] = None, + ): """ Args: name: The name of the background process. Each distinct `name` gets a separate prometheus time series. - + server_name: The homeserver name that this background process is being run for + (this should be `hs.hostname`). instance_id: an identifer to add to `name` to distinguish this instance of the named background process in the logs. If this is `None`, one is made up based on id(self). @@ -331,7 +402,9 @@ class BackgroundProcessLoggingContext(LoggingContext): if instance_id is None: instance_id = id(self) super().__init__("%s-%s" % (name, instance_id)) - self._proc: Optional[_BackgroundProcess] = _BackgroundProcess(name, self) + self._proc: Optional[_BackgroundProcess] = _BackgroundProcess( + desc=name, server_name=server_name, ctx=self + ) def start(self, rusage: "Optional[resource.struct_rusage]") -> None: """Log context has started running (again).""" diff --git a/synapse/metrics/common_usage_metrics.py b/synapse/metrics/common_usage_metrics.py index 970367e9e0..cd1c3c8649 100644 --- a/synapse/metrics/common_usage_metrics.py +++ b/synapse/metrics/common_usage_metrics.py @@ -22,6 +22,7 @@ from typing import TYPE_CHECKING import attr +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process if TYPE_CHECKING: @@ -33,6 +34,7 @@ from prometheus_client import Gauge current_dau_gauge = Gauge( "synapse_admin_daily_active_users", "Current daily active users count", + labelnames=[SERVER_NAME_LABEL], ) @@ -47,6 +49,7 @@ class CommonUsageMetricsManager: """Collects common usage metrics.""" def __init__(self, hs: "HomeServer") -> None: + self.server_name = hs.hostname self._store = hs.get_datastores().main self._clock = hs.get_clock() @@ -62,12 +65,15 @@ class CommonUsageMetricsManager: async def setup(self) -> None: """Keep the gauges for common usage metrics up to date.""" run_as_background_process( - desc="common_usage_metrics_update_gauges", func=self._update_gauges + desc="common_usage_metrics_update_gauges", + server_name=self.server_name, + func=self._update_gauges, ) self._clock.looping_call( run_as_background_process, 5 * 60 * 1000, desc="common_usage_metrics_update_gauges", + server_name=self.server_name, func=self._update_gauges, ) @@ -85,4 +91,6 @@ class CommonUsageMetricsManager: """Update the Prometheus gauges.""" metrics = await self._collect() - current_dau_gauge.set(float(metrics.daily_active_users)) + current_dau_gauge.labels( + **{SERVER_NAME_LABEL: self.server_name}, + ).set(float(metrics.daily_active_users)) diff --git a/synapse/metrics/jemalloc.py b/synapse/metrics/jemalloc.py index bd25985686..fb8adbe060 100644 --- a/synapse/metrics/jemalloc.py +++ b/synapse/metrics/jemalloc.py @@ -23,11 +23,10 @@ import ctypes import logging import os import re -from typing import Iterable, Optional, overload +from typing import Iterable, Literal, Optional, overload import attr from prometheus_client import REGISTRY, Metric -from typing_extensions import Literal from synapse.metrics import GaugeMetricFamily from synapse.metrics._types import Collector @@ -189,7 +188,8 @@ def _setup_jemalloc_stats() -> None: def collect(self) -> Iterable[Metric]: stats.refresh_stats() - g = GaugeMetricFamily( + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + g = GaugeMetricFamily( # type: ignore[missing-server-name-label] "jemalloc_stats_app_memory_bytes", "The stats reported by jemalloc", labels=["type"], @@ -231,7 +231,8 @@ def _setup_jemalloc_stats() -> None: yield g - REGISTRY.register(JemallocCollector()) + # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. + REGISTRY.register(JemallocCollector()) # type: ignore[missing-server-name-label] logger.debug("Added jemalloc stats") diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index f6bfd93d3c..6218135513 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -23,6 +23,7 @@ import logging from typing import ( TYPE_CHECKING, Any, + Awaitable, Callable, Collection, Dict, @@ -45,9 +46,11 @@ from twisted.internet.interfaces import IDelayedCall from twisted.web.resource import Resource from synapse.api import errors +from synapse.api.constants import ProfileFields from synapse.api.errors import SynapseError from synapse.api.presence import UserPresenceState from synapse.config import ConfigError +from synapse.config.repository import MediaUploadLimit from synapse.events import EventBase from synapse.events.presence_router import ( GET_INTERESTED_USERS_CALLBACK, @@ -65,7 +68,6 @@ from synapse.handlers.auth import ( ON_LOGGED_OUT_CALLBACK, AuthHandler, ) -from synapse.handlers.device import DeviceHandler from synapse.handlers.push_rules import RuleSpec, check_actions from synapse.http.client import SimpleHttpClient from synapse.http.server import ( @@ -80,7 +82,9 @@ from synapse.logging.context import ( make_deferred_yieldable, run_in_background, ) -from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.metrics.background_process_metrics import ( + run_as_background_process as _run_as_background_process, +) from synapse.module_api.callbacks.account_validity_callbacks import ( IS_USER_EXPIRED_CALLBACK, ON_LEGACY_ADMIN_REQUEST, @@ -89,12 +93,23 @@ from synapse.module_api.callbacks.account_validity_callbacks import ( ON_USER_LOGIN_CALLBACK, ON_USER_REGISTRATION_CALLBACK, ) +from synapse.module_api.callbacks.media_repository_callbacks import ( + GET_MEDIA_CONFIG_FOR_USER_CALLBACK, + GET_MEDIA_UPLOAD_LIMITS_FOR_USER_CALLBACK, + IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK, + ON_MEDIA_UPLOAD_LIMIT_EXCEEDED_CALLBACK, +) +from synapse.module_api.callbacks.ratelimit_callbacks import ( + GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK, + RatelimitOverride, +) from synapse.module_api.callbacks.spamchecker_callbacks import ( CHECK_EVENT_FOR_SPAM_CALLBACK, CHECK_LOGIN_FOR_SPAM_CALLBACK, CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK, CHECK_REGISTRATION_FOR_SPAM_CALLBACK, CHECK_USERNAME_FOR_SPAM_CALLBACK, + FEDERATED_USER_MAY_INVITE_CALLBACK, SHOULD_DROP_FEDERATED_EVENT_CALLBACK, USER_MAY_CREATE_ROOM_ALIAS_CALLBACK, USER_MAY_CREATE_ROOM_CALLBACK, @@ -102,6 +117,7 @@ from synapse.module_api.callbacks.spamchecker_callbacks import ( USER_MAY_JOIN_ROOM_CALLBACK, USER_MAY_PUBLISH_ROOM_CALLBACK, USER_MAY_SEND_3PID_INVITE_CALLBACK, + USER_MAY_SEND_STATE_EVENT_CALLBACK, SpamCheckerModuleApiCallbacks, ) from synapse.module_api.callbacks.third_party_event_rules_callbacks import ( @@ -148,6 +164,9 @@ from synapse.util.caches.descriptors import CachedFunction, cached as _cached from synapse.util.frozenutils import freeze if TYPE_CHECKING: + # Old versions don't have `LiteralString` + from typing_extensions import LiteralString + from synapse.app.generic_worker import GenericWorkerStore from synapse.server import HomeServer @@ -188,6 +207,8 @@ __all__ = [ "ProfileInfo", "RoomAlias", "UserProfile", + "RatelimitOverride", + "MediaUploadLimit", ] logger = logging.getLogger(__name__) @@ -205,6 +226,65 @@ class UserIpAndAgent: last_seen: int +def run_as_background_process( + desc: "LiteralString", + func: Callable[..., Awaitable[Optional[T]]], + *args: Any, + bg_start_span: bool = True, + **kwargs: Any, +) -> "defer.Deferred[Optional[T]]": + """ + XXX: Deprecated: use `ModuleApi.run_as_background_process` instead. + + Run the given function in its own logcontext, with resource metrics + + This should be used to wrap processes which are fired off to run in the + background, instead of being associated with a particular request. + + It returns a Deferred which completes when the function completes, but it doesn't + follow the synapse logcontext rules, which makes it appropriate for passing to + clock.looping_call and friends (or for firing-and-forgetting in the middle of a + normal synapse async function). + + Args: + desc: a description for this background process type + server_name: The homeserver name that this background process is being run for + (this should be `hs.hostname`). + func: a function, which may return a Deferred or a coroutine + bg_start_span: Whether to start an opentracing span. Defaults to True. + Should only be disabled for processes that will not log to or tag + a span. + args: positional args for func + kwargs: keyword args for func + + Returns: + Deferred which returns the result of func, or `None` if func raises. + Note that the returned Deferred does not follow the synapse logcontext + rules. + """ + + logger.warning( + "Using deprecated `run_as_background_process` that's exported from the Module API. " + "Prefer `ModuleApi.run_as_background_process` instead.", + ) + + # Historically, since this function is exported from the module API, we can't just + # change the signature to require a `server_name` argument. Since + # `run_as_background_process` internally in Synapse requires `server_name` now, we + # just have to stub this out with a placeholder value and tell people to use the new + # function instead. + stub_server_name = "synapse_module_running_from_unknown_server" + + return _run_as_background_process( + desc, + stub_server_name, + func, + *args, + bg_start_span=bg_start_span, + **kwargs, + ) + + def cached( *, max_entries: int = 1000, @@ -266,13 +346,15 @@ class ModuleApi: self._device_handler = hs.get_device_handler() self.custom_template_dir = hs.config.server.custom_template_directory self._callbacks = hs.get_module_api_callbacks() - self.msc3861_oauth_delegation_enabled = hs.config.experimental.msc3861.enabled + self._auth_delegation_enabled = ( + hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + ) self._event_serializer = hs.get_event_client_serializer() try: app_name = self._hs.config.email.email_app_name - self._from_string = self._hs.config.email.email_notif_from % { + self._from_string = self._hs.config.email.email_notif_from % { # type: ignore[operator] "app": app_name } except (KeyError, TypeError): @@ -304,12 +386,14 @@ class ModuleApi: ] = None, user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None, user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None, + federated_user_may_invite: Optional[FEDERATED_USER_MAY_INVITE_CALLBACK] = None, user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None, user_may_create_room: Optional[USER_MAY_CREATE_ROOM_CALLBACK] = None, user_may_create_room_alias: Optional[ USER_MAY_CREATE_ROOM_ALIAS_CALLBACK ] = None, user_may_publish_room: Optional[USER_MAY_PUBLISH_ROOM_CALLBACK] = None, + user_may_send_state_event: Optional[USER_MAY_SEND_STATE_EVENT_CALLBACK] = None, check_username_for_spam: Optional[CHECK_USERNAME_FOR_SPAM_CALLBACK] = None, check_registration_for_spam: Optional[ CHECK_REGISTRATION_FOR_SPAM_CALLBACK @@ -326,6 +410,7 @@ class ModuleApi: should_drop_federated_event=should_drop_federated_event, user_may_join_room=user_may_join_room, user_may_invite=user_may_invite, + federated_user_may_invite=federated_user_may_invite, user_may_send_3pid_invite=user_may_send_3pid_invite, user_may_create_room=user_may_create_room, user_may_create_room_alias=user_may_create_room_alias, @@ -334,6 +419,7 @@ class ModuleApi: check_registration_for_spam=check_registration_for_spam, check_media_file_for_spam=check_media_file_for_spam, check_login_for_spam=check_login_for_spam, + user_may_send_state_event=user_may_send_state_event, ) def register_account_validity_callbacks( @@ -359,6 +445,44 @@ class ModuleApi: on_legacy_admin_request=on_legacy_admin_request, ) + def register_ratelimit_callbacks( + self, + *, + get_ratelimit_override_for_user: Optional[ + GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK + ] = None, + ) -> None: + """Registers callbacks for ratelimit capabilities. + Added in Synapse v1.132.0. + """ + return self._callbacks.ratelimit.register_callbacks( + get_ratelimit_override_for_user=get_ratelimit_override_for_user, + ) + + def register_media_repository_callbacks( + self, + *, + get_media_config_for_user: Optional[GET_MEDIA_CONFIG_FOR_USER_CALLBACK] = None, + is_user_allowed_to_upload_media_of_size: Optional[ + IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK + ] = None, + get_media_upload_limits_for_user: Optional[ + GET_MEDIA_UPLOAD_LIMITS_FOR_USER_CALLBACK + ] = None, + on_media_upload_limit_exceeded: Optional[ + ON_MEDIA_UPLOAD_LIMIT_EXCEEDED_CALLBACK + ] = None, + ) -> None: + """Registers callbacks for media repository capabilities. + Added in Synapse v1.132.0. + """ + return self._callbacks.media_repository.register_callbacks( + get_media_config_for_user=get_media_config_for_user, + is_user_allowed_to_upload_media_of_size=is_user_allowed_to_upload_media_of_size, + get_media_upload_limits_for_user=get_media_upload_limits_for_user, + on_media_upload_limit_exceeded=on_media_upload_limit_exceeded, + ) + def register_third_party_rules_callbacks( self, *, @@ -439,7 +563,7 @@ class ModuleApi: Added in Synapse v1.46.0. """ - if self.msc3861_oauth_delegation_enabled: + if self._auth_delegation_enabled: raise ConfigError( "Cannot use password auth provider callbacks when OAuth delegation is enabled" ) @@ -649,7 +773,7 @@ class ModuleApi: Returns: True if the user is a server admin, False otherwise. """ - return await self._store.is_server_admin(UserID.from_string(user_id)) + return await self._store.is_server_admin(user_id) async def set_user_admin(self, user_id: str, admin: bool) -> None: """Sets if a user is a server admin. @@ -879,8 +1003,6 @@ class ModuleApi: ) -> Generator["defer.Deferred[Any]", Any, None]: """Invalidate an access token for a user - Can only be called from the main process. - Added in Synapse v0.25.0. Args: @@ -893,10 +1015,6 @@ class ModuleApi: Raises: synapse.api.errors.AuthError: the access token is invalid """ - assert isinstance( - self._device_handler, DeviceHandler - ), "invalidate_access_token can only be called on the main process" - # see if the access token corresponds to a device user_info = yield defer.ensureDeferred( self._auth.get_user_by_access_token(access_token) @@ -1086,7 +1204,10 @@ class ModuleApi: content = {} # Set the profile if not already done by the module. - if "avatar_url" not in content or "displayname" not in content: + if ( + ProfileFields.AVATAR_URL not in content + or ProfileFields.DISPLAYNAME not in content + ): try: # Try to fetch the user's profile. profile = await self._hs.get_profile_handler().get_profile( @@ -1095,8 +1216,8 @@ class ModuleApi: except SynapseError as e: # If the profile couldn't be found, use default values. profile = { - "displayname": target_user_id.localpart, - "avatar_url": None, + ProfileFields.DISPLAYNAME: target_user_id.localpart, + ProfileFields.AVATAR_URL: None, } if e.code != 404: @@ -1109,11 +1230,9 @@ class ModuleApi: ) # Set the profile where it needs to be set. - if "avatar_url" not in content: - content["avatar_url"] = profile["avatar_url"] - - if "displayname" not in content: - content["displayname"] = profile["displayname"] + for field_name in [ProfileFields.AVATAR_URL, ProfileFields.DISPLAYNAME]: + if field_name not in content and field_name in profile: + content[field_name] = profile[field_name] event_id, _ = await self._hs.get_room_member_handler().update_membership( requester=requester, @@ -1283,7 +1402,7 @@ class ModuleApi: if self._hs.config.worker.run_background_tasks or run_on_all_instances: self._clock.looping_call( - run_as_background_process, + self.run_as_background_process, msec, desc, lambda: maybe_awaitable(f(*args, **kwargs)), @@ -1341,7 +1460,7 @@ class ModuleApi: return self._clock.call_later( # convert ms to seconds as needed by call_later. msec * 0.001, - run_as_background_process, + self.run_as_background_process, desc, lambda: maybe_awaitable(f(*args, **kwargs)), ) @@ -1548,6 +1667,44 @@ class ModuleApi: return {key: state_events[event_id] for key, event_id in state_ids.items()} + def run_as_background_process( + self, + desc: "LiteralString", + func: Callable[..., Awaitable[Optional[T]]], + *args: Any, + bg_start_span: bool = True, + **kwargs: Any, + ) -> "defer.Deferred[Optional[T]]": + """Run the given function in its own logcontext, with resource metrics + + This should be used to wrap processes which are fired off to run in the + background, instead of being associated with a particular request. + + It returns a Deferred which completes when the function completes, but it doesn't + follow the synapse logcontext rules, which makes it appropriate for passing to + clock.looping_call and friends (or for firing-and-forgetting in the middle of a + normal synapse async function). + + Args: + desc: a description for this background process type + server_name: The homeserver name that this background process is being run for + (this should be `hs.hostname`). + func: a function, which may return a Deferred or a coroutine + bg_start_span: Whether to start an opentracing span. Defaults to True. + Should only be disabled for processes that will not log to or tag + a span. + args: positional args for func + kwargs: keyword args for func + + Returns: + Deferred which returns the result of func, or `None` if func raises. + Note that the returned Deferred does not follow the synapse logcontext + rules. + """ + return _run_as_background_process( + desc, self.server_name, func, *args, bg_start_span=bg_start_span, **kwargs + ) + async def defer_to_thread( self, f: Callable[P, T], @@ -1844,6 +2001,10 @@ class ModuleApi: deactivation=deactivation, ) + def get_current_time_msec(self) -> int: + """Returns the current server time in milliseconds.""" + return self._clock.time_msec() + class PublicRoomListManager: """Contains methods for adding to, removing from and querying whether a room diff --git a/synapse/module_api/callbacks/__init__.py b/synapse/module_api/callbacks/__init__.py index c20d9543fb..16ef7a4b47 100644 --- a/synapse/module_api/callbacks/__init__.py +++ b/synapse/module_api/callbacks/__init__.py @@ -27,6 +27,12 @@ if TYPE_CHECKING: from synapse.module_api.callbacks.account_validity_callbacks import ( AccountValidityModuleApiCallbacks, ) +from synapse.module_api.callbacks.media_repository_callbacks import ( + MediaRepositoryModuleApiCallbacks, +) +from synapse.module_api.callbacks.ratelimit_callbacks import ( + RatelimitModuleApiCallbacks, +) from synapse.module_api.callbacks.spamchecker_callbacks import ( SpamCheckerModuleApiCallbacks, ) @@ -38,5 +44,7 @@ from synapse.module_api.callbacks.third_party_event_rules_callbacks import ( class ModuleApiCallbacks: def __init__(self, hs: "HomeServer") -> None: self.account_validity = AccountValidityModuleApiCallbacks() + self.media_repository = MediaRepositoryModuleApiCallbacks(hs) + self.ratelimit = RatelimitModuleApiCallbacks(hs) self.spam_checker = SpamCheckerModuleApiCallbacks(hs) self.third_party_event_rules = ThirdPartyEventRulesModuleApiCallbacks(hs) diff --git a/synapse/module_api/callbacks/media_repository_callbacks.py b/synapse/module_api/callbacks/media_repository_callbacks.py new file mode 100644 index 0000000000..7d3aed9d66 --- /dev/null +++ b/synapse/module_api/callbacks/media_repository_callbacks.py @@ -0,0 +1,160 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional + +from synapse.config.repository import MediaUploadLimit +from synapse.types import JsonDict +from synapse.util.async_helpers import delay_cancellation +from synapse.util.metrics import Measure + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + +GET_MEDIA_CONFIG_FOR_USER_CALLBACK = Callable[[str], Awaitable[Optional[JsonDict]]] + +IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK = Callable[[str, int], Awaitable[bool]] + +GET_MEDIA_UPLOAD_LIMITS_FOR_USER_CALLBACK = Callable[ + [str], Awaitable[Optional[List[MediaUploadLimit]]] +] + +ON_MEDIA_UPLOAD_LIMIT_EXCEEDED_CALLBACK = Callable[ + [str, MediaUploadLimit, int, int], Awaitable[None] +] + + +class MediaRepositoryModuleApiCallbacks: + def __init__(self, hs: "HomeServer") -> None: + self.server_name = hs.hostname + self.clock = hs.get_clock() + self._get_media_config_for_user_callbacks: List[ + GET_MEDIA_CONFIG_FOR_USER_CALLBACK + ] = [] + self._is_user_allowed_to_upload_media_of_size_callbacks: List[ + IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK + ] = [] + self._get_media_upload_limits_for_user_callbacks: List[ + GET_MEDIA_UPLOAD_LIMITS_FOR_USER_CALLBACK + ] = [] + self._on_media_upload_limit_exceeded_callbacks: List[ + ON_MEDIA_UPLOAD_LIMIT_EXCEEDED_CALLBACK + ] = [] + + def register_callbacks( + self, + get_media_config_for_user: Optional[GET_MEDIA_CONFIG_FOR_USER_CALLBACK] = None, + is_user_allowed_to_upload_media_of_size: Optional[ + IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK + ] = None, + get_media_upload_limits_for_user: Optional[ + GET_MEDIA_UPLOAD_LIMITS_FOR_USER_CALLBACK + ] = None, + on_media_upload_limit_exceeded: Optional[ + ON_MEDIA_UPLOAD_LIMIT_EXCEEDED_CALLBACK + ] = None, + ) -> None: + """Register callbacks from module for each hook.""" + if get_media_config_for_user is not None: + self._get_media_config_for_user_callbacks.append(get_media_config_for_user) + + if is_user_allowed_to_upload_media_of_size is not None: + self._is_user_allowed_to_upload_media_of_size_callbacks.append( + is_user_allowed_to_upload_media_of_size + ) + + if get_media_upload_limits_for_user is not None: + self._get_media_upload_limits_for_user_callbacks.append( + get_media_upload_limits_for_user + ) + + if on_media_upload_limit_exceeded is not None: + self._on_media_upload_limit_exceeded_callbacks.append( + on_media_upload_limit_exceeded + ) + + async def get_media_config_for_user(self, user_id: str) -> Optional[JsonDict]: + for callback in self._get_media_config_for_user_callbacks: + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): + res: Optional[JsonDict] = await delay_cancellation(callback(user_id)) + if res: + return res + + return None + + async def is_user_allowed_to_upload_media_of_size( + self, user_id: str, size: int + ) -> bool: + for callback in self._is_user_allowed_to_upload_media_of_size_callbacks: + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): + res: bool = await delay_cancellation(callback(user_id, size)) + if not res: + return res + + return True + + async def get_media_upload_limits_for_user( + self, user_id: str + ) -> Optional[List[MediaUploadLimit]]: + """ + Get the first non-None list of MediaUploadLimits for the user from the registered callbacks. + If a list is returned it will be sorted in descending order of duration. + """ + for callback in self._get_media_upload_limits_for_user_callbacks: + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): + res: Optional[List[MediaUploadLimit]] = await delay_cancellation( + callback(user_id) + ) + if res is not None: # to allow [] to be returned meaning no limit + # We sort them in descending order of time period + res.sort(key=lambda limit: limit.time_period_ms, reverse=True) + return res + + return None + + async def on_media_upload_limit_exceeded( + self, + user_id: str, + limit: MediaUploadLimit, + sent_bytes: int, + attempted_bytes: int, + ) -> None: + for callback in self._on_media_upload_limit_exceeded_callbacks: + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): + # Use a copy of the data in case the module modifies it + limit_copy = MediaUploadLimit( + max_bytes=limit.max_bytes, time_period_ms=limit.time_period_ms + ) + await delay_cancellation( + callback(user_id, limit_copy, sent_bytes, attempted_bytes) + ) diff --git a/synapse/module_api/callbacks/ratelimit_callbacks.py b/synapse/module_api/callbacks/ratelimit_callbacks.py new file mode 100644 index 0000000000..a580ea7d7c --- /dev/null +++ b/synapse/module_api/callbacks/ratelimit_callbacks.py @@ -0,0 +1,79 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional + +import attr + +from synapse.util.async_helpers import delay_cancellation +from synapse.util.metrics import Measure + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +@attr.s(auto_attribs=True) +class RatelimitOverride: + """Represents a ratelimit being overridden.""" + + per_second: float + """The number of actions that can be performed in a second. `0.0` means that ratelimiting is disabled.""" + burst_count: int + """How many actions that can be performed before being limited.""" + + +GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK = Callable[ + [str, str], Awaitable[Optional[RatelimitOverride]] +] + + +class RatelimitModuleApiCallbacks: + def __init__(self, hs: "HomeServer") -> None: + self.server_name = hs.hostname + self.clock = hs.get_clock() + self._get_ratelimit_override_for_user_callbacks: List[ + GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK + ] = [] + + def register_callbacks( + self, + get_ratelimit_override_for_user: Optional[ + GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK + ] = None, + ) -> None: + """Register callbacks from module for each hook.""" + if get_ratelimit_override_for_user is not None: + self._get_ratelimit_override_for_user_callbacks.append( + get_ratelimit_override_for_user + ) + + async def get_ratelimit_override_for_user( + self, user_id: str, limiter_name: str + ) -> Optional[RatelimitOverride]: + for callback in self._get_ratelimit_override_for_user_callbacks: + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): + res: Optional[RatelimitOverride] = await delay_cancellation( + callback(user_id, limiter_name) + ) + if res: + return res + + return None diff --git a/synapse/module_api/callbacks/spamchecker_callbacks.py b/synapse/module_api/callbacks/spamchecker_callbacks.py index 17079ff781..428e733979 100644 --- a/synapse/module_api/callbacks/spamchecker_callbacks.py +++ b/synapse/module_api/callbacks/spamchecker_callbacks.py @@ -19,8 +19,10 @@ # # +import functools import inspect import logging +from copy import deepcopy from typing import ( TYPE_CHECKING, Any, @@ -28,14 +30,13 @@ from typing import ( Callable, Collection, List, + Literal, Optional, Tuple, Union, + cast, ) -# `Literal` appears with Python 3.8. -from typing_extensions import Literal - import synapse from synapse.api.errors import Codes from synapse.logging.opentracing import trace @@ -104,6 +105,22 @@ USER_MAY_INVITE_CALLBACK = Callable[ ] ], ] +FEDERATED_USER_MAY_INVITE_CALLBACK = Callable[ + ["synapse.events.EventBase"], + Awaitable[ + Union[ + Literal["NOT_SPAM"], + Codes, + # Highly experimental, not officially part of the spamchecker API, may + # disappear without warning depending on the results of ongoing + # experiments. + # Use this to return additional information as part of an error. + Tuple[Codes, JsonDict], + # Deprecated + bool, + ] + ], +] USER_MAY_SEND_3PID_INVITE_CALLBACK = Callable[ [str, str, str, str], Awaitable[ @@ -120,20 +137,24 @@ USER_MAY_SEND_3PID_INVITE_CALLBACK = Callable[ ] ], ] -USER_MAY_CREATE_ROOM_CALLBACK = Callable[ - [str], - Awaitable[ - Union[ - Literal["NOT_SPAM"], - Codes, - # Highly experimental, not officially part of the spamchecker API, may - # disappear without warning depending on the results of ongoing - # experiments. - # Use this to return additional information as part of an error. - Tuple[Codes, JsonDict], - # Deprecated - bool, - ] +USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE = Union[ + Literal["NOT_SPAM"], + Codes, + # Highly experimental, not officially part of the spamchecker API, may + # disappear without warning depending on the results of ongoing + # experiments. + # Use this to return additional information as part of an error. + Tuple[Codes, JsonDict], + # Deprecated + bool, +] +USER_MAY_CREATE_ROOM_CALLBACK = Union[ + Callable[ + [str, JsonDict], + Awaitable[USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE], + ], + Callable[ # Single argument variant for backwards compatibility + [str], Awaitable[USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE] ], ] USER_MAY_CREATE_ROOM_ALIAS_CALLBACK = Callable[ @@ -168,7 +189,24 @@ USER_MAY_PUBLISH_ROOM_CALLBACK = Callable[ ] ], ] -CHECK_USERNAME_FOR_SPAM_CALLBACK = Callable[[UserProfile], Awaitable[bool]] +USER_MAY_SEND_STATE_EVENT_CALLBACK = Callable[ + [str, str, str, str, JsonDict], + Awaitable[ + Union[ + Literal["NOT_SPAM"], + Codes, + # Highly experimental, not officially part of the spamchecker API, may + # disappear without warning depending on the results of ongoing + # experiments. + # Use this to return additional information as part of an error. + Tuple[Codes, JsonDict], + ] + ], +] +CHECK_USERNAME_FOR_SPAM_CALLBACK = Union[ + Callable[[UserProfile], Awaitable[bool]], + Callable[[UserProfile, str], Awaitable[bool]], +] LEGACY_CHECK_REGISTRATION_FOR_SPAM_CALLBACK = Callable[ [ Optional[dict], @@ -244,6 +282,7 @@ def load_legacy_spam_checkers(hs: "synapse.server.HomeServer") -> None: spam_checker_methods = { "check_event_for_spam", "user_may_invite", + "federated_user_may_invite", "user_may_create_room", "user_may_create_room_alias", "user_may_publish_room", @@ -293,6 +332,7 @@ def load_legacy_spam_checkers(hs: "synapse.server.HomeServer") -> None: "Bad signature for callback check_registration_for_spam", ) + @functools.wraps(wrapped_func) def run(*args: Any, **kwargs: Any) -> Awaitable: # Assertion required because mypy can't prove we won't change `f` # back to `None`. See @@ -316,6 +356,7 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM: Literal["NOT_SPAM"] = "NOT_SPAM" def __init__(self, hs: "synapse.server.HomeServer") -> None: + self.server_name = hs.hostname self.clock = hs.get_clock() self._check_event_for_spam_callbacks: List[CHECK_EVENT_FOR_SPAM_CALLBACK] = [] @@ -324,10 +365,16 @@ class SpamCheckerModuleApiCallbacks: ] = [] self._user_may_join_room_callbacks: List[USER_MAY_JOIN_ROOM_CALLBACK] = [] self._user_may_invite_callbacks: List[USER_MAY_INVITE_CALLBACK] = [] + self._federated_user_may_invite_callbacks: List[ + FEDERATED_USER_MAY_INVITE_CALLBACK + ] = [] self._user_may_send_3pid_invite_callbacks: List[ USER_MAY_SEND_3PID_INVITE_CALLBACK ] = [] self._user_may_create_room_callbacks: List[USER_MAY_CREATE_ROOM_CALLBACK] = [] + self._user_may_send_state_event_callbacks: List[ + USER_MAY_SEND_STATE_EVENT_CALLBACK + ] = [] self._user_may_create_room_alias_callbacks: List[ USER_MAY_CREATE_ROOM_ALIAS_CALLBACK ] = [] @@ -351,6 +398,7 @@ class SpamCheckerModuleApiCallbacks: ] = None, user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None, user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None, + federated_user_may_invite: Optional[FEDERATED_USER_MAY_INVITE_CALLBACK] = None, user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None, user_may_create_room: Optional[USER_MAY_CREATE_ROOM_CALLBACK] = None, user_may_create_room_alias: Optional[ @@ -363,6 +411,7 @@ class SpamCheckerModuleApiCallbacks: ] = None, check_media_file_for_spam: Optional[CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK] = None, check_login_for_spam: Optional[CHECK_LOGIN_FOR_SPAM_CALLBACK] = None, + user_may_send_state_event: Optional[USER_MAY_SEND_STATE_EVENT_CALLBACK] = None, ) -> None: """Register callbacks from module for each hook.""" if check_event_for_spam is not None: @@ -379,6 +428,11 @@ class SpamCheckerModuleApiCallbacks: if user_may_invite is not None: self._user_may_invite_callbacks.append(user_may_invite) + if federated_user_may_invite is not None: + self._federated_user_may_invite_callbacks.append( + federated_user_may_invite, + ) + if user_may_send_3pid_invite is not None: self._user_may_send_3pid_invite_callbacks.append( user_may_send_3pid_invite, @@ -387,6 +441,11 @@ class SpamCheckerModuleApiCallbacks: if user_may_create_room is not None: self._user_may_create_room_callbacks.append(user_may_create_room) + if user_may_send_state_event is not None: + self._user_may_send_state_event_callbacks.append( + user_may_send_state_event, + ) + if user_may_create_room_alias is not None: self._user_may_create_room_alias_callbacks.append( user_may_create_room_alias, @@ -432,7 +491,11 @@ class SpamCheckerModuleApiCallbacks: generally discouraged as it doesn't support internationalization. """ for callback in self._check_event_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(event)) if res is False or res == self.NOT_SPAM: # This spam-checker accepts the event. @@ -485,7 +548,11 @@ class SpamCheckerModuleApiCallbacks: True if the event should be silently dropped """ for callback in self._should_drop_federated_event_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res: Union[bool, str] = await delay_cancellation(callback(event)) if res: return res @@ -507,7 +574,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, [Codes, Dict] otherwise. """ for callback in self._user_may_join_room_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(user_id, room_id, is_invited)) # Normalize return values to `Codes` or `"NOT_SPAM"`. if res is True or res is self.NOT_SPAM: @@ -546,7 +617,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, Codes otherwise. """ for callback in self._user_may_invite_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation( callback(inviter_userid, invitee_userid, room_id) ) @@ -573,6 +648,47 @@ class SpamCheckerModuleApiCallbacks: # No spam-checker has rejected the request, let it pass. return self.NOT_SPAM + async def federated_user_may_invite( + self, event: "synapse.events.EventBase" + ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: + """Checks if a given user may send an invite + + Args: + event: The event to be checked + + Returns: + NOT_SPAM if the operation is permitted, Codes otherwise. + """ + for callback in self._federated_user_may_invite_callbacks: + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): + res = await delay_cancellation(callback(event)) + # Normalize return values to `Codes` or `"NOT_SPAM"`. + if res is True or res is self.NOT_SPAM: + continue + elif res is False: + return synapse.api.errors.Codes.FORBIDDEN, {} + elif isinstance(res, synapse.api.errors.Codes): + return res, {} + elif ( + isinstance(res, tuple) + and len(res) == 2 + and isinstance(res[0], synapse.api.errors.Codes) + and isinstance(res[1], dict) + ): + return res + else: + logger.warning( + "Module returned invalid value, rejecting invite as spam" + ) + return synapse.api.errors.Codes.FORBIDDEN, {} + + # Check the standard user_may_invite callback if no module has rejected the invite yet. + return await self.user_may_invite(event.sender, event.state_key, event.room_id) + async def user_may_send_3pid_invite( self, inviter_userid: str, medium: str, address: str, room_id: str ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: @@ -591,7 +707,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, Codes otherwise. """ for callback in self._user_may_send_3pid_invite_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation( callback(inviter_userid, medium, address, room_id) ) @@ -618,16 +738,45 @@ class SpamCheckerModuleApiCallbacks: return self.NOT_SPAM async def user_may_create_room( - self, userid: str + self, userid: str, room_config: JsonDict ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: """Checks if a given user may create a room Args: userid: The ID of the user attempting to create a room + room_config: The room creation configuration which is the body of the /createRoom request """ for callback in self._user_may_create_room_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): - res = await delay_cancellation(callback(userid)) + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): + checker_args = inspect.signature(callback) + # Also ensure backwards compatibility with spam checker callbacks + # that don't expect the room_config argument. + if len(checker_args.parameters) == 2: + callback_with_requester_id = cast( + Callable[ + [str, JsonDict], + Awaitable[USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE], + ], + callback, + ) + # We make a copy of the config to ensure the spam checker cannot modify it. + res = await delay_cancellation( + callback_with_requester_id(userid, deepcopy(room_config)) + ) + else: + callback_without_requester_id = cast( + Callable[ + [str], Awaitable[USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE] + ], + callback, + ) + res = await delay_cancellation( + callback_without_requester_id(userid) + ) if res is True or res is self.NOT_SPAM: continue elif res is False: @@ -649,6 +798,44 @@ class SpamCheckerModuleApiCallbacks: return self.NOT_SPAM + async def user_may_send_state_event( + self, + user_id: str, + room_id: str, + event_type: str, + state_key: str, + content: JsonDict, + ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: + """Checks if a given user may create a room with a given visibility + Args: + user_id: The ID of the user attempting to create a room + room_id: The ID of the room that the event will be sent to + event_type: The type of the state event + state_key: The state key of the state event + content: The content of the state event + """ + for callback in self._user_may_send_state_event_callbacks: + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): + # We make a copy of the content to ensure that the spam checker cannot modify it. + res = await delay_cancellation( + callback(user_id, room_id, event_type, state_key, deepcopy(content)) + ) + if res is self.NOT_SPAM: + continue + elif isinstance(res, synapse.api.errors.Codes): + return res, {} + else: + logger.warning( + "Module returned invalid value, rejecting room creation as spam" + ) + return synapse.api.errors.Codes.FORBIDDEN, {} + + return self.NOT_SPAM + async def user_may_create_room_alias( self, userid: str, room_alias: RoomAlias ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: @@ -660,7 +847,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._user_may_create_room_alias_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(userid, room_alias)) if res is True or res is self.NOT_SPAM: continue @@ -693,7 +884,11 @@ class SpamCheckerModuleApiCallbacks: room_id: The ID of the room that would be published """ for callback in self._user_may_publish_room_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(userid, room_id)) if res is True or res is self.NOT_SPAM: continue @@ -716,7 +911,9 @@ class SpamCheckerModuleApiCallbacks: return self.NOT_SPAM - async def check_username_for_spam(self, user_profile: UserProfile) -> bool: + async def check_username_for_spam( + self, user_profile: UserProfile, requester_id: str + ) -> bool: """Checks if a user ID or display name are considered "spammy" by this server. If the server considers a username spammy, then it will not be included in @@ -727,15 +924,37 @@ class SpamCheckerModuleApiCallbacks: * user_id * display_name * avatar_url + requester_id: The user ID of the user making the user directory search request. Returns: True if the user is spammy. """ for callback in self._check_username_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): + checker_args = inspect.signature(callback) # Make a copy of the user profile object to ensure the spam checker cannot # modify it. - res = await delay_cancellation(callback(user_profile.copy())) + # Also ensure backwards compatibility with spam checker callbacks + # that don't expect the requester_id argument. + if len(checker_args.parameters) == 2: + callback_with_requester_id = cast( + Callable[[UserProfile, str], Awaitable[bool]], callback + ) + res = await delay_cancellation( + callback_with_requester_id(user_profile.copy(), requester_id) + ) + else: + callback_without_requester_id = cast( + Callable[[UserProfile], Awaitable[bool]], callback + ) + res = await delay_cancellation( + callback_without_requester_id(user_profile.copy()) + ) + if res: return True @@ -764,7 +983,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._check_registration_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): behaviour = await delay_cancellation( callback(email_threepid, username, request_info, auth_provider_id) ) @@ -806,7 +1029,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._check_media_file_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(file_wrapper, file_info)) # Normalize return values to `Codes` or `"NOT_SPAM"`. if res is False or res is self.NOT_SPAM: @@ -853,7 +1080,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._check_login_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation( callback( user_id, diff --git a/synapse/notifier.py b/synapse/notifier.py index 88f531182a..e684df4866 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -29,6 +29,7 @@ from typing import ( Iterable, List, Literal, + Mapping, Optional, Set, Tuple, @@ -50,7 +51,7 @@ from synapse.handlers.presence import format_user_presence_state from synapse.logging import issue9533_logger from synapse.logging.context import PreserveLoggingContext from synapse.logging.opentracing import log_kv, start_active_span -from synapse.metrics import LaterGauge +from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.streams.config import PaginationConfig from synapse.types import ( ISynapseReactor, @@ -66,7 +67,6 @@ from synapse.types import ( from synapse.util.async_helpers import ( timeout_deferred, ) -from synapse.util.metrics import Measure from synapse.util.stringutils import shortstr from synapse.visibility import filter_events_for_client @@ -75,10 +75,33 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -notified_events_counter = Counter("synapse_notifier_notified_events", "") +# FIXME: Unused metric, remove if not needed. +notified_events_counter = Counter( + "synapse_notifier_notified_events", "", labelnames=[SERVER_NAME_LABEL] +) users_woken_by_stream_counter = Counter( - "synapse_notifier_users_woken_by_stream", "", ["stream"] + "synapse_notifier_users_woken_by_stream", + "", + labelnames=["stream", SERVER_NAME_LABEL], +) + + +notifier_listeners_gauge = LaterGauge( + name="synapse_notifier_listeners", + desc="", + labelnames=[SERVER_NAME_LABEL], +) + +notifier_rooms_gauge = LaterGauge( + name="synapse_notifier_rooms", + desc="", + labelnames=[SERVER_NAME_LABEL], +) +notifier_users_gauge = LaterGauge( + name="synapse_notifier_users", + desc="", + labelnames=[SERVER_NAME_LABEL], ) T = TypeVar("T") @@ -159,6 +182,9 @@ class _NotifierUserStream: lst = notifier.room_to_user_streams.get(room, set()) lst.discard(self) + if not lst: + notifier.room_to_user_streams.pop(room, None) + notifier.user_to_user_stream.pop(self.user_id) def count_listeners(self) -> int: @@ -222,6 +248,7 @@ class Notifier: self.room_to_user_streams: Dict[str, Set[_NotifierUserStream]] = {} self.hs = hs + self.server_name = hs.hostname self._storage_controllers = hs.get_storage_controllers() self.event_sources = hs.get_event_sources() self.store = hs.get_datastores().main @@ -255,7 +282,10 @@ class Notifier: # This is not a very cheap test to perform, but it's only executed # when rendering the metrics page, which is likely once per minute at # most when scraping it. - def count_listeners() -> int: + # + # Ideally, we'd use `Mapping[Tuple[str], int]` here but mypy doesn't like it. + # This is close enough and better than a type ignore. + def count_listeners() -> Mapping[Tuple[str, ...], int]: all_user_streams: Set[_NotifierUserStream] = set() for streams in list(self.room_to_user_streams.values()): @@ -263,18 +293,26 @@ class Notifier: for stream in list(self.user_to_user_stream.values()): all_user_streams.add(stream) - return sum(stream.count_listeners() for stream in all_user_streams) + return { + (self.server_name,): sum( + stream.count_listeners() for stream in all_user_streams + ) + } - LaterGauge("synapse_notifier_listeners", "", [], count_listeners) - - LaterGauge( - "synapse_notifier_rooms", - "", - [], - lambda: count(bool, list(self.room_to_user_streams.values())), + notifier_listeners_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), hook=count_listeners ) - LaterGauge( - "synapse_notifier_users", "", [], lambda: len(self.user_to_user_stream) + notifier_rooms_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: { + (self.server_name,): count( + bool, list(self.room_to_user_streams.values()) + ) + }, + ) + notifier_users_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: {(self.server_name,): len(self.user_to_user_stream)}, ) def add_replication_callback(self, cb: Callable[[], None]) -> None: @@ -348,9 +386,10 @@ class Notifier: for listener in listeners: listener.callback(current_token) - users_woken_by_stream_counter.labels(StreamKeyType.UN_PARTIAL_STATED_ROOMS).inc( - len(user_streams) - ) + users_woken_by_stream_counter.labels( + stream=StreamKeyType.UN_PARTIAL_STATED_ROOMS, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(len(user_streams)) # Poke the replication so that other workers also see the write to # the un-partial-stated rooms stream. @@ -493,6 +532,7 @@ class Notifier: StreamKeyType.TO_DEVICE, StreamKeyType.TYPING, StreamKeyType.UN_PARTIAL_STATED_ROOMS, + StreamKeyType.THREAD_SUBSCRIPTIONS, ], new_token: int, users: Optional[Collection[Union[str, UserID]]] = None, @@ -520,20 +560,22 @@ class Notifier: users = users or [] rooms = rooms or [] - with Measure(self.clock, "on_new_event"): - user_streams: Set[_NotifierUserStream] = set() + user_streams: Set[_NotifierUserStream] = set() - log_kv( - { - "waking_up_explicit_users": len(users), - "waking_up_explicit_rooms": len(rooms), - "users": shortstr(users), - "rooms": shortstr(rooms), - "stream": stream_key, - "stream_id": new_token, - } - ) + log_kv( + { + "waking_up_explicit_users": len(users), + "waking_up_explicit_rooms": len(rooms), + "users": shortstr(users), + "rooms": shortstr(rooms), + "stream": stream_key, + "stream_id": new_token, + } + ) + # Only calculate which user streams to wake up if there are, in fact, + # any user streams registered. + if self.user_to_user_stream or self.room_to_user_streams: for user in users: user_stream = self.user_to_user_stream.get(str(user)) if user_stream is not None: @@ -565,25 +607,28 @@ class Notifier: # We resolve all these deferreds in one go so that we only need to # call `PreserveLoggingContext` once, as it has a bunch of overhead # (to calculate performance stats) - with PreserveLoggingContext(): - for listener in listeners: - listener.callback(current_token) + if listeners: + with PreserveLoggingContext(): + for listener in listeners: + listener.callback(current_token) - users_woken_by_stream_counter.labels(stream_key).inc(len(user_streams)) + if user_streams: + users_woken_by_stream_counter.labels( + stream=stream_key, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(len(user_streams)) - self.notify_replication() + self.notify_replication() - # Notify appservices. - try: - self.appservice_handler.notify_interested_services_ephemeral( - stream_key, - new_token, - users, - ) - except Exception: - logger.exception( - "Error notifying application services of ephemeral events" - ) + # Notify appservices. + try: + self.appservice_handler.notify_interested_services_ephemeral( + stream_key, + new_token, + users, + ) + except Exception: + logger.exception("Error notifying application services of ephemeral events") def on_new_replication_data(self) -> None: """Used to inform replication listeners that something has happened diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 9c0592a902..ea9169aef0 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -25,6 +25,7 @@ from typing import ( Any, Collection, Dict, + FrozenSet, List, Mapping, Optional, @@ -48,10 +49,12 @@ from synapse.api.constants import ( from synapse.api.room_versions import PushRuleRoomFlag from synapse.event_auth import auth_types_for_event, get_user_power_level from synapse.events import EventBase, relation_from_event -from synapse.events.snapshot import EventContext +from synapse.events.snapshot import EventContext, EventPersistencePair from synapse.logging.context import make_deferred_yieldable, run_in_background -from synapse.state import POWER_KEY +from synapse.metrics import SERVER_NAME_LABEL +from synapse.state import CREATE_KEY, POWER_KEY from synapse.storage.databases.main.roommember import EventIdMembership +from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import ProfileInfo from synapse.synapse_rust.push import FilteredPushRules, PushRuleEvaluator from synapse.types import JsonValue @@ -67,11 +70,17 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# FIXME: Unused metric, remove if not needed. push_rules_invalidation_counter = Counter( - "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter", "" + "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter", + "", + labelnames=[SERVER_NAME_LABEL], ) +# FIXME: Unused metric, remove if not needed. push_rules_state_size_counter = Counter( - "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter", "" + "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter", + "", + labelnames=[SERVER_NAME_LABEL], ) @@ -127,18 +136,21 @@ class BulkPushRuleEvaluator: def __init__(self, hs: "HomeServer"): self.hs = hs + self.server_name = hs.hostname self.store = hs.get_datastores().main - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self._event_auth_handler = hs.get_event_auth_handler() self.should_calculate_push_rules = self.hs.config.push.enable_push self._related_event_match_enabled = self.hs.config.experimental.msc3664_enabled self.room_push_rule_cache_metrics = register_cache( - "cache", - "room_push_rule_cache", + cache_type="cache", + cache_name="room_push_rule_cache", cache=[], # Meaningless size, as this isn't a cache that stores values, resizable=False, + server_name=self.server_name, ) async def _get_rules_for_event( @@ -191,9 +203,17 @@ class BulkPushRuleEvaluator: # if this event is an invite event, we may need to run rules for the user # who's been invited, otherwise they won't get told they've been invited - if event.type == EventTypes.Member and event.membership == Membership.INVITE: + if ( + event.is_state() + and event.type == EventTypes.Member + and event.membership == Membership.INVITE + ): invited = event.state_key - if invited and self.hs.is_mine_id(invited) and invited not in local_users: + invite_config = await self.store.get_invite_config_for_user(invited) + if invite_config.get_invite_rule(event.sender) != InviteRule.ALLOW: + # Invite was blocked or ignored, never notify. + return {} + if self.hs.is_mine_id(invited) and invited not in local_users: local_users.append(invited) if not local_users: @@ -237,6 +257,7 @@ class BulkPushRuleEvaluator: StateFilter.from_types(event_types) ) pl_event_id = prev_state_ids.get(POWER_KEY) + create_event_id = prev_state_ids.get(CREATE_KEY) # fastpath: if there's a power level event, that's all we need, and # not having a power level event is an extreme edge case @@ -259,6 +280,26 @@ class BulkPushRuleEvaluator: if auth_event: auth_events_dict[auth_event_id] = auth_event auth_events = {(e.type, e.state_key): e for e in auth_events_dict.values()} + if auth_events.get(CREATE_KEY) is None: + # if the event being checked is the create event, use its own permissions + if event.type == EventTypes.Create and event.get_state_key() == "": + auth_events[CREATE_KEY] = event + else: + auth_events[ + CREATE_KEY + ] = await self.store.get_create_event_for_room(event.room_id) + + # if we are evaluating the create event, then use itself to determine power levels. + if event.type == EventTypes.Create and event.get_state_key() == "": + auth_events[CREATE_KEY] = event + else: + # if we aren't processing the create event, create_event_id should always be set + assert create_event_id is not None + create_event = event_id_to_event.get(create_event_id) + if create_event: + auth_events[CREATE_KEY] = create_event + else: + auth_events[CREATE_KEY] = await self.store.get_event(create_event_id) sender_level = get_user_power_level(event.sender, auth_events) @@ -311,7 +352,7 @@ class BulkPushRuleEvaluator: return related_events async def action_for_events_by_user( - self, events_and_context: List[Tuple[EventBase, EventContext]] + self, events_and_context: List[EventPersistencePair] ) -> None: """Given a list of events and their associated contexts, evaluate the push rules for each event, check if the message should increment the unread count, and @@ -371,7 +412,7 @@ class BulkPushRuleEvaluator: "Deferred[Tuple[int, Tuple[dict, Optional[int]], Dict[str, Dict[str, JsonValue]], Mapping[str, ProfileInfo]]]", gather_results( ( - run_in_background( # type: ignore[call-arg] + run_in_background( # type: ignore[call-overload] self.store.get_number_joined_users_in_room, event.room_id, # type: ignore[arg-type] ), @@ -382,10 +423,10 @@ class BulkPushRuleEvaluator: event_id_to_event, ), run_in_background(self._related_events, event), - run_in_background( # type: ignore[call-arg] + run_in_background( # type: ignore[call-overload] self.store.get_subset_users_in_room_with_profiles, - event.room_id, # type: ignore[arg-type] - rules_by_user.keys(), # type: ignore[arg-type] + event.room_id, + rules_by_user.keys(), ), ), consumeErrors=True, @@ -437,8 +478,18 @@ class BulkPushRuleEvaluator: event.room_version.msc3931_push_features, self.hs.config.experimental.msc1767_enabled, # MSC3931 flag self.hs.config.experimental.msc4210_enabled, + self.hs.config.experimental.msc4306_enabled, ) + msc4306_thread_subscribers: Optional[FrozenSet[str]] = None + if self.hs.config.experimental.msc4306_enabled and thread_id != MAIN_TIMELINE: + # pull out, in batch, all local subscribers to this thread + # (in the common case, they will all be getting processed for push + # rules right now) + msc4306_thread_subscribers = await self.store.get_subscribers_to_thread( + event.room_id, thread_id + ) + for uid, rules in rules_by_user.items(): if event.sender == uid: continue @@ -463,7 +514,13 @@ class BulkPushRuleEvaluator: # current user, it'll be added to the dict later. actions_by_user[uid] = [] - actions = evaluator.run(rules, uid, display_name) + msc4306_thread_subscription_state: Optional[bool] = None + if msc4306_thread_subscribers is not None: + msc4306_thread_subscription_state = uid in msc4306_thread_subscribers + + actions = evaluator.run( + rules, uid, display_name, msc4306_thread_subscription_state + ) if "notify" in actions: # Push rules say we should notify the user of this event actions_by_user[uid] = actions diff --git a/synapse/push/clientformat.py b/synapse/push/clientformat.py index b4afcfd85b..4f647491f1 100644 --- a/synapse/push/clientformat.py +++ b/synapse/push/clientformat.py @@ -91,7 +91,7 @@ def _rule_to_template(rule: PushRule) -> Optional[Dict[str, Any]]: unscoped_rule_id = _rule_id_from_namespaced(rule.rule_id) template_name = _priority_class_to_template_name(rule.priority_class) - if template_name in ["override", "underride"]: + if template_name in ["override", "underride", "postcontent"]: templaterule = {"conditions": rule.conditions, "actions": rule.actions} elif template_name in ["sender", "room"]: templaterule = {"actions": rule.actions} diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 0a14c534f7..09ca14584a 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -68,6 +68,7 @@ class EmailPusher(Pusher): super().__init__(hs, pusher_config) self.mailer = mailer + self.server_name = hs.hostname self.store = self.hs.get_datastores().main self.email = pusher_config.pushkey self.timed_call: Optional[IDelayedCall] = None @@ -117,7 +118,7 @@ class EmailPusher(Pusher): if self._is_processing: return - run_as_background_process("emailpush.process", self._process) + run_as_background_process("emailpush.process", self.server_name, self._process) def _pause_processing(self) -> None: """Used by tests to temporarily pause processing of events. diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index dd9b64d6ef..5946a6e972 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -31,6 +31,7 @@ from twisted.internet.interfaces import IDelayedCall from synapse.api.constants import EventTypes from synapse.events import EventBase from synapse.logging import opentracing +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.push import Pusher, PusherConfig, PusherConfigException from synapse.storage.databases.main.event_push_actions import HttpPushAction @@ -46,21 +47,25 @@ logger = logging.getLogger(__name__) http_push_processed_counter = Counter( "synapse_http_httppusher_http_pushes_processed", "Number of push notifications successfully sent", + labelnames=[SERVER_NAME_LABEL], ) http_push_failed_counter = Counter( "synapse_http_httppusher_http_pushes_failed", "Number of push notifications which failed", + labelnames=[SERVER_NAME_LABEL], ) http_badges_processed_counter = Counter( "synapse_http_httppusher_badge_updates_processed", "Number of badge updates successfully sent", + labelnames=[SERVER_NAME_LABEL], ) http_badges_failed_counter = Counter( "synapse_http_httppusher_badge_updates_failed", "Number of badge updates which failed", + labelnames=[SERVER_NAME_LABEL], ) @@ -106,6 +111,7 @@ class HttpPusher(Pusher): def __init__(self, hs: "HomeServer", pusher_config: PusherConfig): super().__init__(hs, pusher_config) + self.server_name = hs.hostname self._storage_controllers = self.hs.get_storage_controllers() self.app_display_name = pusher_config.app_display_name self.device_display_name = pusher_config.device_display_name @@ -127,6 +133,11 @@ class HttpPusher(Pusher): if self.data is None: raise PusherConfigException("'data' key can not be null for HTTP pusher") + # Check if badge counts should be disabled for this push gateway + self.disable_badge_count = self.hs.config.experimental.msc4076_enabled and bool( + self.data.get("org.matrix.msc4076.disable_badge_count", False) + ) + self.name = "%s/%s/%s" % ( pusher_config.user_name, pusher_config.app_id, @@ -171,7 +182,9 @@ class HttpPusher(Pusher): # We could check the receipts are actually m.read receipts here, # but currently that's the only type of receipt anyway... - run_as_background_process("http_pusher.on_new_receipts", self._update_badge) + run_as_background_process( + "http_pusher.on_new_receipts", self.server_name, self._update_badge + ) async def _update_badge(self) -> None: # XXX as per https://github.com/matrix-org/matrix-doc/issues/2627, this seems @@ -200,7 +213,13 @@ class HttpPusher(Pusher): if self._is_processing: return - run_as_background_process("httppush.process", self._process) + # Check if we are trying, but failing, to contact the pusher. If so, we + # don't try and start processing immediately and instead wait for the + # retry loop to try again later (which is controlled by the timer). + if self.failing_since and self.timed_call and self.timed_call.active(): + return + + run_as_background_process("httppush.process", self.server_name, self._process) async def _process(self) -> None: # we should never get here if we are already processing @@ -254,7 +273,9 @@ class HttpPusher(Pusher): processed = await self._process_one(push_action) if processed: - http_push_processed_counter.inc() + http_push_processed_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC self.last_stream_ordering = push_action.stream_ordering pusher_still_exists = ( @@ -278,7 +299,9 @@ class HttpPusher(Pusher): self.app_id, self.pushkey, self.user_id, self.failing_since ) else: - http_push_failed_counter.inc() + http_push_failed_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() if not self.failing_since: self.failing_since = self.clock.time_msec() await self.store.update_pusher_failing_since( @@ -461,9 +484,10 @@ class HttpPusher(Pusher): content: JsonDict = { "event_id": event.event_id, "room_id": event.room_id, - "counts": {"unread": badge}, "prio": priority, } + if not self.disable_badge_count: + content["counts"] = {"unread": badge} # event_id_only doesn't include the tweaks, so override them. tweaks = {} else: @@ -478,11 +502,11 @@ class HttpPusher(Pusher): "type": event.type, "sender": event.user_id, "prio": priority, - "counts": { - "unread": badge, - # 'missed_calls': 2 - }, } + if not self.disable_badge_count: + content["counts"] = { + "unread": badge, + } if event.type == "m.room.member" and event.is_state(): content["membership"] = event.content["membership"] content["user_is_target"] = event.state_key == self.user_id @@ -528,9 +552,13 @@ class HttpPusher(Pusher): } try: await self.http_client.post_json_get_json(self.url, d) - http_badges_processed_counter.inc() + http_badges_processed_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() except Exception as e: logger.warning( "Failed to send badge count to %s: %s %s", self.name, type(e), e ) - http_badges_failed_counter.inc() + http_badges_failed_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index cf611bd90b..d76cc8237b 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -32,6 +32,7 @@ from synapse.api.constants import EventContentFields, EventTypes, Membership, Ro from synapse.api.errors import StoreError from synapse.config.emailconfig import EmailSubjectConfig from synapse.events import EventBase +from synapse.metrics import SERVER_NAME_LABEL from synapse.push.presentable_names import ( calculate_room_name, descriptor_from_member_events, @@ -60,7 +61,7 @@ T = TypeVar("T") emails_sent_counter = Counter( "synapse_emails_sent_total", "Emails sent by type", - ["type"], + labelnames=["type", SERVER_NAME_LABEL], ) @@ -123,6 +124,7 @@ class Mailer: template_text: jinja2.Template, ): self.hs = hs + self.server_name = hs.hostname self.template_html = template_html self.template_text = template_text @@ -135,9 +137,7 @@ class Mailer: self.app_name = app_name self.email_subjects: EmailSubjectConfig = hs.config.email.email_subjects - logger.info("Created Mailer for app_name %s" % app_name) - - emails_sent_counter.labels("password_reset") + logger.info("Created Mailer for app_name %s", app_name) async def send_password_reset_mail( self, email_address: str, token: str, client_secret: str, sid: str @@ -162,7 +162,10 @@ class Mailer: template_vars: TemplateVars = {"link": link} - emails_sent_counter.labels("password_reset").inc() + emails_sent_counter.labels( + type="password_reset", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() await self.send_email( email_address, @@ -171,8 +174,6 @@ class Mailer: template_vars, ) - emails_sent_counter.labels("registration") - async def send_registration_mail( self, email_address: str, token: str, client_secret: str, sid: str ) -> None: @@ -196,7 +197,10 @@ class Mailer: template_vars: TemplateVars = {"link": link} - emails_sent_counter.labels("registration").inc() + emails_sent_counter.labels( + type="registration", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() await self.send_email( email_address, @@ -205,8 +209,6 @@ class Mailer: template_vars, ) - emails_sent_counter.labels("already_in_use") - async def send_already_in_use_mail(self, email_address: str) -> None: """Send an email if the address is already bound to an user account @@ -214,6 +216,11 @@ class Mailer: email_address: Email address we're sending to the "already in use" mail """ + emails_sent_counter.labels( + type="already_in_use", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() + await self.send_email( email_address, self.email_subjects.email_already_in_use @@ -221,8 +228,6 @@ class Mailer: {}, ) - emails_sent_counter.labels("add_threepid") - async def send_add_threepid_mail( self, email_address: str, token: str, client_secret: str, sid: str ) -> None: @@ -247,7 +252,10 @@ class Mailer: template_vars: TemplateVars = {"link": link} - emails_sent_counter.labels("add_threepid").inc() + emails_sent_counter.labels( + type="add_threepid", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() await self.send_email( email_address, @@ -256,8 +264,6 @@ class Mailer: template_vars, ) - emails_sent_counter.labels("notification") - async def send_notification_mail( self, app_id: str, @@ -352,7 +358,10 @@ class Mailer: "reason": reason, } - emails_sent_counter.labels("notification").inc() + emails_sent_counter.labels( + type="notification", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() await self.send_email( email_address, summary_text, template_vars, unsubscribe_link diff --git a/synapse/push/push_tools.py b/synapse/push/push_tools.py index 1ef881f702..3f3e4a9234 100644 --- a/synapse/push/push_tools.py +++ b/synapse/push/push_tools.py @@ -74,9 +74,13 @@ async def get_context_for_event( room_state = [] if ev.content.get("membership") == Membership.INVITE: - room_state = ev.unsigned.get("invite_room_state", []) + invite_room_state = ev.unsigned.get("invite_room_state", []) + if isinstance(invite_room_state, list): + room_state = invite_room_state elif ev.content.get("membership") == Membership.KNOCK: - room_state = ev.unsigned.get("knock_room_state", []) + knock_room_state = ev.unsigned.get("knock_room_state", []) + if isinstance(knock_room_state, list): + room_state = knock_room_state # Ideally we'd reuse the logic in `calculate_room_name`, but that gets # complicated to handle partial events vs pulling events from the DB. diff --git a/synapse/push/push_types.py b/synapse/push/push_types.py index 201ec97219..57fa926a46 100644 --- a/synapse/push/push_types.py +++ b/synapse/push/push_types.py @@ -18,9 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import List, Optional - -from typing_extensions import TypedDict +from typing import List, Optional, TypedDict class EmailReason(TypedDict, total=False): diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 0a7541b4c7..d1f79ec999 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -25,13 +25,17 @@ from typing import TYPE_CHECKING, Dict, Iterable, Optional from prometheus_client import Gauge from synapse.api.errors import Codes, SynapseError +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( run_as_background_process, wrap_as_background_process, ) from synapse.push import Pusher, PusherConfig, PusherConfigException from synapse.push.pusher import PusherFactory -from synapse.replication.http.push import ReplicationRemovePusherRestServlet +from synapse.replication.http.push import ( + ReplicationDeleteAllPushersForUserRestServlet, + ReplicationRemovePusherRestServlet, +) from synapse.types import JsonDict, RoomStreamToken, StrCollection from synapse.util.async_helpers import concurrently_execute from synapse.util.threepids import canonicalise_email @@ -44,7 +48,9 @@ logger = logging.getLogger(__name__) synapse_pushers = Gauge( - "synapse_pushers", "Number of active synapse pushers", ["kind", "app_id"] + "synapse_pushers", + "Number of active synapse pushers", + labelnames=["kind", "app_id", SERVER_NAME_LABEL], ) @@ -65,6 +71,9 @@ class PusherPool: def __init__(self, hs: "HomeServer"): self.hs = hs + self.server_name = ( + hs.hostname + ) # nb must be called this for @wrap_as_background_process self.pusher_factory = PusherFactory(hs) self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() @@ -78,10 +87,14 @@ class PusherPool: # We can only delete pushers on master. self._remove_pusher_client = None + self._delete_all_pushers_for_user_client = None if hs.config.worker.worker_app: self._remove_pusher_client = ReplicationRemovePusherRestServlet.make_client( hs ) + self._delete_all_pushers_for_user_client = ( + ReplicationDeleteAllPushersForUserRestServlet.make_client(hs) + ) # Record the last stream ID that we were poked about so we can get # changes since then. We set this to the current max stream ID on @@ -99,7 +112,9 @@ class PusherPool: if not self._should_start_pushers: logger.info("Not starting pushers because they are disabled in the config") return - run_as_background_process("start_pushers", self._start_pushers) + run_as_background_process( + "start_pushers", self.server_name, self._start_pushers + ) async def add_or_update_pusher( self, @@ -415,11 +430,17 @@ class PusherPool: previous_pusher.on_stop() synapse_pushers.labels( - type(previous_pusher).__name__, previous_pusher.app_id + kind=type(previous_pusher).__name__, + app_id=previous_pusher.app_id, + **{SERVER_NAME_LABEL: self.server_name}, ).dec() byuser[appid_pushkey] = pusher - synapse_pushers.labels(type(pusher).__name__, pusher.app_id).inc() + synapse_pushers.labels( + kind=type(pusher).__name__, + app_id=pusher.app_id, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() logger.info("Starting pusher %s / %s", pusher.user_id, appid_pushkey) @@ -454,6 +475,13 @@ class PusherPool: app_id, pushkey, user_id ) + async def delete_all_pushers_for_user(self, user_id: str) -> None: + """Deletes all pushers for a user.""" + if self._delete_all_pushers_for_user_client is not None: + await self._delete_all_pushers_for_user_client(user_id=user_id) + else: + await self.store.delete_all_pushers_for_user(user_id=user_id) + def maybe_stop_pusher(self, app_id: str, pushkey: str, user_id: str) -> None: """Stops a pusher with the given app ID and push key if one is running. @@ -471,4 +499,8 @@ class PusherPool: pusher = byuser.pop(appid_pushkey) pusher.on_stop() - synapse_pushers.labels(type(pusher).__name__, pusher.app_id).dec() + synapse_pushers.labels( + kind=type(pusher).__name__, + app_id=pusher.app_id, + **{SERVER_NAME_LABEL: self.server_name}, + ).dec() diff --git a/synapse/push/rulekinds.py b/synapse/push/rulekinds.py index 781ecc7fae..2eff626f92 100644 --- a/synapse/push/rulekinds.py +++ b/synapse/push/rulekinds.py @@ -19,10 +19,14 @@ # # +# Integer literals for push rule `kind`s +# This is used to store them in the database. PRIORITY_CLASS_MAP = { "underride": 1, "sender": 2, "room": 3, + # MSC4306 + "postcontent": 6, "content": 4, "override": 5, } diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index d500051714..68cc6ce1fc 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING from synapse.http.server import JsonResource from synapse.replication.http import ( account_data, + deactivate_account, delayed_events, devices, federation, @@ -31,7 +32,6 @@ from synapse.replication.http import ( presence, push, register, - send_event, send_events, state, streams, @@ -50,7 +50,6 @@ class ReplicationRestResource(JsonResource): self.register_servlets(hs) def register_servlets(self, hs: "HomeServer") -> None: - send_event.register_servlets(hs, self) send_events.register_servlets(hs, self) federation.register_servlets(hs, self) presence.register_servlets(hs, self) @@ -59,10 +58,11 @@ class ReplicationRestResource(JsonResource): account_data.register_servlets(hs, self) push.register_servlets(hs, self) state.register_servlets(hs, self) + devices.register_servlets(hs, self) # The following can't currently be instantiated on workers. if hs.config.worker.worker_app is None: login.register_servlets(hs, self) register.register_servlets(hs, self) - devices.register_servlets(hs, self) delayed_events.register_servlets(hs, self) + deactivate_account.register_servlets(hs, self) diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 9aa8d90bfe..0850a99e0c 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -38,6 +38,7 @@ from synapse.http.servlet import parse_json_object_from_request from synapse.http.site import SynapseRequest from synapse.logging import opentracing from synapse.logging.opentracing import trace_with_opname +from synapse.metrics import SERVER_NAME_LABEL from synapse.types import JsonDict from synapse.util.caches.response_cache import ResponseCache from synapse.util.cancellation import is_function_cancellable @@ -51,13 +52,13 @@ logger = logging.getLogger(__name__) _pending_outgoing_requests = Gauge( "synapse_pending_outgoing_replication_requests", "Number of active outgoing replication requests, by replication method name", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], ) _outgoing_request_counter = Counter( "synapse_outgoing_replication_requests", "Number of outgoing replication requests, by replication method name and result", - ["name", "code"], + labelnames=["name", "code", SERVER_NAME_LABEL], ) @@ -121,16 +122,21 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): WAIT_FOR_STREAMS: ClassVar[bool] = True def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname + if self.CACHE: self.response_cache: ResponseCache[str] = ResponseCache( - hs.get_clock(), "repl." + self.NAME, timeout_ms=30 * 60 * 1000 + clock=hs.get_clock(), + name="repl." + self.NAME, + server_name=self.server_name, + timeout_ms=30 * 60 * 1000, ) # We reserve `instance_name` as a parameter to sending requests, so we # assert here that sub classes don't try and use the name. - assert ( - "instance_name" not in self.PATH_ARGS - ), "`instance_name` is a reserved parameter name" + assert "instance_name" not in self.PATH_ARGS, ( + "`instance_name` is a reserved parameter name" + ) assert ( "instance_name" not in signature(self.__class__._serialize_payload).parameters @@ -200,13 +206,17 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): parameter to specify which instance to hit (the instance must be in the `instance_map` config). """ + server_name = hs.hostname clock = hs.get_clock() client = hs.get_replication_client() local_instance_name = hs.get_instance_name() instance_map = hs.config.worker.instance_map - outgoing_gauge = _pending_outgoing_requests.labels(cls.NAME) + outgoing_gauge = _pending_outgoing_requests.labels( + name=cls.NAME, + **{SERVER_NAME_LABEL: server_name}, + ) replication_secret = None if hs.config.worker.worker_replication_secret: @@ -328,15 +338,27 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): # We convert to SynapseError as we know that it was a SynapseError # on the main process that we should send to the client. (And # importantly, not stack traces everywhere) - _outgoing_request_counter.labels(cls.NAME, e.code).inc() + _outgoing_request_counter.labels( + name=cls.NAME, + code=e.code, + **{SERVER_NAME_LABEL: server_name}, + ).inc() raise e.to_synapse_error() except Exception as e: - _outgoing_request_counter.labels(cls.NAME, "ERR").inc() + _outgoing_request_counter.labels( + name=cls.NAME, + code="ERR", + **{SERVER_NAME_LABEL: server_name}, + ).inc() raise SynapseError( 502, f"Failed to talk to {instance_name} process" ) from e - _outgoing_request_counter.labels(cls.NAME, 200).inc() + _outgoing_request_counter.labels( + name=cls.NAME, + code=200, + **{SERVER_NAME_LABEL: server_name}, + ).inc() # Wait on any streams that the remote may have written to. for stream_name, position in result.pop( diff --git a/synapse/replication/http/deactivate_account.py b/synapse/replication/http/deactivate_account.py new file mode 100644 index 0000000000..89658350a5 --- /dev/null +++ b/synapse/replication/http/deactivate_account.py @@ -0,0 +1,81 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +import logging +from typing import TYPE_CHECKING, Tuple + +from twisted.web.server import Request + +from synapse.http.server import HttpServer +from synapse.replication.http._base import ReplicationEndpoint +from synapse.types import JsonDict + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ReplicationNotifyAccountDeactivatedServlet(ReplicationEndpoint): + """Notify that an account has been deactivated. + + Request format: + + POST /_synapse/replication/notify_account_deactivated/:user_id + + { + "by_admin": true, + } + + """ + + NAME = "notify_account_deactivated" + PATH_ARGS = ("user_id",) + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + self.deactivate_account_handler = hs.get_deactivate_account_handler() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + user_id: str, + by_admin: bool, + ) -> JsonDict: + """ + Args: + user_id: The user ID which has been deactivated. + by_admin: Whether the user was deactivated by an admin. + """ + return { + "by_admin": by_admin, + } + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> Tuple[int, JsonDict]: + by_admin = content["by_admin"] + await self.deactivate_account_handler.notify_account_deactivated( + user_id, by_admin=by_admin + ) + return 200, {} + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + ReplicationNotifyAccountDeactivatedServlet(hs).register(http_server) diff --git a/synapse/replication/http/devices.py b/synapse/replication/http/devices.py index 08cf9eff97..974d83bb8b 100644 --- a/synapse/replication/http/devices.py +++ b/synapse/replication/http/devices.py @@ -34,6 +34,92 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +class ReplicationNotifyDeviceUpdateRestServlet(ReplicationEndpoint): + """Notify a device writer that a user's device list has changed. + + Request format: + + POST /_synapse/replication/notify_device_update/:user_id + + { + "device_ids": ["JLAFKJWSCS", "JLAFKJWSCS"] + } + """ + + NAME = "notify_device_update" + PATH_ARGS = ("user_id",) + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + self.store = hs.get_datastores().main + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + user_id: str, device_ids: List[str] + ) -> JsonDict: + return {"device_ids": device_ids} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> Tuple[int, JsonDict]: + device_ids = content["device_ids"] + + span = active_span() + if span: + span.set_tag("user_id", user_id) + span.set_tag("device_ids", f"{device_ids!r}") + + await self.device_handler.notify_device_update(user_id, device_ids) + + return 200, {} + + +class ReplicationNotifyUserSignatureUpdateRestServlet(ReplicationEndpoint): + """Notify a device writer that a user have made new signatures of other users. + + Request format: + + POST /_synapse/replication/notify_user_signature_update/:from_user_id + + { + "user_ids": ["@alice:example.org", "@bob:example.org", ...] + } + """ + + NAME = "notify_user_signature_update" + PATH_ARGS = ("from_user_id",) + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + self.store = hs.get_datastores().main + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload(from_user_id: str, user_ids: List[str]) -> JsonDict: # type: ignore[override] + return {"user_ids": user_ids} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, from_user_id: str + ) -> Tuple[int, JsonDict]: + user_ids = content["user_ids"] + + span = active_span() + if span: + span.set_tag("from_user_id", from_user_id) + span.set_tag("user_ids", f"{user_ids!r}") + + await self.device_handler.notify_user_signature_update(from_user_id, user_ids) + + return 200, {} + + class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): """Ask master to resync the device list for multiple users from the same remote server by contacting their server. @@ -73,11 +159,7 @@ class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) - from synapse.handlers.device import DeviceHandler - - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_list_updater = handler.device_list_updater + self.device_list_updater = hs.get_device_handler().device_list_updater self.store = hs.get_datastores().main self.clock = hs.get_clock() @@ -103,32 +185,10 @@ class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): return 200, multi_user_devices +# FIXME(2025-07-22): Remove this on the next release, this will only get used +# during rollout to Synapse 1.135 and can be removed after that release. class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint): - """Ask master to upload keys for the user and send them out over federation to - update other servers. - - For now, only the master is permitted to handle key upload requests; - any worker can handle key query requests (since they're read-only). - - Calls to e2e_keys_handler.upload_keys_for_user(user_id, device_id, keys) on - the main process to accomplish this. - - Request format for this endpoint (borrowed and expanded from KeyUploadServlet): - - POST /_synapse/replication/upload_keys_for_user - - { - "user_id": "", - "device_id": "", - "keys": { - ....this part can be found in KeyUploadServlet in rest/client/keys.py.... - or as defined in https://spec.matrix.org/v1.4/client-server-api/#post_matrixclientv3keysupload - } - } - - Response is equivalent to ` /_matrix/client/v3/keys/upload` found in KeyUploadServlet - - """ + """Unused endpoint, kept for backwards compatibility during rollout.""" NAME = "upload_keys_for_user" PATH_ARGS = () @@ -165,6 +225,71 @@ class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint): return 200, results +class ReplicationHandleNewDeviceUpdateRestServlet(ReplicationEndpoint): + """Wake up a device writer to send local device list changes as federation outbound pokes. + + Request format: + + POST /_synapse/replication/handle_new_device_update + + {} + """ + + NAME = "handle_new_device_update" + PATH_ARGS = () + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + + @staticmethod + async def _serialize_payload() -> JsonDict: # type: ignore[override] + return {} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict + ) -> Tuple[int, JsonDict]: + await self.device_handler.handle_new_device_update() + return 200, {} + + +class ReplicationDeviceHandleRoomUnPartialStated(ReplicationEndpoint): + """Handles sending appropriate device list updates in a room that has + gone from partial to full state. + + Request format: + + POST /_synapse/replication/device_handle_room_un_partial_stated/:room_id + + {} + """ + + NAME = "device_handle_room_un_partial_stated" + PATH_ARGS = ("room_id",) + CACHE = True + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + + @staticmethod + async def _serialize_payload(room_id: str) -> JsonDict: # type: ignore[override] + return {} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, room_id: str + ) -> Tuple[int, JsonDict]: + await self.device_handler.handle_room_un_partial_stated(room_id) + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + ReplicationNotifyDeviceUpdateRestServlet(hs).register(http_server) + ReplicationNotifyUserSignatureUpdateRestServlet(hs).register(http_server) ReplicationMultiUserDevicesResyncRestServlet(hs).register(http_server) + ReplicationHandleNewDeviceUpdateRestServlet(hs).register(http_server) ReplicationUploadKeysForUserRestServlet(hs).register(http_server) + ReplicationDeviceHandleRoomUnPartialStated(hs).register(http_server) diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py index 940f418396..1e302ef59f 100644 --- a/synapse/replication/http/federation.py +++ b/synapse/replication/http/federation.py @@ -24,8 +24,8 @@ from typing import TYPE_CHECKING, List, Tuple from twisted.web.server import Request from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion -from synapse.events import EventBase, make_event_from_dict -from synapse.events.snapshot import EventContext +from synapse.events import make_event_from_dict +from synapse.events.snapshot import EventContext, EventPersistencePair from synapse.http.server import HttpServer from synapse.replication.http._base import ReplicationEndpoint from synapse.types import JsonDict @@ -76,6 +76,7 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.clock = hs.get_clock() @@ -85,7 +86,7 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint): async def _serialize_payload( # type: ignore[override] store: "DataStore", room_id: str, - event_and_contexts: List[Tuple[EventBase, EventContext]], + event_and_contexts: List[EventPersistencePair], backfilled: bool, ) -> JsonDict: """ @@ -122,7 +123,9 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint): async def _handle_request( # type: ignore[override] self, request: Request, content: JsonDict ) -> Tuple[int, JsonDict]: - with Measure(self.clock, "repl_fed_send_events_parse"): + with Measure( + self.clock, name="repl_fed_send_events_parse", server_name=self.server_name + ): room_id = content["room_id"] backfilled = content["backfilled"] @@ -202,6 +205,8 @@ class ReplicationFederationSendEduRestServlet(ReplicationEndpoint): return 200, {} +# FIXME(2025-07-22): Remove this on the next release, this will only get used +# during rollout to Synapse 1.135 and can be removed after that release. class ReplicationGetQueryRestServlet(ReplicationEndpoint): """Handle responding to queries from federation. @@ -249,6 +254,8 @@ class ReplicationGetQueryRestServlet(ReplicationEndpoint): return 200, result +# FIXME(2025-07-22): Remove this on the next release, this will only get used +# during rollout to Synapse 1.135 and can be removed after that release. class ReplicationCleanRoomRestServlet(ReplicationEndpoint): """Called to clean up any data in DB for a given room, ready for the server to join the room. @@ -284,6 +291,8 @@ class ReplicationCleanRoomRestServlet(ReplicationEndpoint): return 200, {} +# FIXME(2025-07-22): Remove this on the next release, this will only get used +# during rollout to Synapse 1.135 and can be removed after that release. class ReplicationStoreRoomOnOutlierMembershipRestServlet(ReplicationEndpoint): """Called to clean up any data in DB for a given room, ready for the server to join the room. diff --git a/synapse/replication/http/push.py b/synapse/replication/http/push.py index 48e254cdb1..6e20a208b6 100644 --- a/synapse/replication/http/push.py +++ b/synapse/replication/http/push.py @@ -118,6 +118,39 @@ class ReplicationCopyPusherRestServlet(ReplicationEndpoint): return 200, {} +class ReplicationDeleteAllPushersForUserRestServlet(ReplicationEndpoint): + """Deletes all pushers for a user. + + Request format: + + POST /_synapse/replication/delete_all_pushers_for_user/:user_id + + {} + + """ + + NAME = "delete_all_pushers_for_user" + PATH_ARGS = ("user_id",) + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self._store = hs.get_datastores().main + + @staticmethod + async def _serialize_payload(user_id: str) -> JsonDict: # type: ignore[override] + return {} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> Tuple[int, JsonDict]: + await self._store.delete_all_pushers_for_user(user_id) + + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ReplicationRemovePusherRestServlet(hs).register(http_server) ReplicationCopyPusherRestServlet(hs).register(http_server) + ReplicationDeleteAllPushersForUserRestServlet(hs).register(http_server) diff --git a/synapse/replication/http/register.py b/synapse/replication/http/register.py index 42a58b2858..27d3504c3c 100644 --- a/synapse/replication/http/register.py +++ b/synapse/replication/http/register.py @@ -33,6 +33,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# FIXME(2025-07-22): Remove this on the next release, this may only be used +# during rollout to Synapse 1.134 and can be removed after that release. class ReplicationRegisterServlet(ReplicationEndpoint): """Register a new user""" diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py deleted file mode 100644 index 01952a8d59..0000000000 --- a/synapse/replication/http/send_event.py +++ /dev/null @@ -1,161 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -import logging -from typing import TYPE_CHECKING, List, Tuple - -from twisted.web.server import Request - -from synapse.api.room_versions import KNOWN_ROOM_VERSIONS -from synapse.events import EventBase, make_event_from_dict -from synapse.events.snapshot import EventContext -from synapse.http.server import HttpServer -from synapse.replication.http._base import ReplicationEndpoint -from synapse.types import JsonDict, Requester, UserID -from synapse.util.metrics import Measure - -if TYPE_CHECKING: - from synapse.server import HomeServer - from synapse.storage.databases.main import DataStore - -logger = logging.getLogger(__name__) - - -class ReplicationSendEventRestServlet(ReplicationEndpoint): - """Handles events newly created on workers, including persisting and - notifying. - - The API looks like: - - POST /_synapse/replication/send_event/:event_id/:txn_id - - { - "event": { .. serialized event .. }, - "room_version": .., // "1", "2", "3", etc: the version of the room - // containing the event - "event_format_version": .., // 1,2,3 etc: the event format version - "internal_metadata": { .. serialized internal_metadata .. }, - "outlier": true|false, - "rejected_reason": .., // The event.rejected_reason field - "context": { .. serialized event context .. }, - "requester": { .. serialized requester .. }, - "ratelimit": true, - "extra_users": [], - } - - 200 OK - - { "stream_id": 12345, "event_id": "$abcdef..." } - - Responds with a 409 when a `PartialStateConflictError` is raised due to an event - context that needs to be recomputed due to the un-partial stating of a room. - - The returned event ID may not match the sent event if it was deduplicated. - """ - - NAME = "send_event" - PATH_ARGS = ("event_id",) - - def __init__(self, hs: "HomeServer"): - super().__init__(hs) - - self.event_creation_handler = hs.get_event_creation_handler() - self.store = hs.get_datastores().main - self._storage_controllers = hs.get_storage_controllers() - self.clock = hs.get_clock() - - @staticmethod - async def _serialize_payload( # type: ignore[override] - event_id: str, - store: "DataStore", - event: EventBase, - context: EventContext, - requester: Requester, - ratelimit: bool, - extra_users: List[UserID], - ) -> JsonDict: - """ - Args: - event_id - store - requester - event - context - ratelimit - extra_users: Any extra users to notify about event - """ - serialized_context = await context.serialize(event, store) - - payload = { - "event": event.get_pdu_json(), - "room_version": event.room_version.identifier, - "event_format_version": event.format_version, - "internal_metadata": event.internal_metadata.get_dict(), - "outlier": event.internal_metadata.is_outlier(), - "rejected_reason": event.rejected_reason, - "context": serialized_context, - "requester": requester.serialize(), - "ratelimit": ratelimit, - "extra_users": [u.to_string() for u in extra_users], - } - - return payload - - async def _handle_request( # type: ignore[override] - self, request: Request, content: JsonDict, event_id: str - ) -> Tuple[int, JsonDict]: - with Measure(self.clock, "repl_send_event_parse"): - event_dict = content["event"] - room_ver = KNOWN_ROOM_VERSIONS[content["room_version"]] - internal_metadata = content["internal_metadata"] - rejected_reason = content["rejected_reason"] - - event = make_event_from_dict( - event_dict, room_ver, internal_metadata, rejected_reason - ) - event.internal_metadata.outlier = content["outlier"] - - requester = Requester.deserialize(self.store, content["requester"]) - context = EventContext.deserialize( - self._storage_controllers, content["context"] - ) - - ratelimit = content["ratelimit"] - extra_users = [UserID.from_string(u) for u in content["extra_users"]] - - logger.info( - "Got event to send with ID: %s into room: %s", event.event_id, event.room_id - ) - - event = await self.event_creation_handler.persist_and_notify_client_events( - requester, [(event, context)], ratelimit=ratelimit, extra_users=extra_users - ) - - return ( - 200, - { - "stream_id": event.internal_metadata.stream_ordering, - "event_id": event.event_id, - }, - ) - - -def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - ReplicationSendEventRestServlet(hs).register(http_server) diff --git a/synapse/replication/http/send_events.py b/synapse/replication/http/send_events.py index d965ce5492..6b1a5a9956 100644 --- a/synapse/replication/http/send_events.py +++ b/synapse/replication/http/send_events.py @@ -25,8 +25,8 @@ from typing import TYPE_CHECKING, List, Tuple from twisted.web.server import Request from synapse.api.room_versions import KNOWN_ROOM_VERSIONS -from synapse.events import EventBase, make_event_from_dict -from synapse.events.snapshot import EventContext +from synapse.events import make_event_from_dict +from synapse.events.snapshot import EventContext, EventPersistencePair from synapse.http.server import HttpServer from synapse.replication.http._base import ReplicationEndpoint from synapse.types import JsonDict, Requester, UserID @@ -77,6 +77,7 @@ class ReplicationSendEventsRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.event_creation_handler = hs.get_event_creation_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -84,7 +85,7 @@ class ReplicationSendEventsRestServlet(ReplicationEndpoint): @staticmethod async def _serialize_payload( # type: ignore[override] - events_and_context: List[Tuple[EventBase, EventContext]], + events_and_context: List[EventPersistencePair], store: "DataStore", requester: Requester, ratelimit: bool, @@ -122,7 +123,9 @@ class ReplicationSendEventsRestServlet(ReplicationEndpoint): async def _handle_request( # type: ignore[override] self, request: Request, payload: JsonDict ) -> Tuple[int, JsonDict]: - with Measure(self.clock, "repl_send_events_parse"): + with Measure( + self.clock, name="repl_send_events_parse", server_name=self.server_name + ): events_and_context = [] events = payload["events"] rooms = set() diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 0bd5478cd3..7a86b2e65e 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -44,6 +44,7 @@ from synapse.replication.tcp.streams import ( UnPartialStatedEventStream, UnPartialStatedRoomStream, ) +from synapse.replication.tcp.streams._base import ThreadSubscriptionsStream from synapse.replication.tcp.streams.events import ( EventsStream, EventsStreamEventRow, @@ -75,6 +76,7 @@ class ReplicationDataHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self._reactor = hs.get_reactor() @@ -115,7 +117,11 @@ class ReplicationDataHandler: all_room_ids: Set[str] = set() if stream_name == DeviceListsStream.NAME: if any(not row.is_signature and not row.hosts_calculated for row in rows): - prev_token = self.store.get_device_stream_token() + # This only uses the minimum stream position on the device lists + # stream, which means that we may process a device list change + # twice in case of concurrent writes. This is fine, as this only + # triggers cache invalidation, which is harmless if done twice. + prev_token = self.store.get_device_stream_token().stream all_room_ids = await self.store.get_all_device_list_changes( prev_token, token ) @@ -250,6 +256,12 @@ class ReplicationDataHandler: self._state_storage_controller.notify_event_un_partial_stated( row.event_id ) + elif stream_name == ThreadSubscriptionsStream.NAME: + self.notifier.on_new_event( + StreamKeyType.THREAD_SUBSCRIPTIONS, + token, + users=[row.user_id for row in rows], + ) await self._presence_handler.process_replication_rows( stream_name, instance_name, token, rows @@ -342,7 +354,11 @@ class ReplicationDataHandler: waiting_list.add((position, deferred)) # We measure here to get in flight counts and average waiting time. - with Measure(self._clock, "repl.wait_for_stream_position"): + with Measure( + self._clock, + name="repl.wait_for_stream_position", + server_name=self.server_name, + ): logger.info( "Waiting for repl stream %r to reach %s (%s); currently at: %s", stream_name, @@ -404,6 +420,7 @@ class FederationSenderHandler: def __init__(self, hs: "HomeServer"): assert hs.should_send_federation() + self.server_name = hs.hostname self.store = hs.get_datastores().main self._is_mine_id = hs.is_mine_id self._hs = hs @@ -494,7 +511,9 @@ class FederationSenderHandler: # no need to queue up another task. return - run_as_background_process("_save_and_send_ack", self._save_and_send_ack) + run_as_background_process( + "_save_and_send_ack", self.server_name, self._save_and_send_ack + ) async def _save_and_send_ack(self) -> None: """Save the current federation position in the database and send an ACK diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index 7d51441e91..6ab5356660 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -495,7 +495,7 @@ class LockReleasedCommand(Command): class NewActiveTaskCommand(_SimpleCommand): - """Sent to inform instance handling background tasks that a new active task is available to run. + """Sent to inform instance handling background tasks that a new task is ready to run. Format:: diff --git a/synapse/replication/tcp/external_cache.py b/synapse/replication/tcp/external_cache.py index a95771b5f6..497b26fcaf 100644 --- a/synapse/replication/tcp/external_cache.py +++ b/synapse/replication/tcp/external_cache.py @@ -26,6 +26,7 @@ from prometheus_client import Counter, Histogram from synapse.logging import opentracing from synapse.logging.context import make_deferred_yieldable +from synapse.metrics import SERVER_NAME_LABEL from synapse.util import json_decoder, json_encoder if TYPE_CHECKING: @@ -36,19 +37,19 @@ if TYPE_CHECKING: set_counter = Counter( "synapse_external_cache_set", "Number of times we set a cache", - labelnames=["cache_name"], + labelnames=["cache_name", SERVER_NAME_LABEL], ) get_counter = Counter( "synapse_external_cache_get", "Number of times we get a cache", - labelnames=["cache_name", "hit"], + labelnames=["cache_name", "hit", SERVER_NAME_LABEL], ) response_timer = Histogram( "synapse_external_cache_response_time_seconds", "Time taken to get a response from Redis for a cache get/set request", - labelnames=["method"], + labelnames=["method", SERVER_NAME_LABEL], buckets=( 0.001, 0.002, @@ -69,6 +70,8 @@ class ExternalCache: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname + if hs.config.redis.redis_enabled: self._redis_connection: Optional["ConnectionHandler"] = ( hs.get_outbound_redis_connection() @@ -93,7 +96,9 @@ class ExternalCache: if self._redis_connection is None: return - set_counter.labels(cache_name).inc() + set_counter.labels( + cache_name=cache_name, **{SERVER_NAME_LABEL: self.server_name} + ).inc() # txredisapi requires the value to be string, bytes or numbers, so we # encode stuff in JSON. @@ -105,7 +110,9 @@ class ExternalCache: "ExternalCache.set", tags={opentracing.SynapseTags.CACHE_NAME: cache_name}, ): - with response_timer.labels("set").time(): + with response_timer.labels( + method="set", **{SERVER_NAME_LABEL: self.server_name} + ).time(): return await make_deferred_yieldable( self._redis_connection.set( self._get_redis_key(cache_name, key), @@ -124,14 +131,20 @@ class ExternalCache: "ExternalCache.get", tags={opentracing.SynapseTags.CACHE_NAME: cache_name}, ): - with response_timer.labels("get").time(): + with response_timer.labels( + method="get", **{SERVER_NAME_LABEL: self.server_name} + ).time(): result = await make_deferred_yieldable( self._redis_connection.get(self._get_redis_key(cache_name, key)) ) logger.debug("Got cache result %s %s: %r", cache_name, key, result) - get_counter.labels(cache_name, result is not None).inc() + get_counter.labels( + cache_name=cache_name, + hit=result is not None, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() if not result: return None diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 6101226938..dd7e38dd78 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -40,7 +40,7 @@ from prometheus_client import Counter from twisted.internet.protocol import ReconnectingClientFactory -from synapse.metrics import LaterGauge +from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.commands import ( ClearUserSyncsCommand, @@ -72,6 +72,10 @@ from synapse.replication.tcp.streams import ( ToDeviceStream, TypingStream, ) +from synapse.replication.tcp.streams._base import ( + DeviceListsStream, + ThreadSubscriptionsStream, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -81,13 +85,38 @@ logger = logging.getLogger(__name__) # number of updates received for each RDATA stream inbound_rdata_count = Counter( - "synapse_replication_tcp_protocol_inbound_rdata_count", "", ["stream_name"] + "synapse_replication_tcp_protocol_inbound_rdata_count", + "", + labelnames=["stream_name", SERVER_NAME_LABEL], +) +user_sync_counter = Counter( + "synapse_replication_tcp_resource_user_sync", "", labelnames=[SERVER_NAME_LABEL] +) +federation_ack_counter = Counter( + "synapse_replication_tcp_resource_federation_ack", + "", + labelnames=[SERVER_NAME_LABEL], +) +# FIXME: Unused metric, remove if not needed. +remove_pusher_counter = Counter( + "synapse_replication_tcp_resource_remove_pusher", "", labelnames=[SERVER_NAME_LABEL] ) -user_sync_counter = Counter("synapse_replication_tcp_resource_user_sync", "") -federation_ack_counter = Counter("synapse_replication_tcp_resource_federation_ack", "") -remove_pusher_counter = Counter("synapse_replication_tcp_resource_remove_pusher", "") -user_ip_cache_counter = Counter("synapse_replication_tcp_resource_user_ip_cache", "") +user_ip_cache_counter = Counter( + "synapse_replication_tcp_resource_user_ip_cache", "", labelnames=[SERVER_NAME_LABEL] +) + +tcp_resource_total_connections_gauge = LaterGauge( + name="synapse_replication_tcp_resource_total_connections", + desc="", + labelnames=[SERVER_NAME_LABEL], +) + +tcp_command_queue_gauge = LaterGauge( + name="synapse_replication_tcp_command_queue", + desc="Number of inbound RDATA/POSITION commands queued for processing", + labelnames=["stream_name", SERVER_NAME_LABEL], +) # the type of the entries in _command_queues_by_stream @@ -102,6 +131,7 @@ class ReplicationCommandHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._replication_data_handler = hs.get_replication_data_handler() self._presence_handler = hs.get_presence_handler() self._store = hs.get_datastores().main @@ -185,6 +215,21 @@ class ReplicationCommandHandler: continue + if isinstance(stream, ThreadSubscriptionsStream): + if ( + hs.get_instance_name() + in hs.config.worker.writers.thread_subscriptions + ): + self._streams_to_replicate.append(stream) + + continue + + if isinstance(stream, DeviceListsStream): + if hs.get_instance_name() in hs.config.worker.writers.device_lists: + self._streams_to_replicate.append(stream) + + continue + # Only add any other streams if we're on master. if hs.config.worker.worker_app is not None: continue @@ -210,11 +255,9 @@ class ReplicationCommandHandler: # outgoing replication commands to.) self._connections: List[IReplicationConnection] = [] - LaterGauge( - "synapse_replication_tcp_resource_total_connections", - "", - [], - lambda: len(self._connections), + tcp_resource_total_connections_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: {(self.server_name,): len(self._connections)}, ) # When POSITION or RDATA commands arrive, we stick them in a queue and process @@ -233,12 +276,10 @@ class ReplicationCommandHandler: # from that connection. self._streams_by_connection: Dict[IReplicationConnection, Set[str]] = {} - LaterGauge( - "synapse_replication_tcp_command_queue", - "Number of inbound RDATA/POSITION commands queued for processing", - ["stream_name"], - lambda: { - (stream_name,): len(queue) + tcp_command_queue_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: { + (stream_name, self.server_name): len(queue) for stream_name, queue in self._command_queues_by_stream.items() }, ) @@ -321,7 +362,10 @@ class ReplicationCommandHandler: # fire off a background process to start processing the queue. run_as_background_process( - "process-replication-data", self._unsafe_process_queue, stream_name + "process-replication-data", + self.server_name, + self._unsafe_process_queue, + stream_name, ) async def _unsafe_process_queue(self, stream_name: str) -> None: @@ -437,7 +481,7 @@ class ReplicationCommandHandler: def on_USER_SYNC( self, conn: IReplicationConnection, cmd: UserSyncCommand ) -> Optional[Awaitable[None]]: - user_sync_counter.inc() + user_sync_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc() if self._is_presence_writer: return self._presence_handler.update_external_syncs_row( @@ -461,7 +505,7 @@ class ReplicationCommandHandler: def on_FEDERATION_ACK( self, conn: IReplicationConnection, cmd: FederationAckCommand ) -> None: - federation_ack_counter.inc() + federation_ack_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc() if self._federation_sender: self._federation_sender.federation_ack(cmd.instance_name, cmd.token) @@ -469,7 +513,7 @@ class ReplicationCommandHandler: def on_USER_IP( self, conn: IReplicationConnection, cmd: UserIpCommand ) -> Optional[Awaitable[None]]: - user_ip_cache_counter.inc() + user_ip_cache_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc() if self._is_master or self._should_insert_client_ips: # We make a point of only returning an awaitable if there's actually @@ -509,7 +553,9 @@ class ReplicationCommandHandler: return stream_name = cmd.stream_name - inbound_rdata_count.labels(stream_name).inc() + inbound_rdata_count.labels( + stream_name=stream_name, **{SERVER_NAME_LABEL: self.server_name} + ).inc() # We put the received command into a queue here for two reasons: # 1. so we don't try and concurrently handle multiple rows for the @@ -727,7 +773,7 @@ class ReplicationCommandHandler: ) -> None: """Called when get a new NEW_ACTIVE_TASK command.""" if self._task_scheduler: - self._task_scheduler.launch_task_by_id(cmd.data) + self._task_scheduler.on_new_task(cmd.data) def new_connection(self, connection: IReplicationConnection) -> None: """Called when we have a new connection.""" diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index fb9c539122..2ec25bf43d 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -39,7 +39,7 @@ from twisted.protocols.basic import LineOnlyReceiver from twisted.python.failure import Failure from synapse.logging.context import PreserveLoggingContext -from synapse.metrics import LaterGauge +from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import ( BackgroundProcessLoggingContext, run_as_background_process, @@ -64,19 +64,21 @@ if TYPE_CHECKING: connection_close_counter = Counter( - "synapse_replication_tcp_protocol_close_reason", "", ["reason_type"] + "synapse_replication_tcp_protocol_close_reason", + "", + labelnames=["reason_type", SERVER_NAME_LABEL], ) tcp_inbound_commands_counter = Counter( "synapse_replication_tcp_protocol_inbound_commands", "Number of commands received from replication, by command and name of process connected to", - ["command", "name"], + labelnames=["command", "name", SERVER_NAME_LABEL], ) tcp_outbound_commands_counter = Counter( "synapse_replication_tcp_protocol_outbound_commands", "Number of commands sent to replication, by command and name of process connected to", - ["command", "name"], + labelnames=["command", "name", SERVER_NAME_LABEL], ) # A list of all connected protocols. This allows us to send metrics about the @@ -137,7 +139,10 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): max_line_buffer = 10000 - def __init__(self, clock: Clock, handler: "ReplicationCommandHandler"): + def __init__( + self, server_name: str, clock: Clock, handler: "ReplicationCommandHandler" + ): + self.server_name = server_name self.clock = clock self.command_handler = handler @@ -166,7 +171,9 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): # capture the sentinel context as its containing context and won't prevent # GC of / unintentionally reactivate what would be the current context. self._logging_context = BackgroundProcessLoggingContext( - "replication-conn", self.conn_id + name="replication-conn", + server_name=self.server_name, + instance_id=self.conn_id, ) def connectionMade(self) -> None: @@ -244,7 +251,11 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): self.last_received_command = self.clock.time_msec() - tcp_inbound_commands_counter.labels(cmd.NAME, self.name).inc() + tcp_inbound_commands_counter.labels( + command=cmd.NAME, + name=self.name, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() self.handle_command(cmd) @@ -280,7 +291,9 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): if isawaitable(res): run_as_background_process( - "replication-" + cmd.get_logcontext_id(), lambda: res + "replication-" + cmd.get_logcontext_id(), + self.server_name, + lambda: res, ) handled = True @@ -318,7 +331,11 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): self._queue_command(cmd) return - tcp_outbound_commands_counter.labels(cmd.NAME, self.name).inc() + tcp_outbound_commands_counter.labels( + command=cmd.NAME, + name=self.name, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() string = "%s %s" % (cmd.NAME, cmd.to_line()) if "\n" in string: @@ -390,9 +407,15 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): logger.info("[%s] Replication connection closed: %r", self.id(), reason) if isinstance(reason, Failure): assert reason.type is not None - connection_close_counter.labels(reason.type.__name__).inc() + connection_close_counter.labels( + reason_type=reason.type.__name__, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() else: - connection_close_counter.labels(reason.__class__.__name__).inc() # type: ignore[unreachable] + connection_close_counter.labels( # type: ignore[unreachable] + reason_type=reason.__class__.__name__, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() try: # Remove us from list of connections to be monitored @@ -449,7 +472,7 @@ class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol): def __init__( self, server_name: str, clock: Clock, handler: "ReplicationCommandHandler" ): - super().__init__(clock, handler) + super().__init__(server_name, clock, handler) self.server_name = server_name @@ -474,7 +497,7 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): clock: Clock, command_handler: "ReplicationCommandHandler", ): - super().__init__(clock, command_handler) + super().__init__(server_name, clock, command_handler) self.client_name = client_name self.server_name = server_name @@ -501,10 +524,15 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): # The following simply registers metrics for the replication connections pending_commands = LaterGauge( - "synapse_replication_tcp_protocol_pending_commands", - "", - ["name"], - lambda: {(p.name,): len(p.pending_commands) for p in connected_connections}, + name="synapse_replication_tcp_protocol_pending_commands", + desc="", + labelnames=["name", SERVER_NAME_LABEL], +) +pending_commands.register_hook( + homeserver_instance_id=None, + hook=lambda: { + (p.name, p.server_name): len(p.pending_commands) for p in connected_connections + }, ) @@ -516,10 +544,15 @@ def transport_buffer_size(protocol: BaseReplicationStreamProtocol) -> int: transport_send_buffer = LaterGauge( - "synapse_replication_tcp_protocol_transport_send_buffer", - "", - ["name"], - lambda: {(p.name,): transport_buffer_size(p) for p in connected_connections}, + name="synapse_replication_tcp_protocol_transport_send_buffer", + desc="", + labelnames=["name", SERVER_NAME_LABEL], +) +transport_send_buffer.register_hook( + homeserver_instance_id=None, + hook=lambda: { + (p.name, p.server_name): transport_buffer_size(p) for p in connected_connections + }, ) @@ -541,22 +574,28 @@ def transport_kernel_read_buffer_size( tcp_transport_kernel_send_buffer = LaterGauge( - "synapse_replication_tcp_protocol_transport_kernel_send_buffer", - "", - ["name"], - lambda: { - (p.name,): transport_kernel_read_buffer_size(p, False) + name="synapse_replication_tcp_protocol_transport_kernel_send_buffer", + desc="", + labelnames=["name", SERVER_NAME_LABEL], +) +tcp_transport_kernel_send_buffer.register_hook( + homeserver_instance_id=None, + hook=lambda: { + (p.name, p.server_name): transport_kernel_read_buffer_size(p, False) for p in connected_connections }, ) tcp_transport_kernel_read_buffer = LaterGauge( - "synapse_replication_tcp_protocol_transport_kernel_read_buffer", - "", - ["name"], - lambda: { - (p.name,): transport_kernel_read_buffer_size(p, True) + name="synapse_replication_tcp_protocol_transport_kernel_read_buffer", + desc="", + labelnames=["name", SERVER_NAME_LABEL], +) +tcp_transport_kernel_read_buffer.register_hook( + homeserver_instance_id=None, + hook=lambda: { + (p.name, p.server_name): transport_kernel_read_buffer_size(p, True) for p in connected_connections }, ) diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index c4601a6141..aba79b2378 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -37,6 +37,7 @@ from twisted.internet.interfaces import IAddress, IConnector from twisted.python.failure import Failure from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( BackgroundProcessLoggingContext, run_as_background_process, @@ -97,6 +98,9 @@ class RedisSubscriber(SubscriberProtocol): immediately after initialisation. Attributes: + server_name: The homeserver name of the Synapse instance that this connection + is associated with. This is used to label metrics and should be set to + `hs.hostname`. synapse_handler: The command handler to handle incoming commands. synapse_stream_prefix: The *redis* stream name to subscribe to and publish from (not anything to do with Synapse replication streams). @@ -104,6 +108,7 @@ class RedisSubscriber(SubscriberProtocol): commands. """ + server_name: str synapse_handler: "ReplicationCommandHandler" synapse_stream_prefix: str synapse_channel_names: List[str] @@ -114,18 +119,36 @@ class RedisSubscriber(SubscriberProtocol): # a logcontext which we use for processing incoming commands. We declare it as a # background process so that the CPU stats get reported to prometheus. - with PreserveLoggingContext(): - # thanks to `PreserveLoggingContext()`, the new logcontext is guaranteed to - # capture the sentinel context as its containing context and won't prevent - # GC of / unintentionally reactivate what would be the current context. - self._logging_context = BackgroundProcessLoggingContext( - "replication_command_handler" - ) + self._logging_context: Optional[BackgroundProcessLoggingContext] = None + + def _get_logging_context(self) -> BackgroundProcessLoggingContext: + """ + We lazily create the logging context so that `self.server_name` is set and + available. See `RedisDirectTcpReplicationClientFactory.buildProtocol` for more + details on why we set `self.server_name` after the fact instead of in the + constructor. + """ + assert self.server_name is not None, ( + "self.server_name must be set before using _get_logging_context()" + ) + if self._logging_context is None: + # a logcontext which we use for processing incoming commands. We declare it as a + # background process so that the CPU stats get reported to prometheus. + with PreserveLoggingContext(): + # thanks to `PreserveLoggingContext()`, the new logcontext is guaranteed to + # capture the sentinel context as its containing context and won't prevent + # GC of / unintentionally reactivate what would be the current context. + self._logging_context = BackgroundProcessLoggingContext( + name="replication_command_handler", server_name=self.server_name + ) + return self._logging_context def connectionMade(self) -> None: logger.info("Connected to redis") super().connectionMade() - run_as_background_process("subscribe-replication", self._send_subscribe) + run_as_background_process( + "subscribe-replication", self.server_name, self._send_subscribe + ) async def _send_subscribe(self) -> None: # it's important to make sure that we only send the REPLICATE command once we @@ -152,7 +175,7 @@ class RedisSubscriber(SubscriberProtocol): def messageReceived(self, pattern: str, channel: str, message: str) -> None: """Received a message from redis.""" - with PreserveLoggingContext(self._logging_context): + with PreserveLoggingContext(self._get_logging_context()): self._parse_and_dispatch_message(message) def _parse_and_dispatch_message(self, message: str) -> None: @@ -171,7 +194,11 @@ class RedisSubscriber(SubscriberProtocol): # We use "redis" as the name here as we don't have 1:1 connections to # remote instances. - tcp_inbound_commands_counter.labels(cmd.NAME, "redis").inc() + tcp_inbound_commands_counter.labels( + command=cmd.NAME, + name="redis", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() self.handle_command(cmd) @@ -197,7 +224,7 @@ class RedisSubscriber(SubscriberProtocol): if isawaitable(res): run_as_background_process( - "replication-" + cmd.get_logcontext_id(), lambda: res + "replication-" + cmd.get_logcontext_id(), self.server_name, lambda: res ) def connectionLost(self, reason: Failure) -> None: # type: ignore[override] @@ -207,7 +234,7 @@ class RedisSubscriber(SubscriberProtocol): # mark the logging context as finished by triggering `__exit__()` with PreserveLoggingContext(): - with self._logging_context: + with self._get_logging_context(): pass # the sentinel context is now active, which may not be correct. # PreserveLoggingContext() will restore the correct logging context. @@ -219,7 +246,11 @@ class RedisSubscriber(SubscriberProtocol): cmd: The command to send """ run_as_background_process( - "send-cmd", self._async_send_command, cmd, bg_start_span=False + "send-cmd", + self.server_name, + self._async_send_command, + cmd, + bg_start_span=False, ) async def _async_send_command(self, cmd: Command) -> None: @@ -232,7 +263,11 @@ class RedisSubscriber(SubscriberProtocol): # We use "redis" as the name here as we don't have 1:1 connections to # remote instances. - tcp_outbound_commands_counter.labels(cmd.NAME, "redis").inc() + tcp_outbound_commands_counter.labels( + command=cmd.NAME, + name="redis", + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() channel_name = cmd.redis_channel_name(self.synapse_stream_prefix) @@ -275,6 +310,10 @@ class SynapseRedisFactory(RedisFactory): convertNumbers=convertNumbers, ) + self.server_name = ( + hs.hostname + ) # nb must be called this for @wrap_as_background_process + hs.get_clock().looping_call(self._send_ping, 30 * 1000) @wrap_as_background_process("redis_ping") @@ -350,6 +389,7 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory): password=hs.config.redis.redis_password, ) + self.server_name = hs.hostname self.synapse_handler = hs.get_replication_command_handler() self.synapse_stream_prefix = hs.hostname self.synapse_channel_names = channel_names @@ -364,6 +404,7 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory): # as to do so would involve overriding `buildProtocol` entirely, however # the base method does some other things than just instantiating the # protocol. + p.server_name = self.server_name p.synapse_handler = self.synapse_handler p.synapse_outbound_redis_connection = self.synapse_outbound_redis_connection p.synapse_stream_prefix = self.synapse_stream_prefix diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index d647a2b332..d800cfe6f6 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -29,6 +29,7 @@ from prometheus_client import Counter from twisted.internet.interfaces import IAddress from twisted.internet.protocol import ServerFactory +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.commands import PositionCommand from synapse.replication.tcp.protocol import ServerReplicationStreamProtocol @@ -40,7 +41,9 @@ if TYPE_CHECKING: from synapse.server import HomeServer stream_updates_counter = Counter( - "synapse_replication_tcp_resource_stream_updates", "", ["stream_name"] + "synapse_replication_tcp_resource_stream_updates", + "", + labelnames=["stream_name", SERVER_NAME_LABEL], ) logger = logging.getLogger(__name__) @@ -78,6 +81,7 @@ class ReplicationStreamer: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.clock = hs.get_clock() self.notifier = hs.get_notifier() @@ -143,7 +147,9 @@ class ReplicationStreamer: logger.debug("Notifier poke loop already running") return - run_as_background_process("replication_notifier", self._run_notifier_loop) + run_as_background_process( + "replication_notifier", self.server_name, self._run_notifier_loop + ) async def _run_notifier_loop(self) -> None: self.is_looping = True @@ -155,7 +161,11 @@ class ReplicationStreamer: while self.pending_updates: self.pending_updates = False - with Measure(self.clock, "repl.stream.get_updates"): + with Measure( + self.clock, + name="repl.stream.get_updates", + server_name=self.server_name, + ): all_streams = self.streams if self._replication_torture_level is not None: @@ -219,7 +229,10 @@ class ReplicationStreamer: len(updates), current_token, ) - stream_updates_counter.labels(stream.NAME).inc(len(updates)) + stream_updates_counter.labels( + stream_name=stream.NAME, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(len(updates)) else: # The token has advanced but there is no data to diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index 677dcb7b40..25c15e5d48 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -41,6 +41,7 @@ from synapse.replication.tcp.streams._base import ( PushRulesStream, ReceiptsStream, Stream, + ThreadSubscriptionsStream, ToDeviceStream, TypingStream, ) @@ -67,6 +68,7 @@ STREAMS_MAP = { ToDeviceStream, FederationStream, AccountDataStream, + ThreadSubscriptionsStream, UnPartialStatedRoomStream, UnPartialStatedEventStream, ) @@ -86,6 +88,7 @@ __all__ = [ "DeviceListsStream", "ToDeviceStream", "AccountDataStream", + "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", "UnPartialStatedEventStream", ] diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index ebf5964d29..ec7e935d6a 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -723,3 +723,46 @@ class AccountDataStream(_StreamFromIdGen): heapq.merge(room_rows, global_rows, tag_rows, key=lambda row: row[0]) ) return updates, to_token, limited + + +class ThreadSubscriptionsStream(_StreamFromIdGen): + """A thread subscription was changed.""" + + @attr.s(slots=True, auto_attribs=True) + class ThreadSubscriptionsStreamRow: + """Stream to inform workers about changes to thread subscriptions.""" + + user_id: str + room_id: str + event_id: str # The event ID of the thread root + + NAME = "thread_subscriptions" + ROW_TYPE = ThreadSubscriptionsStreamRow + + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + super().__init__( + hs.get_instance_name(), + self._update_function, + self.store._thread_subscriptions_id_gen, + ) + + async def _update_function( + self, instance_name: str, from_token: int, to_token: int, limit: int + ) -> StreamUpdateResult: + updates = await self.store.get_updated_thread_subscriptions( + from_id=from_token, to_id=to_token, limit=limit + ) + rows = [ + ( + stream_id, + # These are the args to `ThreadSubscriptionsStreamRow` + (user_id, room_id, event_id), + ) + for stream_id, user_id, room_id, event_id in updates + ] + + if not rows: + return [], to_token, False + + return rows, rows[-1][0], len(updates) == limit diff --git a/synapse/replication/tcp/streams/events.py b/synapse/replication/tcp/streams/events.py index ea0803dfc2..05b55fb033 100644 --- a/synapse/replication/tcp/streams/events.py +++ b/synapse/replication/tcp/streams/events.py @@ -200,9 +200,9 @@ class EventsStream(_StreamFromIdGen): # we rely on get_all_new_forward_event_rows strictly honouring the limit, so # that we know it is safe to just take upper_limit = event_rows[-1][0]. - assert ( - len(event_rows) <= target_row_count - ), "get_all_new_forward_event_rows did not honour row limit" + assert len(event_rows) <= target_row_count, ( + "get_all_new_forward_event_rows did not honour row limit" + ) # if we hit the limit on event_updates, there's no point in going beyond the # last stream_id in the batch for the other sources. diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 4e594e6595..a24ca09846 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -29,7 +29,7 @@ from synapse.rest.client import ( account_validity, appservice_ping, auth, - auth_issuer, + auth_metadata, capabilities, delayed_events, devices, @@ -63,6 +63,7 @@ from synapse.rest.client import ( sync, tags, thirdparty, + thread_subscriptions, tokenrefresh, user_directory, versions, @@ -121,7 +122,8 @@ CLIENT_SERVLET_FUNCTIONS: Tuple[RegisterServletsFunc, ...] = ( mutual_rooms.register_servlets, login_token_request.register_servlets, rendezvous.register_servlets, - auth_issuer.register_servlets, + auth_metadata.register_servlets, + thread_subscriptions.register_servlets, ) SERVLET_GROUPS: Dict[str, Iterable[RegisterServletsFunc]] = { @@ -165,7 +167,7 @@ class ClientRestResource(JsonResource): # Fail on unknown servlet groups. if servlet_group not in SERVLET_GROUPS: if servlet_group == "media": - logger.warn( + logger.warning( "media.can_load_media_repo needs to be configured for the media servlet to be available" ) raise RuntimeError( @@ -187,7 +189,6 @@ class ClientRestResource(JsonResource): mutual_rooms.register_servlets, login_token_request.register_servlets, rendezvous.register_servlets, - auth_issuer.register_servlets, ]: continue diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 4db8975674..d9a6e99c5d 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -39,7 +39,7 @@ from typing import TYPE_CHECKING, Optional, Tuple from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.handlers.pagination import PURGE_HISTORY_ACTION_NAME -from synapse.http.server import HttpServer, JsonResource +from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin @@ -86,6 +86,7 @@ from synapse.rest.admin.rooms import ( RoomStateRestServlet, RoomTimestampToEventRestServlet, ) +from synapse.rest.admin.scheduled_tasks import ScheduledTasksRestServlet from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet from synapse.rest.admin.statistics import ( LargestRoomsStatistics, @@ -107,6 +108,8 @@ from synapse.rest.admin.users import ( UserAdminServlet, UserByExternalId, UserByThreePid, + UserInvitesCount, + UserJoinedRoomCount, UserMembershipRestServlet, UserRegisterServlet, UserReplaceMasterCrossSigningKeyRestServlet, @@ -203,8 +206,7 @@ class PurgeHistoryRestServlet(RestServlet): (stream, topo, _event_id) = r token = "t%d-%d" % (topo, stream) logger.info( - "[purge] purging up to token %s (received_ts %i => " - "stream_ordering %i)", + "[purge] purging up to token %s (received_ts %i => stream_ordering %i)", token, ts, stream_ordering, @@ -261,37 +263,38 @@ class PurgeHistoryStatusRestServlet(RestServlet): ######################################################################################## -class AdminRestResource(JsonResource): - """The REST resource which gets mounted at /_synapse/admin""" - - def __init__(self, hs: "HomeServer"): - JsonResource.__init__(self, hs, canonical_json=False) - register_servlets(hs, self) - - def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: """ Register all the admin servlets. """ - # Admin servlets aren't registered on workers. + RoomRestServlet(hs).register(http_server) + + # Admin servlets below may not work on workers. if hs.config.worker.worker_app is not None: + # Some admin servlets can be mounted on workers when MSC3861 is enabled. + # Note that this is only for MSC3861 mode, as modern MAS using the + # matrix_authentication_service integration uses the dedicated MAS API. + if hs.config.experimental.msc3861.enabled: + register_servlets_for_msc3861_delegation(hs, http_server) + return + auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + register_servlets_for_client_rest_resource(hs, http_server) BlockRoomRestServlet(hs).register(http_server) ListRoomRestServlet(hs).register(http_server) RoomStateRestServlet(hs).register(http_server) - RoomRestServlet(hs).register(http_server) RoomRestV2Servlet(hs).register(http_server) RoomMembersRestServlet(hs).register(http_server) DeleteRoomStatusByDeleteIdRestServlet(hs).register(http_server) DeleteRoomStatusByRoomIdRestServlet(hs).register(http_server) JoinRoomAliasServlet(hs).register(http_server) VersionServlet(hs).register(http_server) - if not hs.config.experimental.msc3861.enabled: + if not auth_delegated: UserAdminServlet(hs).register(http_server) UserMembershipRestServlet(hs).register(http_server) - if not hs.config.experimental.msc3861.enabled: + if not auth_delegated: UserTokenRestServlet(hs).register(http_server) UserRestServletV2(hs).register(http_server) UsersRestServletV2(hs).register(http_server) @@ -308,7 +311,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: RoomEventContextServlet(hs).register(http_server) RateLimitRestServlet(hs).register(http_server) UsernameAvailableRestServlet(hs).register(http_server) - if not hs.config.experimental.msc3861.enabled: + if not auth_delegated: ListRegistrationTokensRestServlet(hs).register(http_server) NewRegistrationTokenRestServlet(hs).register(http_server) RegistrationTokenRestServlet(hs).register(http_server) @@ -323,6 +326,8 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: UserByThreePid(hs).register(http_server) RedactUser(hs).register(http_server) RedactUserStatus(hs).register(http_server) + UserInvitesCount(hs).register(http_server) + UserJoinedRoomCount(hs).register(http_server) DeviceRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) @@ -332,24 +337,26 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: BackgroundUpdateRestServlet(hs).register(http_server) BackgroundUpdateStartJobRestServlet(hs).register(http_server) ExperimentalFeaturesRestServlet(hs).register(http_server) - if hs.config.experimental.msc3823_account_suspension: - SuspendAccountRestServlet(hs).register(http_server) + SuspendAccountRestServlet(hs).register(http_server) + ScheduledTasksRestServlet(hs).register(http_server) def register_servlets_for_client_rest_resource( hs: "HomeServer", http_server: HttpServer ) -> None: """Register only the servlets which need to be exposed on /_matrix/client/xxx""" + auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + WhoisRestServlet(hs).register(http_server) PurgeHistoryStatusRestServlet(hs).register(http_server) PurgeHistoryRestServlet(hs).register(http_server) # The following resources can only be run on the main process. if hs.config.worker.worker_app is None: DeactivateAccountRestServlet(hs).register(http_server) - if not hs.config.experimental.msc3861.enabled: + if not auth_delegated: ResetPasswordRestServlet(hs).register(http_server) SearchUsersRestServlet(hs).register(http_server) - if not hs.config.experimental.msc3861.enabled: + if not auth_delegated: UserRegisterServlet(hs).register(http_server) AccountValidityRenewServlet(hs).register(http_server) @@ -361,4 +368,17 @@ def register_servlets_for_client_rest_resource( ListMediaInRoom(hs).register(http_server) # don't add more things here: new servlets should only be exposed on - # /_synapse/admin so should not go here. Instead register them in AdminRestResource. + # /_synapse/admin so should not go here. Instead register them in register_servlets. + + +def register_servlets_for_msc3861_delegation( + hs: "HomeServer", http_server: HttpServer +) -> None: + """Register servlets needed by MAS when MSC3861 is enabled""" + assert hs.config.experimental.msc3861.enabled + + UserRestServletV2(hs).register(http_server) + UsernameAvailableRestServlet(hs).register(http_server) + UserReplaceMasterCrossSigningKeyRestServlet(hs).register(http_server) + DeviceRestServlet(hs).register(http_server) + DevicesRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py index 449b066923..c488bce58e 100644 --- a/synapse/rest/admin/devices.py +++ b/synapse/rest/admin/devices.py @@ -23,7 +23,6 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Tuple from synapse.api.errors import NotFoundError, SynapseError -from synapse.handlers.device import DeviceHandler from synapse.http.servlet import ( RestServlet, assert_params_in_dict, @@ -51,9 +50,7 @@ class DeviceRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.auth = hs.get_auth() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_handler = handler + self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main self.is_mine = hs.is_mine @@ -116,15 +113,16 @@ class DeviceRestServlet(RestServlet): class DevicesRestServlet(RestServlet): """ Retrieve the given user's devices + + This can be mounted on workers as it is read-only, as opposed + to `DevicesRestServlet`. """ PATTERNS = admin_patterns("/users/(?P[^/]*)/devices$", "v2") def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_handler = handler + self.device_worker_handler = hs.get_device_handler() self.store = hs.get_datastores().main self.is_mine = hs.is_mine @@ -141,7 +139,20 @@ class DevicesRestServlet(RestServlet): if u is None: raise NotFoundError("Unknown user") - devices = await self.device_handler.get_devices_by_user(target_user.to_string()) + devices = await self.device_worker_handler.get_devices_by_user( + target_user.to_string() + ) + + # mark the dehydrated device by adding a "dehydrated" flag + dehydrated_device_info = await self.device_worker_handler.get_dehydrated_device( + target_user.to_string() + ) + if dehydrated_device_info: + dehydrated_device_id = dehydrated_device_info[0] + for device in devices: + is_dehydrated = device["device_id"] == dehydrated_device_id + device["dehydrated"] = is_dehydrated + return HTTPStatus.OK, {"devices": devices, "total": len(devices)} async def on_POST( @@ -167,7 +178,7 @@ class DevicesRestServlet(RestServlet): if not isinstance(device_id, str): raise SynapseError(HTTPStatus.BAD_REQUEST, "device_id must be a string") - await self.device_handler.check_device_registered( + await self.device_worker_handler.check_device_registered( user_id=user_id, device_id=device_id ) @@ -184,9 +195,7 @@ class DeleteDevicesRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_handler = handler + self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main self.is_mine = hs.is_mine diff --git a/synapse/rest/admin/event_reports.py b/synapse/rest/admin/event_reports.py index 9fb68bfa46..ff1abc0697 100644 --- a/synapse/rest/admin/event_reports.py +++ b/synapse/rest/admin/event_reports.py @@ -50,8 +50,10 @@ class EventReportsRestServlet(RestServlet): The parameters `from` and `limit` are required only for pagination. By default, a `limit` of 100 is used. The parameter `dir` can be used to define the order of results. - The parameter `user_id` can be used to filter by user id. - The parameter `room_id` can be used to filter by room id. + The `user_id` query parameter filters by the user ID of the reporter of the event. + The `room_id` query parameter filters by room id. + The `event_sender_user_id` query parameter can be used to filter by the user id + of the sender of the reported event. Returns: A list of reported events and an integer representing the total number of reported events that exist given this query @@ -71,6 +73,7 @@ class EventReportsRestServlet(RestServlet): direction = parse_enum(request, "dir", Direction, Direction.BACKWARDS) user_id = parse_string(request, "user_id") room_id = parse_string(request, "room_id") + event_sender_user_id = parse_string(request, "event_sender_user_id") if start < 0: raise SynapseError( @@ -87,7 +90,7 @@ class EventReportsRestServlet(RestServlet): ) event_reports, total = await self._store.get_event_reports_paginate( - start, limit, direction, user_id, room_id + start, limit, direction, user_id, room_id, event_sender_user_id ) ret = {"event_reports": event_reports, "total": total} if (start + limit) < total: diff --git a/synapse/rest/admin/experimental_features.py b/synapse/rest/admin/experimental_features.py index afb71f4a0f..3d3015cef7 100644 --- a/synapse/rest/admin/experimental_features.py +++ b/synapse/rest/admin/experimental_features.py @@ -92,9 +92,9 @@ class ExperimentalFeaturesRestServlet(RestServlet): user_features = {} for feature in ExperimentalFeature: if feature in enabled_features: - user_features[feature] = True + user_features[feature.value] = True else: - user_features[feature] = False + user_features[feature.value] = False return HTTPStatus.OK, {"features": user_features} async def on_PUT( diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index ee6a681285..195f22a4c2 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -71,7 +71,7 @@ class QuarantineMediaInRoom(RestServlet): requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - logging.info("Quarantining room: %s", room_id) + logger.info("Quarantining room: %s", room_id) # Quarantine all media in this room num_quarantined = await self.store.quarantine_media_ids_in_room( @@ -98,7 +98,7 @@ class QuarantineMediaByUser(RestServlet): requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - logging.info("Quarantining media by user: %s", user_id) + logger.info("Quarantining media by user: %s", user_id) # Quarantine all media this user has uploaded num_quarantined = await self.store.quarantine_media_ids_by_user( @@ -127,7 +127,7 @@ class QuarantineMediaByID(RestServlet): requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - logging.info("Quarantining media by ID: %s/%s", server_name, media_id) + logger.info("Quarantining media by ID: %s/%s", server_name, media_id) # Quarantine this media id await self.store.quarantine_media_by_id( @@ -155,7 +155,7 @@ class UnquarantineMediaByID(RestServlet): ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - logging.info("Remove from quarantine media by ID: %s/%s", server_name, media_id) + logger.info("Remove from quarantine media by ID: %s/%s", server_name, media_id) # Remove from quarantine this media id await self.store.quarantine_media_by_id(server_name, media_id, None) @@ -177,7 +177,7 @@ class ProtectMediaByID(RestServlet): ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - logging.info("Protecting local media by ID: %s", media_id) + logger.info("Protecting local media by ID: %s", media_id) # Protect this media id await self.store.mark_local_media_as_safe(media_id, safe=True) @@ -199,7 +199,7 @@ class UnprotectMediaByID(RestServlet): ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - logging.info("Unprotecting local media by ID: %s", media_id) + logger.info("Unprotecting local media by ID: %s", media_id) # Unprotect this media id await self.store.mark_local_media_as_safe(media_id, safe=False) @@ -280,7 +280,7 @@ class DeleteMediaByID(RestServlet): if await self.store.get_local_media(media_id) is None: raise NotFoundError("Unknown media") - logging.info("Deleting local media by ID: %s", media_id) + logger.info("Deleting local media by ID: %s", media_id) deleted_media, total = await self.media_repository.delete_local_media_ids( [media_id] @@ -327,9 +327,11 @@ class DeleteMediaByDateSize(RestServlet): if server_name is not None and self.server_name != server_name: raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only delete local media") - logging.info( - "Deleting local media by timestamp: %s, size larger than: %s, keep profile media: %s" - % (before_ts, size_gt, keep_profiles) + logger.info( + "Deleting local media by timestamp: %s, size larger than: %s, keep profile media: %s", + before_ts, + size_gt, + keep_profiles, ) deleted_media, total = await self.media_repository.delete_old_local_media( diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 01f9de9ffa..5bed89c2c4 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -23,6 +23,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Tuple, cast import attr +from immutabledict import immutabledict from synapse.api.constants import Direction, EventTypes, JoinRules, Membership from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError @@ -149,6 +150,7 @@ class RoomRestV2Servlet(RestServlet): def _convert_delete_task_to_response(task: ScheduledTask) -> JsonDict: return { "delete_id": task.id, + "room_id": task.resource_id, "status": task.status, "shutdown_room": task.result, } @@ -463,7 +465,18 @@ class RoomStateRestServlet(RestServlet): if not room: raise NotFoundError("Room not found") - event_ids = await self._storage_controllers.state.get_current_state_ids(room_id) + state_filter = None + type = parse_string(request, "type") + + if type: + state_filter = StateFilter( + types=immutabledict({type: None}), + include_others=False, + ) + + event_ids = await self._storage_controllers.state.get_current_state_ids( + room_id, state_filter + ) events = await self.store.get_events(event_ids.values()) now = self.clock.time_msec() room_state = await self._event_serializer.serialize_events(events.values(), now) @@ -614,6 +627,15 @@ class MakeRoomAdminRestServlet(ResolveRoomIdMixin, RestServlet): ] admin_users.sort(key=lambda user: user_power[user]) + if create_event.room_version.msc4289_creator_power_enabled: + creators = create_event.content.get("additional_creators", []) + [ + create_event.sender + ] + for creator in creators: + if self.is_mine_id(creator): + # include the creator as they won't be in the PL users map. + admin_users.append(creator) + if not admin_users: raise SynapseError( HTTPStatus.BAD_REQUEST, "No local admin user in room" @@ -653,7 +675,11 @@ class MakeRoomAdminRestServlet(ResolveRoomIdMixin, RestServlet): # updated power level event. new_pl_content = dict(pl_content) new_pl_content["users"] = dict(pl_content.get("users", {})) - new_pl_content["users"][user_to_add] = new_pl_content["users"][admin_user_id] + # give the new user the same PL as the admin, default to 100 in case there is no PL event. + # This means in v12+ rooms we get PL100 if the creator promotes us. + new_pl_content["users"][user_to_add] = new_pl_content["users"].get( + admin_user_id, 100 + ) fake_requester = create_requester( admin_user_id, diff --git a/synapse/rest/admin/scheduled_tasks.py b/synapse/rest/admin/scheduled_tasks.py new file mode 100644 index 0000000000..2ae13021b9 --- /dev/null +++ b/synapse/rest/admin/scheduled_tasks.py @@ -0,0 +1,70 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +# +from typing import TYPE_CHECKING, Tuple + +from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.site import SynapseRequest +from synapse.rest.admin import admin_patterns, assert_requester_is_admin +from synapse.types import JsonDict, TaskStatus + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +class ScheduledTasksRestServlet(RestServlet): + """Get a list of scheduled tasks and their statuses + optionally filtered by action name, resource id, status, and max timestamp + """ + + PATTERNS = admin_patterns("/scheduled_tasks$") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self._store = hs.get_datastores().main + + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + + # extract query params + action_name = parse_string(request, "action_name") + resource_id = parse_string(request, "resource_id") + status = parse_string(request, "job_status") + max_timestamp = parse_integer(request, "max_timestamp") + + actions = [action_name] if action_name else None + statuses = [TaskStatus(status)] if status else None + + tasks = await self._store.get_scheduled_tasks( + actions=actions, + resource_id=resource_id, + statuses=statuses, + max_timestamp=max_timestamp, + ) + + json_tasks = [] + for task in tasks: + result_task = { + "id": task.id, + "action": task.action, + "status": task.status, + "timestamp_ms": task.timestamp, + "resource_id": task.resource_id, + "result": task.result, + "error": task.error, + } + json_tasks.append(result_task) + + return 200, {"scheduled_tasks": json_tasks} diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index b146c2754d..25a38dc4ac 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -28,7 +28,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union import attr from synapse._pydantic_compat import StrictBool, StrictInt, StrictStr -from synapse.api.constants import Direction, UserTypes +from synapse.api.constants import Direction from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.http.servlet import ( RestServlet, @@ -42,6 +42,7 @@ from synapse.http.servlet import ( parse_strings_from_args, ) from synapse.http.site import SynapseRequest +from synapse.logging.loggers import ExplicitlyConfiguredLogger from synapse.rest.admin._base import ( admin_patterns, assert_requester_is_admin, @@ -60,6 +61,25 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +original_logger_class = logging.getLoggerClass() +# Because this can log sensitive information, use a custom logger class that only allows +# logging if the logger is explicitly configured. +logging.setLoggerClass(ExplicitlyConfiguredLogger) +user_registration_sensitive_debug_logger = logging.getLogger( + "synapse.rest.admin.users.registration_debug" +) +""" +A logger for debugging the user registration process. + +Because this can log sensitive information (such as passwords and +`registration_shared_secret`), we want people to explictly opt-in before seeing anything +in the logs. Requires explicitly setting `synapse.rest.admin.users.registration_debug` +in the logging configuration and does not inherit the log level from the parent logger. +""" +# Restore the original logger class +logging.setLoggerClass(original_logger_class) + + class UsersRestServletV2(RestServlet): PATTERNS = admin_patterns("/users$", "v2") @@ -89,7 +109,9 @@ class UsersRestServletV2(RestServlet): self.auth = hs.get_auth() self.admin_handler = hs.get_admin_handler() self._msc3866_enabled = hs.config.experimental.msc3866.enabled - self._msc3861_enabled = hs.config.experimental.msc3861.enabled + self._auth_delegation_enabled = ( + hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + ) async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) @@ -101,10 +123,10 @@ class UsersRestServletV2(RestServlet): name = parse_string(request, "name", encoding="utf-8") guests = parse_boolean(request, "guests", default=True) - if self._msc3861_enabled and guests: + if self._auth_delegation_enabled and guests: raise SynapseError( HTTPStatus.BAD_REQUEST, - "The guests parameter is not supported when MSC3861 is enabled.", + "The guests parameter is not supported when delegating to MAS.", errcode=Codes.INVALID_PARAM, ) @@ -230,6 +252,7 @@ class UserRestServletV2(RestServlet): self.registration_handler = hs.get_registration_handler() self.pusher_pool = hs.get_pusherpool() self._msc3866_enabled = hs.config.experimental.msc3866.enabled + self._all_user_types = hs.config.user_types.all_user_types async def on_GET( self, request: SynapseRequest, user_id: str @@ -277,7 +300,7 @@ class UserRestServletV2(RestServlet): assert_params_in_dict(external_id, ["auth_provider", "external_id"]) user_type = body.get("user_type", None) - if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES: + if user_type is not None and user_type not in self._all_user_types: raise SynapseError(HTTPStatus.BAD_REQUEST, "Invalid user type") set_admin_to = body.get("admin", False) @@ -524,6 +547,7 @@ class UserRegisterServlet(RestServlet): self.reactor = hs.get_reactor() self.nonces: Dict[str, int] = {} self.hs = hs + self._all_user_types = hs.config.user_types.all_user_types def _clear_old_nonces(self) -> None: """ @@ -605,7 +629,7 @@ class UserRegisterServlet(RestServlet): user_type = body.get("user_type", None) displayname = body.get("displayname", None) - if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES: + if user_type is not None and user_type not in self._all_user_types: raise SynapseError(HTTPStatus.BAD_REQUEST, "Invalid user type") if "mac" not in body: @@ -633,6 +657,34 @@ class UserRegisterServlet(RestServlet): want_mac = want_mac_builder.hexdigest() if not hmac.compare_digest(want_mac.encode("ascii"), got_mac.encode("ascii")): + # If the sensitive debug logger is enabled, log the full details. + # + # For reference, the `user_registration_sensitive_debug_logger.debug(...)` + # call is enough to gate the logging of sensitive information unless + # explicitly enabled. We only have this if-statement to avoid logging the + # suggestion to enable the debug logger if you already have it enabled. + if user_registration_sensitive_debug_logger.isEnabledFor(logging.DEBUG): + user_registration_sensitive_debug_logger.debug( + "UserRegisterServlet: Incorrect HMAC digest: actual=%s, expected=%s, registration_shared_secret=%s, body=%s", + got_mac, + want_mac, + self.hs.config.registration.registration_shared_secret, + body, + ) + else: + # Otherwise, just log the non-sensitive essentials and advertise the + # debug logger for sensitive information. + logger.debug( + ( + "UserRegisterServlet: HMAC incorrect (username=%s): actual=%s, expected=%s - " + "If you need more information, explicitly enable the `synapse.rest.admin.users.registration_debug` " + "logger at the `DEBUG` level to log things like the full request body and " + "`registration_shared_secret` used to calculate the HMAC." + ), + username, + got_mac, + want_mac, + ) raise SynapseError(HTTPStatus.FORBIDDEN, "HMAC incorrect") should_issue_refresh_token = body.get("refresh_token", False) @@ -948,7 +1000,7 @@ class UserAdminServlet(RestServlet): "Only local users can be admins of this homeserver", ) - is_admin = await self.store.is_server_admin(target_user) + is_admin = await self.store.is_server_admin(target_user.to_string()) return HTTPStatus.OK, {"admin": is_admin} @@ -983,7 +1035,7 @@ class UserAdminServlet(RestServlet): class UserMembershipRestServlet(RestServlet): """ - Get room list of an user. + Get list of joined room ID's for a user. """ PATTERNS = admin_patterns("/users/(?P[^/]*)/joined_rooms$") @@ -999,8 +1051,9 @@ class UserMembershipRestServlet(RestServlet): await assert_requester_is_admin(self.auth, request) room_ids = await self.store.get_rooms_for_user(user_id) - ret = {"joined_rooms": list(room_ids), "total": len(room_ids)} - return HTTPStatus.OK, ret + rooms_response = {"joined_rooms": list(room_ids), "total": len(room_ids)} + + return HTTPStatus.OK, rooms_response class PushersRestServlet(RestServlet): @@ -1411,7 +1464,7 @@ class RedactUser(RestServlet): """ Redact all the events of a given user in the given rooms or if empty dict is provided then all events in all rooms user is member of. Kicks off a background process and - returns an id that can be used to check on the progress of the redaction progress + returns an id that can be used to check on the progress of the redaction progress. """ PATTERNS = admin_patterns("/user/(?P[^/]*)/redact") @@ -1425,6 +1478,7 @@ class RedactUser(RestServlet): rooms: List[StrictStr] reason: Optional[StrictStr] limit: Optional[StrictInt] + use_admin: Optional[StrictBool] async def on_POST( self, request: SynapseRequest, user_id: str @@ -1452,8 +1506,12 @@ class RedactUser(RestServlet): ) rooms = current_rooms + banned_rooms + use_admin = body.use_admin + if not use_admin: + use_admin = False + redact_id = await self.admin_handler.start_redact_events( - user_id, rooms, requester.serialize(), body.reason, limit + user_id, rooms, requester.serialize(), use_admin, body.reason, limit ) return HTTPStatus.OK, {"redact_id": redact_id} @@ -1501,3 +1559,50 @@ class RedactUserStatus(RestServlet): } else: raise NotFoundError("redact id '%s' not found" % redact_id) + + +class UserInvitesCount(RestServlet): + """ + Return the count of invites that the user has sent after the given timestamp + """ + + PATTERNS = admin_patterns("/users/(?P[^/]*)/sent_invite_count") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self.store = hs.get_datastores().main + + async def on_GET( + self, request: SynapseRequest, user_id: str + ) -> Tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + from_ts = parse_integer(request, "from_ts", required=True) + + sent_invite_count = await self.store.get_sent_invite_count_by_user( + user_id, from_ts + ) + + return HTTPStatus.OK, {"invite_count": sent_invite_count} + + +class UserJoinedRoomCount(RestServlet): + """ + Return the count of rooms that the user has joined at or after the given timestamp, even + if they have subsequently left/been banned from those rooms. + """ + + PATTERNS = admin_patterns("/users/(?P[^/]*)/cumulative_joined_room_count") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self.store = hs.get_datastores().main + + async def on_GET( + self, request: SynapseRequest, user_id: str + ) -> Tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + from_ts = parse_integer(request, "from_ts", required=True) + + joined_rooms = await self.store.get_rooms_for_user_by_date(user_id, from_ts) + + return HTTPStatus.OK, {"cumulative_joined_room_count": len(joined_rooms)} diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 32fa7b4ec4..d9f0c169e8 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -21,11 +21,10 @@ # import logging import random -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, List, Literal, Optional, Tuple from urllib.parse import urlparse import attr -from typing_extensions import Literal from twisted.web.server import Request @@ -48,7 +47,7 @@ from synapse.http.servlet import ( parse_string, ) from synapse.http.site import SynapseRequest -from synapse.metrics import threepid_send_requests +from synapse.metrics import SERVER_NAME_LABEL, threepid_send_requests from synapse.push.mailer import Mailer from synapse.types import JsonDict from synapse.types.rest import RequestBodyModel @@ -77,6 +76,7 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.hs = hs + self.server_name = hs.hostname self.datastore = hs.get_datastores().main self.config = hs.config self.identity_handler = hs.get_identity_handler() @@ -137,9 +137,11 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): self.mailer.send_password_reset_mail, body.next_link, ) - threepid_send_requests.labels(type="email", reason="password_reset").observe( - body.send_attempt - ) + threepid_send_requests.labels( + type="email", + reason="password_reset", + **{SERVER_NAME_LABEL: self.server_name}, + ).observe(body.send_attempt) # Wrap the session id in a JSON object return 200, {"sid": sid} @@ -326,6 +328,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.hs = hs + self.server_name = hs.hostname self.config = hs.config self.identity_handler = hs.get_identity_handler() self.store = self.hs.get_datastores().main @@ -351,6 +354,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): raise SynapseError( 400, "Adding an email to your account is disabled on this server", + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, ) body = parse_and_validate_json_object_from_request( @@ -394,9 +398,11 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): body.next_link, ) - threepid_send_requests.labels(type="email", reason="add_threepid").observe( - body.send_attempt - ) + threepid_send_requests.labels( + type="email", + reason="add_threepid", + **{SERVER_NAME_LABEL: self.server_name}, + ).observe(body.send_attempt) # Wrap the session id in a JSON object return 200, {"sid": sid} @@ -407,6 +413,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.hs = hs + self.server_name = hs.hostname super().__init__() self.store = self.hs.get_datastores().main self.identity_handler = hs.get_identity_handler() @@ -457,6 +464,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): raise SynapseError( 400, "Adding phone numbers to user account is not supported by this homeserver", + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, ) ret = await self.identity_handler.requestMsisdnToken( @@ -468,9 +476,11 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): body.next_link, ) - threepid_send_requests.labels(type="msisdn", reason="add_threepid").observe( - body.send_attempt - ) + threepid_send_requests.labels( + type="msisdn", + reason="add_threepid", + **{SERVER_NAME_LABEL: self.server_name}, + ).observe(body.send_attempt) logger.info("MSISDN %s: got response from identity server: %s", msisdn, ret) return 200, ret @@ -499,7 +509,9 @@ class AddThreepidEmailSubmitTokenServlet(RestServlet): "Adding emails have been disabled due to lack of an email config" ) raise SynapseError( - 400, "Adding an email to your account is disabled on this server" + 400, + "Adding an email to your account is disabled on this server", + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, ) sid = parse_string(request, "sid", required=True) @@ -601,7 +613,7 @@ class ThreepidRestServlet(RestServlet): # ThreePidBindRestServelet.PostBody with an `alias_generator` to handle # `threePidCreds` versus `three_pid_creds`. async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: - if self.hs.config.experimental.msc3861.enabled: + if self.hs.config.mas.enabled or self.hs.config.experimental.msc3861.enabled: raise NotFoundError(errcode=Codes.UNRECOGNIZED) if not self.hs.config.registration.enable_3pid_changes: @@ -893,23 +905,27 @@ class AccountStatusRestServlet(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + + ThreepidRestServlet(hs).register(http_server) + WhoamiRestServlet(hs).register(http_server) + + if not auth_delegated: + DeactivateAccountRestServlet(hs).register(http_server) + if hs.config.worker.worker_app is None: - if not hs.config.experimental.msc3861.enabled: + ThreepidBindRestServlet(hs).register(http_server) + ThreepidUnbindRestServlet(hs).register(http_server) + + if not auth_delegated: EmailPasswordRequestTokenRestServlet(hs).register(http_server) - DeactivateAccountRestServlet(hs).register(http_server) PasswordRestServlet(hs).register(http_server) EmailThreepidRequestTokenRestServlet(hs).register(http_server) MsisdnThreepidRequestTokenRestServlet(hs).register(http_server) AddThreepidEmailSubmitTokenServlet(hs).register(http_server) AddThreepidMsisdnSubmitTokenServlet(hs).register(http_server) - ThreepidRestServlet(hs).register(http_server) - if hs.config.worker.worker_app is None: - ThreepidBindRestServlet(hs).register(http_server) - ThreepidUnbindRestServlet(hs).register(http_server) - if not hs.config.experimental.msc3861.enabled: ThreepidAddRestServlet(hs).register(http_server) ThreepidDeleteRestServlet(hs).register(http_server) - WhoamiRestServlet(hs).register(http_server) - if hs.config.worker.worker_app is None and hs.config.experimental.msc3720_enabled: + if hs.config.experimental.msc3720_enabled: AccountStatusRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/appservice_ping.py b/synapse/rest/client/appservice_ping.py index d6b4e32453..1f9662a95a 100644 --- a/synapse/rest/client/appservice_ping.py +++ b/synapse/rest/client/appservice_ping.py @@ -2,7 +2,7 @@ # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright 2023 Tulir Asokan -# Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2023, 2025 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -53,6 +53,7 @@ class AppservicePingRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.as_api = hs.get_application_service_api() + self.scheduler = hs.get_application_service_scheduler() self.auth = hs.get_auth() async def on_POST( @@ -85,6 +86,10 @@ class AppservicePingRestServlet(RestServlet): start = time.monotonic() try: await self.as_api.ping(requester.app_service, txn_id) + + # We got a OK response, so if the AS needs to be recovered then lets recover it now. + # This sets off a task in the background and so is safe to execute and forget. + self.scheduler.txn_ctrl.force_retry(requester.app_service) except RequestTimedOutError as e: raise SynapseError( HTTPStatus.GATEWAY_TIMEOUT, diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index b8dca7c797..600bb51a7e 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -20,10 +20,11 @@ # import logging -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING from twisted.web.server import Request +from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.constants import LoginType from synapse.api.errors import LoginError, SynapseError from synapse.api.urls import CLIENT_API_PREFIX @@ -66,22 +67,30 @@ class AuthRestServlet(RestServlet): if not session: raise SynapseError(400, "No session supplied") - if ( - self.hs.config.experimental.msc3861.enabled - and stagetype == "org.matrix.cross_signing_reset" - ): - # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth + if stagetype == "org.matrix.cross_signing_reset": + if self.hs.config.mas.enabled: + assert isinstance(self.auth, MasDelegatedAuth) - auth = cast(MSC3861DelegatedAuth, self.auth) - - url = await auth.account_management_url() - if url is not None: + url = await self.auth.account_management_url() url = f"{url}?action=org.matrix.cross_signing_reset" - else: - url = await auth.issuer() - respond_with_redirect(request, str.encode(url)) + return respond_with_redirect( + request, + url.encode(), + ) + + elif self.hs.config.experimental.msc3861.enabled: + # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth + # We import lazily here because of the authlib requirement + from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth + + assert isinstance(self.auth, MSC3861DelegatedAuth) + + base = await self.auth.account_management_url() + if base is not None: + url = f"{base}?action=org.matrix.cross_signing_reset" + else: + url = await self.auth.issuer() + return respond_with_redirect(request, url.encode()) if stagetype == LoginType.RECAPTCHA: html = self.recaptcha_template.render( diff --git a/synapse/rest/client/auth_issuer.py b/synapse/rest/client/auth_metadata.py similarity index 52% rename from synapse/rest/client/auth_issuer.py rename to synapse/rest/client/auth_metadata.py index acd0399d85..4b5d997478 100644 --- a/synapse/rest/client/auth_issuer.py +++ b/synapse/rest/client/auth_metadata.py @@ -15,6 +15,7 @@ import logging import typing from typing import Tuple, cast +from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.errors import Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet @@ -32,6 +33,8 @@ logger = logging.getLogger(__name__) class AuthIssuerServlet(RestServlet): """ Advertises what OpenID Connect issuer clients should use to authorise users. + This endpoint was defined in a previous iteration of MSC2965, and is still + used by some clients. """ PATTERNS = client_patterns( @@ -46,13 +49,18 @@ class AuthIssuerServlet(RestServlet): self._auth = hs.get_auth() async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: - if self._config.experimental.msc3861.enabled: + if self._config.mas.enabled: + assert isinstance(self._auth, MasDelegatedAuth) + return 200, {"issuer": await self._auth.issuer()} + + elif self._config.experimental.msc3861.enabled: # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth # We import lazily here because of the authlib requirement from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - auth = cast(MSC3861DelegatedAuth, self._auth) - return 200, {"issuer": await auth.issuer()} + assert isinstance(self._auth, MSC3861DelegatedAuth) + return 200, {"issuer": await self._auth.issuer()} + else: # Wouldn't expect this to be reached: the servelet shouldn't have been # registered. Still, fail gracefully if we are registered for some reason. @@ -63,7 +71,52 @@ class AuthIssuerServlet(RestServlet): ) +class AuthMetadataServlet(RestServlet): + """ + Advertises the OAuth 2.0 server metadata for the homeserver. + """ + + PATTERNS = [ + *client_patterns( + "/auth_metadata$", + releases=("v1",), + ), + *client_patterns( + "/org.matrix.msc2965/auth_metadata$", + unstable=True, + releases=(), + ), + ] + + def __init__(self, hs: "HomeServer"): + super().__init__() + self._config = hs.config + self._auth = hs.get_auth() + + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + if self._config.mas.enabled: + assert isinstance(self._auth, MasDelegatedAuth) + return 200, await self._auth.auth_metadata() + + elif self._config.experimental.msc3861.enabled: + # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth + # We import lazily here because of the authlib requirement + from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth + + auth = cast(MSC3861DelegatedAuth, self._auth) + return 200, await auth.auth_metadata() + + else: + # Wouldn't expect this to be reached: the servlet shouldn't have been + # registered. Still, fail gracefully if we are registered for some reason. + raise SynapseError( + 404, + "OIDC discovery has not been configured on this homeserver", + Codes.NOT_FOUND, + ) + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - # We use the MSC3861 values as they are used by multiple MSCs - if hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: AuthIssuerServlet(hs).register(http_server) + AuthMetadataServlet(hs).register(http_server) diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 63b8a9364a..a279db1cc5 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -92,6 +92,28 @@ class CapabilitiesRestServlet(RestServlet): "enabled": self.config.experimental.msc3664_enabled, } + disallowed_profile_fields = [] + response["capabilities"]["m.profile_fields"] = {"enabled": True} + if not self.config.registration.enable_set_displayname: + disallowed_profile_fields.append("displayname") + if not self.config.registration.enable_set_avatar_url: + disallowed_profile_fields.append("avatar_url") + if disallowed_profile_fields: + response["capabilities"]["m.profile_fields"]["disallowed"] = ( + disallowed_profile_fields + ) + + # For transition from unstable to stable identifiers. + if self.config.experimental.msc4133_enabled: + response["capabilities"]["uk.tcpip.msc4133.profile_fields"] = response[ + "capabilities" + ]["m.profile_fields"] + + if self.config.experimental.msc4267_enabled: + response["capabilities"]["org.matrix.msc4267.forget_forced_upon_leave"] = { + "enabled": self.config.room.forget_on_leave, + } + return HTTPStatus.OK, response diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 6a45a5d130..0777abde7f 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -27,7 +27,6 @@ from typing import TYPE_CHECKING, List, Optional, Tuple from synapse._pydantic_compat import Extra, StrictStr from synapse.api import errors from synapse.api.errors import NotFoundError, SynapseError, UnrecognizedRequestError -from synapse.handlers.device import DeviceHandler from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -91,7 +90,6 @@ class DeleteDevicesRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler self.auth_handler = hs.get_auth_handler() @@ -114,15 +112,19 @@ class DeleteDevicesRestServlet(RestServlet): else: raise e - await self.auth_handler.validate_user_via_ui_auth( - requester, - request, - body.dict(exclude_unset=True), - "remove device(s) from your account", - # Users might call this multiple times in a row while cleaning up - # devices, allow a single UI auth session to be re-used. - can_skip_ui_auth=True, - ) + if requester.app_service and requester.app_service.msc4190_device_management: + # MSC4190 can skip UIA for this endpoint + pass + else: + await self.auth_handler.validate_user_via_ui_auth( + requester, + request, + body.dict(exclude_unset=True), + "remove device(s) from your account", + # Users might call this multiple times in a row while cleaning up + # devices, allow a single UI auth session to be re-used. + can_skip_ui_auth=True, + ) await self.device_handler.delete_devices( requester.user.to_string(), body.devices @@ -139,11 +141,12 @@ class DeviceRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler self.auth_handler = hs.get_auth_handler() self._msc3852_enabled = hs.config.experimental.msc3852_enabled - self._msc3861_oauth_delegation_enabled = hs.config.experimental.msc3861.enabled + self._auth_delegation_enabled = ( + hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + ) async def on_GET( self, request: SynapseRequest, device_id: str @@ -175,9 +178,6 @@ class DeviceRestServlet(RestServlet): async def on_DELETE( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: - if self._msc3861_oauth_delegation_enabled: - raise UnrecognizedRequestError(code=404) - requester = await self.auth.get_user_by_req(request) try: @@ -192,15 +192,24 @@ class DeviceRestServlet(RestServlet): else: raise - await self.auth_handler.validate_user_via_ui_auth( - requester, - request, - body.dict(exclude_unset=True), - "remove a device from your account", - # Users might call this multiple times in a row while cleaning up - # devices, allow a single UI auth session to be re-used. - can_skip_ui_auth=True, - ) + if requester.app_service and requester.app_service.msc4190_device_management: + # MSC4190 allows appservices to delete devices through this endpoint without UIA + # It's also allowed with MSC3861 enabled + pass + + else: + if self._auth_delegation_enabled: + raise UnrecognizedRequestError(code=404) + + await self.auth_handler.validate_user_via_ui_auth( + requester, + request, + body.dict(exclude_unset=True), + "remove a device from your account", + # Users might call this multiple times in a row while cleaning up + # devices, allow a single UI auth session to be re-used. + can_skip_ui_auth=True, + ) await self.device_handler.delete_devices( requester.user.to_string(), [device_id] @@ -216,6 +225,16 @@ class DeviceRestServlet(RestServlet): requester = await self.auth.get_user_by_req(request, allow_guest=True) body = parse_and_validate_json_object_from_request(request, self.PutBody) + + # MSC4190 allows appservices to create devices through this endpoint + if requester.app_service and requester.app_service.msc4190_device_management: + created = await self.device_handler.upsert_device( + user_id=requester.user.to_string(), + device_id=device_id, + display_name=body.display_name, + ) + return 201 if created else 200, {} + await self.device_handler.update_device( requester.user.to_string(), device_id, body.dict() ) @@ -281,7 +300,6 @@ class DehydratedDeviceServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -341,7 +359,6 @@ class ClaimDehydratedDeviceServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler class PostBody(RequestBodyModel): @@ -481,7 +498,6 @@ class DehydratedDeviceV2Servlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.e2e_keys_handler = hs.get_e2e_keys_handler() self.device_handler = handler @@ -559,18 +575,15 @@ class DehydratedDeviceV2Servlet(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if ( - hs.config.worker.worker_app is None - and not hs.config.experimental.msc3861.enabled - ): + auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + if not auth_delegated: DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) + DeviceRestServlet(hs).register(http_server) - if hs.config.worker.worker_app is None: - DeviceRestServlet(hs).register(http_server) - if hs.config.experimental.msc2697_enabled: - DehydratedDeviceServlet(hs).register(http_server) - ClaimDehydratedDeviceServlet(hs).register(http_server) - if hs.config.experimental.msc3814_enabled: - DehydratedDeviceV2Servlet(hs).register(http_server) - DehydratedDeviceEventsServlet(hs).register(http_server) + if hs.config.experimental.msc2697_enabled: + DehydratedDeviceServlet(hs).register(http_server) + ClaimDehydratedDeviceServlet(hs).register(http_server) + if hs.config.experimental.msc3814_enabled: + DehydratedDeviceV2Servlet(hs).register(http_server) + DehydratedDeviceEventsServlet(hs).register(http_server) diff --git a/synapse/rest/client/directory.py b/synapse/rest/client/directory.py index 98ba5c4c2a..479f489623 100644 --- a/synapse/rest/client/directory.py +++ b/synapse/rest/client/directory.py @@ -20,9 +20,7 @@ # import logging -from typing import TYPE_CHECKING, List, Optional, Tuple - -from typing_extensions import Literal +from typing import TYPE_CHECKING, List, Literal, Optional, Tuple from twisted.web.server import Request diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 7025662fdc..9f39889c75 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -23,8 +23,9 @@ import logging import re from collections import Counter -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, cast +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple +from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.errors import ( InteractiveAuthIncompleteError, InvalidAPICallError, @@ -404,19 +405,11 @@ class SigningKeyUploadServlet(RestServlet): if is_cross_signing_setup: # With MSC3861, UIA is not possible. Instead, the auth service has to # explicitly mark the master key as replaceable. - if self.hs.config.experimental.msc3861.enabled: + if self.hs.config.mas.enabled: if not master_key_updatable_without_uia: - # If MSC3861 is enabled, we can assume self.auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - auth = cast(MSC3861DelegatedAuth, self.auth) - - uri = await auth.account_management_url() - if uri is not None: - url = f"{uri}?action=org.matrix.cross_signing_reset" - else: - url = await auth.issuer() + assert isinstance(self.auth, MasDelegatedAuth) + url = await self.auth.account_management_url() + url = f"{url}?action=org.matrix.cross_signing_reset" # We use a dummy session ID as this isn't really a UIA flow, but we # reuse the same API shape for better client compatibility. @@ -437,6 +430,41 @@ class SigningKeyUploadServlet(RestServlet): "then try again.", }, ) + + elif self.hs.config.experimental.msc3861.enabled: + if not master_key_updatable_without_uia: + # If MSC3861 is enabled, we can assume self.auth is an instance of MSC3861DelegatedAuth + # We import lazily here because of the authlib requirement + from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth + + assert isinstance(self.auth, MSC3861DelegatedAuth) + + uri = await self.auth.account_management_url() + if uri is not None: + url = f"{uri}?action=org.matrix.cross_signing_reset" + else: + url = await self.auth.issuer() + + # We use a dummy session ID as this isn't really a UIA flow, but we + # reuse the same API shape for better client compatibility. + raise InteractiveAuthIncompleteError( + "dummy", + { + "session": "dummy", + "flows": [ + {"stages": ["org.matrix.cross_signing_reset"]}, + ], + "params": { + "org.matrix.cross_signing_reset": { + "url": url, + }, + }, + "msg": "To reset your end-to-end encryption cross-signing " + f"identity, you first need to approve it at {url} and " + "then try again.", + }, + ) + else: # Without MSC3861, we require UIA. await self.auth_handler.validate_user_via_ui_auth( @@ -504,6 +532,5 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: OneTimeKeyServlet(hs).register(http_server) if hs.config.experimental.msc3983_appservice_otk_claims: UnstableOneTimeKeyServlet(hs).register(http_server) - if hs.config.worker.worker_app is None: - SigningKeyUploadServlet(hs).register(http_server) - SignaturesUploadServlet(hs).register(http_server) + SigningKeyUploadServlet(hs).register(http_server) + SignaturesUploadServlet(hs).register(http_server) diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 3271b02d40..acb9111ad2 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -30,11 +30,10 @@ from typing import ( List, Optional, Tuple, + TypedDict, Union, ) -from typing_extensions import TypedDict - from synapse.api.constants import ApprovalNoticeMedium from synapse.api.errors import ( Codes, @@ -43,6 +42,7 @@ from synapse.api.errors import ( NotApprovedError, SynapseError, UserDeactivatedError, + UserLockedError, ) from synapse.api.ratelimiting import Ratelimiter from synapse.api.urls import CLIENT_API_PREFIX @@ -314,7 +314,9 @@ class LoginRestServlet(RestServlet): should_issue_refresh_token=should_issue_refresh_token, # The user represented by an appservice's configured sender_localpart # is not actually created in Synapse. - should_check_deactivated=qualified_user_id != appservice.sender, + should_check_deactivated_or_locked=( + qualified_user_id != appservice.sender.to_string() + ), request_info=request_info, ) @@ -368,7 +370,7 @@ class LoginRestServlet(RestServlet): auth_provider_id: Optional[str] = None, should_issue_refresh_token: bool = False, auth_provider_session_id: Optional[str] = None, - should_check_deactivated: bool = True, + should_check_deactivated_or_locked: bool = True, *, request_info: RequestInfo, ) -> LoginResponse: @@ -390,8 +392,8 @@ class LoginRestServlet(RestServlet): should_issue_refresh_token: True if this login should issue a refresh token alongside the access token. auth_provider_session_id: The session ID got during login from the SSO IdP. - should_check_deactivated: True if the user should be checked for - deactivation status before logging in. + should_check_deactivated_or_locked: True if the user should be checked for + deactivation or locked status before logging in. This exists purely for appservice's configured sender_localpart which doesn't have an associated user in the database. @@ -416,11 +418,14 @@ class LoginRestServlet(RestServlet): ) user_id = canonical_uid - # If the account has been deactivated, do not proceed with the login. - if should_check_deactivated: + # If the account has been deactivated or locked, do not proceed with the login. + if should_check_deactivated_or_locked: deactivated = await self._main_store.get_user_deactivated_status(user_id) if deactivated: raise UserDeactivatedError("This account has been deactivated") + locked = await self._main_store.get_user_locked_status(user_id) + if locked: + raise UserLockedError() device_id = login_submission.get("device_id") @@ -710,7 +715,7 @@ class CasTicketServlet(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: return LoginRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/logout.py b/synapse/rest/client/logout.py index e6b4a34d51..39c62b9e26 100644 --- a/synapse/rest/client/logout.py +++ b/synapse/rest/client/logout.py @@ -22,7 +22,6 @@ import logging from typing import TYPE_CHECKING, Tuple -from synapse.handlers.device import DeviceHandler from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest @@ -42,9 +41,7 @@ class LogoutRestServlet(RestServlet): super().__init__() self.auth = hs.get_auth() self._auth_handler = hs.get_auth_handler() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self._device_handler = handler + self._device_handler = hs.get_device_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req( @@ -71,9 +68,7 @@ class LogoutAllRestServlet(RestServlet): super().__init__() self.auth = hs.get_auth() self._auth_handler = hs.get_auth_handler() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self._device_handler = handler + self._device_handler = hs.get_device_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req( @@ -91,7 +86,7 @@ class LogoutAllRestServlet(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: return LogoutRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index 25b302370f..4c044ae900 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -102,10 +102,17 @@ class MediaConfigResource(RestServlet): self.clock = hs.get_clock() self.auth = hs.get_auth() self.limits_dict = {"m.upload.size": config.media.max_upload_size} + self.media_repository_callbacks = hs.get_module_api_callbacks().media_repository async def on_GET(self, request: SynapseRequest) -> None: - await self.auth.get_user_by_req(request) - respond_with_json(request, 200, self.limits_dict, send_cors=True) + requester = await self.auth.get_user_by_req(request) + user_specific_config = ( + await self.media_repository_callbacks.get_media_config_for_user( + requester.user.to_string(), + ) + ) + response = user_specific_config if user_specific_config else self.limits_dict + respond_with_json(request, 200, response, send_cors=True) class ThumbnailResource(RestServlet): diff --git a/synapse/rest/client/presence.py b/synapse/rest/client/presence.py index ecc52956e4..104d54cd89 100644 --- a/synapse/rest/client/presence.py +++ b/synapse/rest/client/presence.py @@ -24,7 +24,8 @@ import logging from typing import TYPE_CHECKING, Tuple -from synapse.api.errors import AuthError, SynapseError +from synapse.api.errors import AuthError, Codes, LimitExceededError, SynapseError +from synapse.api.ratelimiting import Ratelimiter from synapse.handlers.presence import format_user_presence_state from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request @@ -48,6 +49,14 @@ class PresenceStatusRestServlet(RestServlet): self.presence_handler = hs.get_presence_handler() self.clock = hs.get_clock() self.auth = hs.get_auth() + self.store = hs.get_datastores().main + + # Ratelimiter for presence updates, keyed by requester. + self._presence_per_user_limiter = Ratelimiter( + store=self.store, + clock=self.clock, + cfg=hs.config.ratelimiting.rc_presence_per_user, + ) async def on_GET( self, request: SynapseRequest, user_id: str @@ -82,6 +91,17 @@ class PresenceStatusRestServlet(RestServlet): if requester.user != user: raise AuthError(403, "Can only set your own presence state") + # ignore the presence update if the ratelimit is exceeded + try: + await self._presence_per_user_limiter.ratelimit(requester) + except LimitExceededError as e: + logger.debug("User presence ratelimit exceeded; ignoring it.") + return 429, { + "errcode": Codes.LIMIT_EXCEEDED, + "error": "Too many requests", + "retry_after_ms": e.retry_after_ms, + } + state = {} content = parse_json_object_from_request(request) diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 7a95b9445d..8bc532c811 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -21,10 +21,13 @@ """This module contains REST servlets to do with profile: /profile/""" +import re from http import HTTPStatus from typing import TYPE_CHECKING, Tuple +from synapse.api.constants import ProfileFields from synapse.api.errors import Codes, SynapseError +from synapse.handlers.profile import MAX_CUSTOM_FIELD_LEN from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -33,7 +36,8 @@ from synapse.http.servlet import ( ) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, JsonValue, UserID +from synapse.util.stringutils import is_namedspaced_grammar if TYPE_CHECKING: from synapse.server import HomeServer @@ -53,153 +57,6 @@ def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: return propagate -class ProfileDisplaynameRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)/displayname", v1=True) - CATEGORY = "Event sending requests" - - def __init__(self, hs: "HomeServer"): - super().__init__() - self.hs = hs - self.profile_handler = hs.get_profile_handler() - self.auth = hs.get_auth() - - async def on_GET( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user - - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - user = UserID.from_string(user_id) - await self.profile_handler.check_profile_query_allowed(user, requester_user) - - displayname = await self.profile_handler.get_displayname(user) - - ret = {} - if displayname is not None: - ret["displayname"] = displayname - - return 200, ret - - async def on_PUT( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - requester = await self.auth.get_user_by_req(request, allow_guest=True) - user = UserID.from_string(user_id) - is_admin = await self.auth.is_server_admin(requester) - - content = parse_json_object_from_request(request) - - try: - new_name = content["displayname"] - except Exception: - raise SynapseError( - code=400, - msg="Unable to parse name", - errcode=Codes.BAD_JSON, - ) - - propagate = _read_propagate(self.hs, request) - - requester_suspended = ( - await self.hs.get_datastores().main.get_user_suspended_status( - requester.user.to_string() - ) - ) - - if requester_suspended: - raise SynapseError( - 403, - "Updating displayname while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) - - await self.profile_handler.set_displayname( - user, requester, new_name, is_admin, propagate=propagate - ) - - return 200, {} - - -class ProfileAvatarURLRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)/avatar_url", v1=True) - CATEGORY = "Event sending requests" - - def __init__(self, hs: "HomeServer"): - super().__init__() - self.hs = hs - self.profile_handler = hs.get_profile_handler() - self.auth = hs.get_auth() - - async def on_GET( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user - - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - user = UserID.from_string(user_id) - await self.profile_handler.check_profile_query_allowed(user, requester_user) - - avatar_url = await self.profile_handler.get_avatar_url(user) - - ret = {} - if avatar_url is not None: - ret["avatar_url"] = avatar_url - - return 200, ret - - async def on_PUT( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - requester = await self.auth.get_user_by_req(request) - user = UserID.from_string(user_id) - is_admin = await self.auth.is_server_admin(requester) - - content = parse_json_object_from_request(request) - try: - new_avatar_url = content["avatar_url"] - except KeyError: - raise SynapseError( - 400, "Missing key 'avatar_url'", errcode=Codes.MISSING_PARAM - ) - - propagate = _read_propagate(self.hs, request) - - requester_suspended = ( - await self.hs.get_datastores().main.get_user_suspended_status( - requester.user.to_string() - ) - ) - - if requester_suspended: - raise SynapseError( - 403, - "Updating avatar URL while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) - - await self.profile_handler.set_avatar_url( - user, requester, new_avatar_url, is_admin, propagate=propagate - ) - - return 200, {} - - class ProfileRestServlet(RestServlet): PATTERNS = client_patterns("/profile/(?P[^/]*)", v1=True) CATEGORY = "Event sending requests" @@ -227,19 +84,208 @@ class ProfileRestServlet(RestServlet): user = UserID.from_string(user_id) await self.profile_handler.check_profile_query_allowed(user, requester_user) - displayname = await self.profile_handler.get_displayname(user) - avatar_url = await self.profile_handler.get_avatar_url(user) - - ret = {} - if displayname is not None: - ret["displayname"] = displayname - if avatar_url is not None: - ret["avatar_url"] = avatar_url + ret = await self.profile_handler.get_profile(user_id) return 200, ret +class ProfileFieldRestServlet(RestServlet): + PATTERNS = [ + *client_patterns( + "/profile/(?P[^/]*)/(?Pdisplayname)", v1=True + ), + *client_patterns( + "/profile/(?P[^/]*)/(?Pavatar_url)", v1=True + ), + re.compile( + r"^/_matrix/client/v3/profile/(?P[^/]*)/(?P[^/]*)" + ), + ] + + CATEGORY = "Event sending requests" + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.hs = hs + self.profile_handler = hs.get_profile_handler() + self.auth = hs.get_auth() + if hs.config.experimental.msc4133_enabled: + self.PATTERNS.append( + re.compile( + r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" + ) + ) + + async def on_GET( + self, request: SynapseRequest, user_id: str, field_name: str + ) -> Tuple[int, JsonDict]: + requester_user = None + + if self.hs.config.server.require_auth_for_profile_requests: + requester = await self.auth.get_user_by_req(request) + requester_user = requester.user + + if not UserID.is_valid(user_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM + ) + + if not field_name: + raise SynapseError(400, "Field name too short", errcode=Codes.INVALID_PARAM) + + if len(field_name.encode("utf-8")) > MAX_CUSTOM_FIELD_LEN: + raise SynapseError(400, "Field name too long", errcode=Codes.KEY_TOO_LARGE) + if not is_namedspaced_grammar(field_name): + raise SynapseError( + 400, + "Field name does not follow Common Namespaced Identifier Grammar", + errcode=Codes.INVALID_PARAM, + ) + + user = UserID.from_string(user_id) + await self.profile_handler.check_profile_query_allowed(user, requester_user) + + if field_name == ProfileFields.DISPLAYNAME: + field_value: JsonValue = await self.profile_handler.get_displayname(user) + elif field_name == ProfileFields.AVATAR_URL: + field_value = await self.profile_handler.get_avatar_url(user) + else: + field_value = await self.profile_handler.get_profile_field(user, field_name) + + return 200, {field_name: field_value} + + async def on_PUT( + self, request: SynapseRequest, user_id: str, field_name: str + ) -> Tuple[int, JsonDict]: + if not UserID.is_valid(user_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM + ) + + # Guest users are able to set their own displayname. + requester = await self.auth.get_user_by_req( + request, allow_guest=field_name == ProfileFields.DISPLAYNAME + ) + user = UserID.from_string(user_id) + is_admin = await self.auth.is_server_admin(requester) + + if not field_name: + raise SynapseError(400, "Field name too short", errcode=Codes.INVALID_PARAM) + + if len(field_name.encode("utf-8")) > MAX_CUSTOM_FIELD_LEN: + raise SynapseError(400, "Field name too long", errcode=Codes.KEY_TOO_LARGE) + if not is_namedspaced_grammar(field_name): + raise SynapseError( + 400, + "Field name does not follow Common Namespaced Identifier Grammar", + errcode=Codes.INVALID_PARAM, + ) + + content = parse_json_object_from_request(request) + try: + new_value = content[field_name] + except KeyError: + raise SynapseError( + 400, f"Missing key '{field_name}'", errcode=Codes.MISSING_PARAM + ) + + propagate = _read_propagate(self.hs, request) + + requester_suspended = ( + await self.hs.get_datastores().main.get_user_suspended_status( + requester.user.to_string() + ) + ) + + if requester_suspended: + raise SynapseError( + 403, + "Updating profile while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) + + if field_name == ProfileFields.DISPLAYNAME: + await self.profile_handler.set_displayname( + user, requester, new_value, is_admin, propagate=propagate + ) + elif field_name == ProfileFields.AVATAR_URL: + await self.profile_handler.set_avatar_url( + user, requester, new_value, is_admin, propagate=propagate + ) + else: + await self.profile_handler.set_profile_field( + user, requester, field_name, new_value, is_admin + ) + + return 200, {} + + async def on_DELETE( + self, request: SynapseRequest, user_id: str, field_name: str + ) -> Tuple[int, JsonDict]: + if not UserID.is_valid(user_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM + ) + + # Guest users are able to set their own displayname. + requester = await self.auth.get_user_by_req( + request, allow_guest=field_name == ProfileFields.DISPLAYNAME + ) + user = UserID.from_string(user_id) + is_admin = await self.auth.is_server_admin(requester) + + if not field_name: + raise SynapseError(400, "Field name too short", errcode=Codes.INVALID_PARAM) + + if len(field_name.encode("utf-8")) > MAX_CUSTOM_FIELD_LEN: + raise SynapseError(400, "Field name too long", errcode=Codes.KEY_TOO_LARGE) + if not is_namedspaced_grammar(field_name): + raise SynapseError( + 400, + "Field name does not follow Common Namespaced Identifier Grammar", + errcode=Codes.INVALID_PARAM, + ) + + propagate = _read_propagate(self.hs, request) + + requester_suspended = ( + await self.hs.get_datastores().main.get_user_suspended_status( + requester.user.to_string() + ) + ) + + if requester_suspended: + raise SynapseError( + 403, + "Updating profile while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) + + if field_name == ProfileFields.DISPLAYNAME: + await self.profile_handler.set_displayname( + user, requester, "", is_admin, propagate=propagate + ) + elif field_name == ProfileFields.AVATAR_URL: + await self.profile_handler.set_avatar_url( + user, requester, "", is_admin, propagate=propagate + ) + else: + await self.profile_handler.delete_profile_field( + user, requester, field_name, is_admin + ) + + return 200, {} + + +class UnstableProfileFieldRestServlet(ProfileFieldRestServlet): + re.compile( + r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" + ) + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - ProfileDisplaynameRestServlet(hs).register(http_server) - ProfileAvatarURLRestServlet(hs).register(http_server) + # The specific field endpoint *must* appear before the generic profile endpoint. + ProfileFieldRestServlet(hs).register(http_server) ProfileRestServlet(hs).register(http_server) + if hs.config.experimental.msc4133_enabled: + UnstableProfileFieldRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index af042504c9..c20de89bf7 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -19,9 +19,11 @@ # # +from http import HTTPStatus from typing import TYPE_CHECKING, List, Tuple, Union from synapse.api.errors import ( + Codes, NotFoundError, StoreError, SynapseError, @@ -239,6 +241,15 @@ def _rule_spec_from_path(path: List[str]) -> RuleSpec: def _rule_tuple_from_request_object( rule_template: str, rule_id: str, req_obj: JsonDict ) -> Tuple[List[JsonDict], List[Union[str, JsonDict]]]: + if rule_template == "postcontent": + # postcontent is from MSC4306, which says that clients + # cannot create their own postcontent rules right now. + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "user-defined rules using `postcontent` are not accepted", + errcode=Codes.INVALID_PARAM, + ) + if rule_template in ["override", "underride"]: if "conditions" not in req_obj: raise InvalidRuleException("Missing 'conditions'") diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index 89203dc45a..4bf93f485c 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -39,9 +39,7 @@ logger = logging.getLogger(__name__) class ReceiptRestServlet(RestServlet): PATTERNS = client_patterns( - "/rooms/(?P[^/]*)" - "/receipt/(?P[^/]*)" - "/(?P[^/]*)$" + "/rooms/(?P[^/]*)/receipt/(?P[^/]*)/(?P[^/]*)$" ) CATEGORY = "Receipts requests" diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 61e1436841..102c04bb67 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -56,7 +56,7 @@ from synapse.http.servlet import ( parse_string, ) from synapse.http.site import SynapseRequest -from synapse.metrics import threepid_send_requests +from synapse.metrics import SERVER_NAME_LABEL, threepid_send_requests from synapse.push.mailer import Mailer from synapse.types import JsonDict from synapse.util.msisdn import phone_number_to_msisdn @@ -82,6 +82,7 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.hs = hs + self.server_name = hs.hostname self.identity_handler = hs.get_identity_handler() self.config = hs.config @@ -163,9 +164,11 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): next_link, ) - threepid_send_requests.labels(type="email", reason="register").observe( - send_attempt - ) + threepid_send_requests.labels( + type="email", + reason="register", + **{SERVER_NAME_LABEL: self.server_name}, + ).observe(send_attempt) # Wrap the session id in a JSON object return 200, {"sid": sid} @@ -177,6 +180,7 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.hs = hs + self.server_name = hs.hostname self.identity_handler = hs.get_identity_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -240,9 +244,11 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): next_link, ) - threepid_send_requests.labels(type="msisdn", reason="register").observe( - send_attempt - ) + threepid_send_requests.labels( + type="msisdn", + reason="register", + **{SERVER_NAME_LABEL: self.server_name}, + ).observe(send_attempt) return 200, ret @@ -323,10 +329,12 @@ class UsernameAvailabilityRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.hs = hs + self.server_name = hs.hostname self.registration_handler = hs.get_registration_handler() self.ratelimiter = FederationRateLimiter( - hs.get_clock(), - FederationRatelimitSettings( + our_server_name=self.server_name, + clock=hs.get_clock(), + config=FederationRatelimitSettings( # Time window of 2s window_size=2000, # Artificially delay requests if rate > sleep_limit/window_size @@ -771,9 +779,12 @@ class RegisterRestServlet(RestServlet): body: JsonDict, should_issue_refresh_token: bool = False, ) -> JsonDict: - user_id = await self.registration_handler.appservice_register( + user_id, appservice = await self.registration_handler.appservice_register( username, as_token ) + if appservice.msc4190_device_management: + body["inhibit_login"] = True + return await self._create_registration_details( user_id, body, @@ -905,6 +916,14 @@ class RegisterAppServiceOnlyRestServlet(RestServlet): await self.ratelimiter.ratelimit(None, client_addr, update=False) + # Allow only ASes to use this API. + if body.get("type") != APP_SERVICE_REGISTRATION_TYPE: + raise SynapseError( + 403, + "Registration has been disabled. Only m.login.application_service registrations are allowed.", + errcode=Codes.FORBIDDEN, + ) + kind = parse_string(request, "kind", default="user") if kind == "guest": @@ -920,10 +939,6 @@ class RegisterAppServiceOnlyRestServlet(RestServlet): if not isinstance(desired_username, str) or len(desired_username) > 512: raise SynapseError(400, "Invalid username") - # Allow only ASes to use this API. - if body.get("type") != APP_SERVICE_REGISTRATION_TYPE: - raise SynapseError(403, "Non-application service registration type") - if not self.auth.has_access_token(request): raise SynapseError( 400, @@ -937,7 +952,7 @@ class RegisterAppServiceOnlyRestServlet(RestServlet): as_token = self.auth.get_access_token_from_request(request) - user_id = await self.registration_handler.appservice_register( + user_id, _ = await self.registration_handler.appservice_register( desired_username, as_token ) return 200, {"user_id": user_id} @@ -1029,7 +1044,7 @@ def _calculate_registration_flows( def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: RegisterAppServiceOnlyRestServlet(hs).register(http_server) return diff --git a/synapse/rest/client/rendezvous.py b/synapse/rest/client/rendezvous.py index 02f166b4ea..a1808847f0 100644 --- a/synapse/rest/client/rendezvous.py +++ b/synapse/rest/client/rendezvous.py @@ -44,9 +44,9 @@ class MSC4108DelegationRendezvousServlet(RestServlet): redirection_target: Optional[str] = ( hs.config.experimental.msc4108_delegation_endpoint ) - assert ( - redirection_target is not None - ), "Servlet is only registered if there is a delegation target" + assert redirection_target is not None, ( + "Servlet is only registered if there is a delegation target" + ) self.endpoint = redirection_target.encode("utf-8") async def on_POST(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index 949f077035..81faf38a7f 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -20,13 +20,11 @@ # import logging -import re from http import HTTPStatus from typing import TYPE_CHECKING, Tuple from synapse._pydantic_compat import StrictStr from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError -from synapse.api.urls import CLIENT_API_PREFIX from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -71,7 +69,10 @@ class ReportEventRestServlet(RestServlet): "Param 'reason' must be a string", Codes.BAD_JSON, ) - if type(body.get("score", 0)) is not int: # noqa: E721 + if ( + not self.hs.config.experimental.msc4277_enabled + and type(body.get("score", 0)) is not int + ): # noqa: E721 raise SynapseError( HTTPStatus.BAD_REQUEST, "Param 'score' must be an integer", @@ -87,10 +88,15 @@ class ReportEventRestServlet(RestServlet): event = None if event is None: - raise NotFoundError( - "Unable to report event: " - "it does not exist or you aren't able to see it." - ) + if self.hs.config.experimental.msc4277_enabled: + # Respond with 200 and no content regardless of whether the event + # exists to prevent enumeration attacks. + return 200, {} + else: + raise NotFoundError( + "Unable to report event: " + "it does not exist or you aren't able to see it." + ) await self.store.add_event_report( room_id=room_id, @@ -127,16 +133,6 @@ class ReportRoomRestServlet(RestServlet): self.clock = hs.get_clock() self.store = hs.get_datastores().main - # TODO: Remove the unstable variant after 2-3 releases - # https://github.com/element-hq/synapse/issues/17373 - if hs.config.experimental.msc4151_enabled: - self.PATTERNS.append( - re.compile( - f"^{CLIENT_API_PREFIX}/unstable/org.matrix.msc4151" - "/rooms/(?P[^/]*)/report$" - ) - ) - class PostBody(RequestBodyModel): reason: StrictStr @@ -150,7 +146,12 @@ class ReportRoomRestServlet(RestServlet): room = await self.store.get_room(room_id) if room is None: - raise NotFoundError("Room does not exist") + if self.hs.config.experimental.msc4277_enabled: + # Respond with 200 and no content regardless of whether the room + # exists to prevent enumeration attacks. + return 200, {} + else: + raise NotFoundError("Room does not exist") await self.store.add_room_report( room_id=room_id, @@ -162,6 +163,44 @@ class ReportRoomRestServlet(RestServlet): return 200, {} +class ReportUserRestServlet(RestServlet): + """This endpoint lets clients report a user for abuse. + + Introduced by MSC4260: https://github.com/matrix-org/matrix-spec-proposals/pull/4260 + """ + + PATTERNS = list( + client_patterns( + "/users/(?P[^/]*)/report$", + releases=("v3",), + unstable=False, + v1=False, + ) + ) + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.hs = hs + self.auth = hs.get_auth() + self.clock = hs.get_clock() + self.store = hs.get_datastores().main + self.handler = hs.get_reports_handler() + + class PostBody(RequestBodyModel): + reason: StrictStr + + async def on_POST( + self, request: SynapseRequest, target_user_id: str + ) -> Tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + body = parse_and_validate_json_object_from_request(request, self.PostBody) + + await self.handler.report_user(requester, target_user_id, body.reason) + + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ReportEventRestServlet(hs).register(http_server) ReportRoomRestServlet(hs).register(http_server) + ReportUserRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 8883cd6bc0..64deae7650 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -44,7 +44,11 @@ from synapse.api.errors import ( UnredactedContentDeletedError, ) from synapse.api.filtering import Filter -from synapse.events.utils import SerializeEventConfig, format_event_for_client_v2 +from synapse.events.utils import ( + SerializeEventConfig, + format_event_for_client_v2, + serialize_event, +) from synapse.http.server import HttpServer from synapse.http.servlet import ( ResolveRoomIdMixin, @@ -61,9 +65,11 @@ from synapse.http.servlet import ( from synapse.http.site import SynapseRequest from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import set_tag +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.rest.client._base import client_patterns from synapse.rest.client.transactions import HttpTransactionCache +from synapse.state import CREATE_KEY, POWER_KEY from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, Requester, StreamToken, ThirdPartyInstanceID, UserID from synapse.types.state import StateFilter @@ -115,7 +121,7 @@ messsages_response_timer = Histogram( # picture of /messages response time for bigger rooms. We don't want the # tiny rooms that can always respond fast skewing our results when we're trying # to optimize the bigger cases. - ["room_size"], + labelnames=["room_size", SERVER_NAME_LABEL], buckets=( 0.005, 0.01, @@ -197,7 +203,9 @@ class RoomStateEventRestServlet(RestServlet): self.message_handler = hs.get_message_handler() self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() + self.clock = hs.get_clock() self._max_event_delay_ms = hs.config.server.max_event_delay_ms + self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker def register(self, http_server: HttpServer) -> None: # /rooms/$roomid/state/$eventtype @@ -266,7 +274,14 @@ class RoomStateEventRestServlet(RestServlet): raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) if format == "event": - event = format_event_for_client_v2(data.get_dict()) + event = serialize_event( + data, + self.clock.time_msec(), + config=SerializeEventConfig( + event_format=format_event_for_client_v2, + requester=requester, + ), + ) return 200, event elif format == "content": return 200, data.get_dict()["content"] @@ -289,6 +304,25 @@ class RoomStateEventRestServlet(RestServlet): content = parse_json_object_from_request(request) + is_requester_admin = await self.auth.is_server_admin(requester) + if not is_requester_admin: + spam_check = ( + await self._spam_checker_module_callbacks.user_may_send_state_event( + user_id=requester.user.to_string(), + room_id=room_id, + event_type=event_type, + state_key=state_key, + content=content, + ) + ) + if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: + raise SynapseError( + 403, + "You are not permitted to send the state event", + errcode=spam_check[0], + additional_fields=spam_check[1], + ) + origin_server_ts = None if requester.app_service: origin_server_ts = parse_integer(request, "ts") @@ -768,6 +802,7 @@ class RoomMessageListRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self._hs = hs + self.server_name = hs.hostname self.clock = hs.get_clock() self.pagination_handler = hs.get_pagination_handler() self.auth = hs.get_auth() @@ -783,9 +818,9 @@ class RoomMessageListRestServlet(RestServlet): # decorator on `get_number_joined_users_in_room` doesn't play well with # the type system. Maybe in the future, it can use some ParamSpec # wizardry to fix it up. - room_member_count_deferred = run_in_background( # type: ignore[call-arg] + room_member_count_deferred = run_in_background( # type: ignore[call-overload] self.store.get_number_joined_users_in_room, - room_id, # type: ignore[arg-type] + room_id, ) requester = await self.auth.get_user_by_req(request, allow_guest=True) @@ -816,7 +851,8 @@ class RoomMessageListRestServlet(RestServlet): processing_end_time = self.clock.time_msec() room_member_count = await make_deferred_yieldable(room_member_count_deferred) messsages_response_timer.labels( - room_size=_RoomSize.from_member_count(room_member_count) + room_size=_RoomSize.from_member_count(room_member_count), + **{SERVER_NAME_LABEL: self.server_name}, ).observe((processing_end_time - processing_start_time) / 1000) return 200, msgs @@ -904,16 +940,16 @@ class RoomEventServlet(RestServlet): if include_unredacted_content and not await self.auth.is_server_admin( requester ): - power_level_event = ( - await self._storage_controllers.state.get_current_state_event( - room_id, EventTypes.PowerLevels, "" - ) + auth_events = await self._storage_controllers.state.get_current_state( + room_id, + StateFilter.from_types( + [ + POWER_KEY, + CREATE_KEY, + ] + ), ) - auth_events = {} - if power_level_event: - auth_events[(EventTypes.PowerLevels, "")] = power_level_event - redact_level = event_auth.get_named_level(auth_events, "redact", 50) user_level = event_auth.get_user_power_level( requester.user.to_string(), auth_events @@ -1067,6 +1103,7 @@ class RoomMembershipRestServlet(TransactionRestServlet): super().__init__(hs) self.room_member_handler = hs.get_room_member_handler() self.auth = hs.get_auth() + self.config = hs.config def register(self, http_server: HttpServer) -> None: # /rooms/$roomid/[join|invite|leave|ban|unban|kick] @@ -1090,12 +1127,12 @@ class RoomMembershipRestServlet(TransactionRestServlet): }: raise AuthError(403, "Guest access not allowed") - content = parse_json_object_from_request(request, allow_empty_body=True) + request_body = parse_json_object_from_request(request, allow_empty_body=True) if membership_action == "invite" and all( - key in content for key in ("medium", "address") + key in request_body for key in ("medium", "address") ): - if not all(key in content for key in ("id_server", "id_access_token")): + if not all(key in request_body for key in ("id_server", "id_access_token")): raise SynapseError( HTTPStatus.BAD_REQUEST, "`id_server` and `id_access_token` are required when doing 3pid invite", @@ -1106,12 +1143,12 @@ class RoomMembershipRestServlet(TransactionRestServlet): await self.room_member_handler.do_3pid_invite( room_id, requester.user, - content["medium"], - content["address"], - content["id_server"], + request_body["medium"], + request_body["address"], + request_body["id_server"], requester, txn_id, - content["id_access_token"], + request_body["id_access_token"], ) except ShadowBanError: # Pretend the request succeeded. @@ -1120,12 +1157,19 @@ class RoomMembershipRestServlet(TransactionRestServlet): target = requester.user if membership_action in ["invite", "ban", "unban", "kick"]: - assert_params_in_dict(content, ["user_id"]) - target = UserID.from_string(content["user_id"]) + assert_params_in_dict(request_body, ["user_id"]) + target = UserID.from_string(request_body["user_id"]) event_content = None - if "reason" in content: - event_content = {"reason": content["reason"]} + if "reason" in request_body: + event_content = {"reason": request_body["reason"]} + if self.config.experimental.msc4293_enabled: + if "org.matrix.msc4293.redact_events" in request_body: + if event_content is None: + event_content = {} + event_content["org.matrix.msc4293.redact_events"] = request_body[ + "org.matrix.msc4293.redact_events" + ] try: await self.room_member_handler.update_membership( @@ -1134,7 +1178,7 @@ class RoomMembershipRestServlet(TransactionRestServlet): room_id=room_id, action=membership_action, txn_id=txn_id, - third_party_signed=content.get("third_party_signed", None), + third_party_signed=request_body.get("third_party_signed", None), content=event_content, ) except ShadowBanError: @@ -1180,6 +1224,7 @@ class RoomRedactEventRestServlet(TransactionRestServlet): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.event_creation_handler = hs.get_event_creation_handler() self.auth = hs.get_auth() self._store = hs.get_datastores().main @@ -1264,6 +1309,7 @@ class RoomRedactEventRestServlet(TransactionRestServlet): if with_relations: run_as_background_process( "redact_related_events", + self.server_name, self._relation_handler.redact_events_related_to, requester=requester, event_id=event_id, @@ -1517,6 +1563,7 @@ class RoomHierarchyRestServlet(RestServlet): super().__init__() self._auth = hs.get_auth() self._room_summary_handler = hs.get_room_summary_handler() + self.msc4235_enabled = hs.config.experimental.msc4235_enabled async def on_GET( self, request: SynapseRequest, room_id: str @@ -1526,6 +1573,15 @@ class RoomHierarchyRestServlet(RestServlet): max_depth = parse_integer(request, "max_depth") limit = parse_integer(request, "limit") + # twisted.web.server.Request.args is incorrectly defined as Optional[Any] + remote_room_hosts = None + if self.msc4235_enabled: + args: Dict[bytes, List[bytes]] = request.args # type: ignore + via_param = parse_strings_from_args( + args, "org.matrix.msc4235.via", required=False + ) + remote_room_hosts = tuple(via_param or []) + return 200, await self._room_summary_handler.get_room_hierarchy( requester, room_id, @@ -1533,6 +1589,7 @@ class RoomHierarchyRestServlet(RestServlet): max_depth=max_depth, limit=limit, from_token=parse_string(request, "from"), + remote_room_hosts=remote_room_hosts, ) diff --git a/synapse/rest/client/room_upgrade_rest_servlet.py b/synapse/rest/client/room_upgrade_rest_servlet.py index 130ae31619..a9717781b0 100644 --- a/synapse/rest/client/room_upgrade_rest_servlet.py +++ b/synapse/rest/client/room_upgrade_rest_servlet.py @@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Tuple from synapse.api.errors import Codes, ShadowBanError, SynapseError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.event_auth import check_valid_additional_creators from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.http.server import HttpServer from synapse.http.servlet import ( @@ -85,13 +86,18 @@ class RoomUpgradeRestServlet(RestServlet): "Your homeserver does not support this room version", Codes.UNSUPPORTED_ROOM_VERSION, ) + additional_creators = None + if new_version.msc4289_creator_power_enabled: + additional_creators = content.get("additional_creators") + if additional_creators is not None: + check_valid_additional_creators(additional_creators) try: async with self._worker_lock_handler.acquire_read_write_lock( NEW_EVENT_DURING_PURGE_LOCK_NAME, room_id, write=False ): new_room_id = await self._room_creation_handler.upgrade_room( - requester, room_id, new_version + requester, room_id, new_version, additional_creators ) except ShadowBanError: # Generate a random room ID. diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 5c62a74f41..c424ca5325 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -23,10 +23,13 @@ import logging from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple, Union +import attr + from synapse.api.constants import AccountDataTypes, EduTypes, Membership, PresenceState from synapse.api.errors import Codes, StoreError, SynapseError from synapse.api.filtering import FilterCollection from synapse.api.presence import UserPresenceState +from synapse.api.ratelimiting import Ratelimiter from synapse.events.utils import ( SerializeEventConfig, format_event_for_client_v2_without_room_id, @@ -41,7 +44,6 @@ from synapse.handlers.sync import ( KnockedSyncResult, SyncConfig, SyncResult, - SyncVersion, ) from synapse.http.server import HttpServer from synapse.http.servlet import ( @@ -110,6 +112,7 @@ class SyncRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.hs = hs + self.server_name = hs.hostname self.auth = hs.get_auth() self.store = hs.get_datastores().main self.sync_handler = hs.get_sync_handler() @@ -124,6 +127,14 @@ class SyncRestServlet(RestServlet): self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, cache_name="sync_valid_filter", + server_name=self.server_name, + ) + + # Ratelimiter for presence updates, keyed by requester. + self._presence_per_user_limiter = Ratelimiter( + store=self.store, + clock=self.clock, + cfg=hs.config.ratelimiting.rc_presence_per_user, ) async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -239,7 +250,13 @@ class SyncRestServlet(RestServlet): # send any outstanding server notices to the user. await self._server_notices_sender.on_user_syncing(user.to_string()) - affect_presence = set_presence != PresenceState.OFFLINE + # ignore the presence update if the ratelimit is exceeded but do not pause the request + allowed, _ = await self._presence_per_user_limiter.can_do_action(requester) + if not allowed: + affect_presence = False + logger.debug("User set_presence ratelimit exceeded; ignoring it.") + else: + affect_presence = set_presence != PresenceState.OFFLINE context = await self.presence_handler.user_syncing( user.to_string(), @@ -251,7 +268,6 @@ class SyncRestServlet(RestServlet): sync_result = await self.sync_handler.wait_for_sync_for_user( requester, sync_config, - SyncVersion.SYNC_V2, request_key, since_token=since_token, timeout=timeout, @@ -436,7 +452,12 @@ class SyncRestServlet(RestServlet): ) unsigned = dict(invite.get("unsigned", {})) invite["unsigned"] = unsigned - invited_state = list(unsigned.pop("invite_room_state", [])) + + invited_state = unsigned.pop("invite_room_state", []) + if not isinstance(invited_state, list): + invited_state = [] + + invited_state = list(invited_state) invited_state.append(invite) invited[room.room_id] = {"invite_state": {"events": invited_state}} @@ -476,7 +497,10 @@ class SyncRestServlet(RestServlet): # Extract the stripped room state from the unsigned dict # This is for clients to get a little bit of information about # the room they've knocked on, without revealing any sensitive information - knocked_state = list(unsigned.pop("knock_room_state", [])) + knocked_state = unsigned.pop("knock_room_state", []) + if not isinstance(knocked_state, list): + knocked_state = [] + knocked_state = list(knocked_state) # Append the actual knock membership event itself as well. This provides # the client with: @@ -608,185 +632,23 @@ class SyncRestServlet(RestServlet): return result -class SlidingSyncE2eeRestServlet(RestServlet): - """ - API endpoint for MSC3575 Sliding Sync `/sync/e2ee`. This is being introduced as part - of Sliding Sync but doesn't have any sliding window component. It's just a way to - get E2EE events without having to sit through a big initial sync (`/sync` v2). And - we can avoid encryption events being backed up by the main sync response. - - Having To-Device messages split out to this sync endpoint also helps when clients - need to have 2 or more sync streams open at a time, e.g a push notification process - and a main process. This can cause the two processes to race to fetch the To-Device - events, resulting in the need for complex synchronisation rules to ensure the token - is correctly and atomically exchanged between processes. - - GET parameters:: - timeout(int): How long to wait for new events in milliseconds. - since(batch_token): Batch token when asking for incremental deltas. - - Response JSON:: - { - "next_batch": // batch token for the next /sync - "to_device": { - // list of to-device events - "events": [ - { - "content: { "algorithm": "m.olm.v1.curve25519-aes-sha2", "ciphertext": { ... }, "org.matrix.msgid": "abcd", "session_id": "abcd" }, - "type": "m.room.encrypted", - "sender": "@alice:example.com", - } - // ... - ] - }, - "device_lists": { - "changed": ["@alice:example.com"], - "left": ["@bob:example.com"] - }, - "device_one_time_keys_count": { - "signed_curve25519": 50 - }, - "device_unused_fallback_key_types": [ - "signed_curve25519" - ] - } - """ - - PATTERNS = client_patterns( - "/org.matrix.msc3575/sync/e2ee$", releases=[], v1=False, unstable=True - ) - - def __init__(self, hs: "HomeServer"): - super().__init__() - self.hs = hs - self.auth = hs.get_auth() - self.store = hs.get_datastores().main - self.sync_handler = hs.get_sync_handler() - - # Filtering only matters for the `device_lists` because it requires a bunch of - # derived information from rooms (see how `_generate_sync_entry_for_rooms()` - # prepares a bunch of data for `_generate_sync_entry_for_device_list()`). - self.only_member_events_filter_collection = FilterCollection( - self.hs, - { - "room": { - # We only care about membership events for the `device_lists`. - # Membership will tell us whether a user has joined/left a room and - # if there are new devices to encrypt for. - "timeline": { - "types": ["m.room.member"], - }, - "state": { - "types": ["m.room.member"], - }, - # We don't want any extra account_data generated because it's not - # returned by this endpoint. This helps us avoid work in - # `_generate_sync_entry_for_rooms()` - "account_data": { - "not_types": ["*"], - }, - # We don't want any extra ephemeral data generated because it's not - # returned by this endpoint. This helps us avoid work in - # `_generate_sync_entry_for_rooms()` - "ephemeral": { - "not_types": ["*"], - }, - }, - # We don't want any extra account_data generated because it's not - # returned by this endpoint. (This is just here for good measure) - "account_data": { - "not_types": ["*"], - }, - # We don't want any extra presence data generated because it's not - # returned by this endpoint. (This is just here for good measure) - "presence": { - "not_types": ["*"], - }, - }, - ) - - async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: - requester = await self.auth.get_user_by_req_experimental_feature( - request, allow_guest=True, feature=ExperimentalFeature.MSC3575 - ) - user = requester.user - device_id = requester.device_id - - timeout = parse_integer(request, "timeout", default=0) - since = parse_string(request, "since") - - sync_config = SyncConfig( - user=user, - filter_collection=self.only_member_events_filter_collection, - is_guest=requester.is_guest, - device_id=device_id, - use_state_after=False, # We don't return any rooms so this flag is a no-op - ) - - since_token = None - if since is not None: - since_token = await StreamToken.from_string(self.store, since) - - # Request cache key - request_key = ( - SyncVersion.E2EE_SYNC, - user, - timeout, - since, - ) - - # Gather data for the response - sync_result = await self.sync_handler.wait_for_sync_for_user( - requester, - sync_config, - SyncVersion.E2EE_SYNC, - request_key, - since_token=since_token, - timeout=timeout, - full_state=False, - ) - - # The client may have disconnected by now; don't bother to serialize the - # response if so. - if request._disconnected: - logger.info("Client has disconnected; not serializing response.") - return 200, {} - - response: JsonDict = defaultdict(dict) - response["next_batch"] = await sync_result.next_batch.to_string(self.store) - - if sync_result.to_device: - response["to_device"] = {"events": sync_result.to_device} - - if sync_result.device_lists.changed: - response["device_lists"]["changed"] = list(sync_result.device_lists.changed) - if sync_result.device_lists.left: - response["device_lists"]["left"] = list(sync_result.device_lists.left) - - # We always include this because https://github.com/vector-im/element-android/issues/3725 - # The spec isn't terribly clear on when this can be omitted and how a client would tell - # the difference between "no keys present" and "nothing changed" in terms of whole field - # absent / individual key type entry absent - # Corresponding synapse issue: https://github.com/matrix-org/synapse/issues/10456 - response["device_one_time_keys_count"] = sync_result.device_one_time_keys_count - - # https://github.com/matrix-org/matrix-doc/blob/54255851f642f84a4f1aaf7bc063eebe3d76752b/proposals/2732-olm-fallback-keys.md - # states that this field should always be included, as long as the server supports the feature. - response["device_unused_fallback_key_types"] = ( - sync_result.device_unused_fallback_key_types - ) - - return 200, response - - class SlidingSyncRestServlet(RestServlet): """ - API endpoint for MSC3575 Sliding Sync `/sync`. Allows for clients to request a + API endpoint for MSC4186 Simplified Sliding Sync `/sync`, which was historically derived + from MSC3575 (Sliding Sync; now abandoned). Allows for clients to request a subset (sliding window) of rooms, state, and timeline events (just what they need) in order to bootstrap quickly and subscribe to only what the client cares about. Because the client can specify what it cares about, we can respond quickly and skip all of the work we would normally have to do with a sync v2 response. + Extensions of various features are defined in: + - to-device messaging (MSC3885) + - end-to-end encryption (MSC3884) + - typing notifications (MSC3961) + - receipts (MSC3960) + - account data (MSC3959) + - thread subscriptions (MSC4308) + Request query parameters: timeout: How long to wait for new events in milliseconds. pos: Stream position token when asking for incremental deltas. @@ -970,12 +832,18 @@ class SlidingSyncRestServlet(RestServlet): extensions=body.extensions, ) - sliding_sync_results = await self.sliding_sync_handler.wait_for_sync_for_user( + ( + sliding_sync_results, + did_wait, + ) = await self.sliding_sync_handler.wait_for_sync_for_user( requester, sync_config, from_token, timeout, ) + # Knowing whether we waited is useful in traces to filter out long-running + # requests where we were just waiting. + set_tag("sliding_sync.did_wait", str(did_wait)) # The client may have disconnected by now; don't bother to serialize the # response if so. @@ -987,6 +855,7 @@ class SlidingSyncRestServlet(RestServlet): return 200, response_content + @trace_with_opname("sliding_sync.encode_response") async def encode_response( self, requester: Requester, @@ -1007,6 +876,7 @@ class SlidingSyncRestServlet(RestServlet): return response + @trace_with_opname("sliding_sync.encode_lists") def encode_lists( self, lists: Mapping[str, SlidingSyncResult.SlidingWindowList] ) -> JsonDict: @@ -1028,6 +898,7 @@ class SlidingSyncRestServlet(RestServlet): return serialized_lists + @trace_with_opname("sliding_sync.encode_rooms") async def encode_rooms( self, requester: Requester, @@ -1148,6 +1019,7 @@ class SlidingSyncRestServlet(RestServlet): return serialized_rooms + @trace_with_opname("sliding_sync.encode_extensions") async def encode_extensions( self, requester: Requester, extensions: SlidingSyncResult.Extensions ) -> JsonDict: @@ -1213,11 +1085,49 @@ class SlidingSyncRestServlet(RestServlet): "rooms": extensions.typing.room_id_to_typing_map, } + # excludes both None and falsy `thread_subscriptions` + if extensions.thread_subscriptions: + serialized_extensions["io.element.msc4308.thread_subscriptions"] = ( + _serialise_thread_subscriptions(extensions.thread_subscriptions) + ) + return serialized_extensions +def _serialise_thread_subscriptions( + thread_subscriptions: SlidingSyncResult.Extensions.ThreadSubscriptionsExtension, +) -> JsonDict: + out: JsonDict = {} + + if thread_subscriptions.subscribed: + out["subscribed"] = { + room_id: { + thread_root_id: attr.asdict( + change, filter=lambda _attr, v: v is not None + ) + for thread_root_id, change in room_threads.items() + } + for room_id, room_threads in thread_subscriptions.subscribed.items() + } + + if thread_subscriptions.unsubscribed: + out["unsubscribed"] = { + room_id: { + thread_root_id: attr.asdict( + change, filter=lambda _attr, v: v is not None + ) + for thread_root_id, change in room_threads.items() + } + for room_id, room_threads in thread_subscriptions.unsubscribed.items() + } + + if thread_subscriptions.prev_batch: + out["prev_batch"] = thread_subscriptions.prev_batch.to_string() + + return out + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: SyncRestServlet(hs).register(http_server) SlidingSyncRestServlet(hs).register(http_server) - SlidingSyncE2eeRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/tags.py b/synapse/rest/client/tags.py index b6648f3499..fb59efb11f 100644 --- a/synapse/rest/client/tags.py +++ b/synapse/rest/client/tags.py @@ -20,9 +20,10 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING, Tuple -from synapse.api.errors import AuthError +from synapse.api.errors import AuthError, Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -35,6 +36,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +MAX_TAG_LENGTH = 255 + class TagListServlet(RestServlet): """ @@ -86,6 +89,16 @@ class TagServlet(RestServlet): requester = await self.auth.get_user_by_req(request) if user_id != requester.user.to_string(): raise AuthError(403, "Cannot add tags for other users.") + + # check if the tag exceeds the length allowed by the matrix-specification + # as defined in: https://spec.matrix.org/v1.15/client-server-api/#events-14 + if len(tag.encode("utf-8")) > MAX_TAG_LENGTH: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "tag parameter's length is over 255 bytes", + errcode=Codes.INVALID_PARAM, + ) + # Check if the user has any membership in the room and raise error if not. # Although it's not harmful for users to tag random rooms, it's just superfluous # data we don't need to track or allow. diff --git a/synapse/rest/client/thread_subscriptions.py b/synapse/rest/client/thread_subscriptions.py new file mode 100644 index 0000000000..039aba1721 --- /dev/null +++ b/synapse/rest/client/thread_subscriptions.py @@ -0,0 +1,247 @@ +from http import HTTPStatus +from typing import TYPE_CHECKING, Dict, Optional, Tuple + +import attr +from typing_extensions import TypeAlias + +from synapse.api.errors import Codes, NotFoundError, SynapseError +from synapse.http.server import HttpServer +from synapse.http.servlet import ( + RestServlet, + parse_and_validate_json_object_from_request, + parse_integer, + parse_string, +) +from synapse.http.site import SynapseRequest +from synapse.rest.client._base import client_patterns +from synapse.types import ( + JsonDict, + RoomID, + SlidingSyncStreamToken, + ThreadSubscriptionsToken, +) +from synapse.types.handlers.sliding_sync import SlidingSyncResult +from synapse.types.rest import RequestBodyModel +from synapse.util.pydantic_models import AnyEventId + +if TYPE_CHECKING: + from synapse.server import HomeServer + +_ThreadSubscription: TypeAlias = ( + SlidingSyncResult.Extensions.ThreadSubscriptionsExtension.ThreadSubscription +) +_ThreadUnsubscription: TypeAlias = ( + SlidingSyncResult.Extensions.ThreadSubscriptionsExtension.ThreadUnsubscription +) + + +class ThreadSubscriptionsRestServlet(RestServlet): + PATTERNS = client_patterns( + "/io.element.msc4306/rooms/(?P[^/]*)/thread/(?P[^/]*)/subscription$", + unstable=True, + releases=(), + ) + CATEGORY = "Thread Subscriptions requests (unstable)" + + def __init__(self, hs: "HomeServer"): + self.auth = hs.get_auth() + self.is_mine = hs.is_mine + self.store = hs.get_datastores().main + self.handler = hs.get_thread_subscriptions_handler() + + class PutBody(RequestBodyModel): + automatic: Optional[AnyEventId] + """ + If supplied, the event ID of an event giving rise to this automatic subscription. + + If omitted, this subscription is a manual subscription. + """ + + async def on_GET( + self, request: SynapseRequest, room_id: str, thread_root_id: str + ) -> Tuple[int, JsonDict]: + RoomID.from_string(room_id) + if not thread_root_id.startswith("$"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid event ID", errcode=Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request) + + subscription = await self.handler.get_thread_subscription_settings( + requester.user, + room_id, + thread_root_id, + ) + + if subscription is None: + raise NotFoundError("Not subscribed.") + + return HTTPStatus.OK, {"automatic": subscription.automatic} + + async def on_PUT( + self, request: SynapseRequest, room_id: str, thread_root_id: str + ) -> Tuple[int, JsonDict]: + RoomID.from_string(room_id) + if not thread_root_id.startswith("$"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid event ID", errcode=Codes.INVALID_PARAM + ) + body = parse_and_validate_json_object_from_request(request, self.PutBody) + + requester = await self.auth.get_user_by_req(request) + + await self.handler.subscribe_user_to_thread( + requester.user, + room_id, + thread_root_id, + automatic_event_id=body.automatic, + ) + + return HTTPStatus.OK, {} + + async def on_DELETE( + self, request: SynapseRequest, room_id: str, thread_root_id: str + ) -> Tuple[int, JsonDict]: + RoomID.from_string(room_id) + if not thread_root_id.startswith("$"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid event ID", errcode=Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request) + + await self.handler.unsubscribe_user_from_thread( + requester.user, + room_id, + thread_root_id, + ) + + return HTTPStatus.OK, {} + + +class ThreadSubscriptionsPaginationRestServlet(RestServlet): + PATTERNS = client_patterns( + "/io.element.msc4308/thread_subscriptions$", + unstable=True, + releases=(), + ) + CATEGORY = "Thread Subscriptions requests (unstable)" + + # Maximum number of thread subscriptions to return in one request. + MAX_LIMIT = 512 + + def __init__(self, hs: "HomeServer"): + self.auth = hs.get_auth() + self.is_mine = hs.is_mine + self.store = hs.get_datastores().main + + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + + limit = min( + parse_integer(request, "limit", default=100, negative=False), + ThreadSubscriptionsPaginationRestServlet.MAX_LIMIT, + ) + from_end_opt = parse_string(request, "from", required=False) + to_start_opt = parse_string(request, "to", required=False) + _direction = parse_string(request, "dir", required=True, allowed_values=("b",)) + + if limit <= 0: + # condition needed because `negative=False` still allows 0 + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "limit must be greater than 0", + errcode=Codes.INVALID_PARAM, + ) + + if from_end_opt is not None: + try: + # because of backwards pagination, the `from` token is actually the + # bound closest to the end of the stream + end_stream_id = ThreadSubscriptionsToken.from_string( + from_end_opt + ).stream_id + except ValueError: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "`from` is not a valid token", + errcode=Codes.INVALID_PARAM, + ) + else: + end_stream_id = self.store.get_max_thread_subscriptions_stream_id() + + if to_start_opt is not None: + # because of backwards pagination, the `to` token is actually the + # bound closest to the start of the stream + try: + start_stream_id = ThreadSubscriptionsToken.from_string( + to_start_opt + ).stream_id + except ValueError: + # we also accept sliding sync `pos` tokens on this parameter + try: + sliding_sync_pos = await SlidingSyncStreamToken.from_string( + self.store, to_start_opt + ) + start_stream_id = ( + sliding_sync_pos.stream_token.thread_subscriptions_key + ) + except ValueError: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "`to` is not a valid token", + errcode=Codes.INVALID_PARAM, + ) + else: + # the start of time is ID 1; the lower bound is exclusive though + start_stream_id = 0 + + subscriptions = ( + await self.store.get_latest_updated_thread_subscriptions_for_user( + requester.user.to_string(), + from_id=start_stream_id, + to_id=end_stream_id, + limit=limit, + ) + ) + + subscribed_threads: Dict[str, Dict[str, JsonDict]] = {} + unsubscribed_threads: Dict[str, Dict[str, JsonDict]] = {} + for stream_id, room_id, thread_root_id, subscribed, automatic in subscriptions: + if subscribed: + subscribed_threads.setdefault(room_id, {})[thread_root_id] = ( + attr.asdict( + _ThreadSubscription( + automatic=automatic, + bump_stamp=stream_id, + ) + ) + ) + else: + unsubscribed_threads.setdefault(room_id, {})[thread_root_id] = ( + attr.asdict(_ThreadUnsubscription(bump_stamp=stream_id)) + ) + + result: JsonDict = {} + if subscribed_threads: + result["subscribed"] = subscribed_threads + if unsubscribed_threads: + result["unsubscribed"] = unsubscribed_threads + + if len(subscriptions) == limit: + # We hit the limit, so there might be more entries to return. + # Generate a new token that has moved backwards, ready for the next + # request. + min_returned_stream_id, _, _, _, _ = subscriptions[0] + result["end"] = ThreadSubscriptionsToken( + # We subtract one because the 'later in the stream' bound is inclusive, + # and we already saw the element at index 0. + stream_id=min_returned_stream_id - 1 + ).to_string() + + return HTTPStatus.OK, result + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + if hs.config.experimental.msc4306_enabled: + ThreadSubscriptionsRestServlet(hs).register(http_server) + ThreadSubscriptionsPaginationRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index f791904168..1a57996aec 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -94,9 +94,9 @@ class HttpTransactionCache: # (appservice and guest users), but does not cover access tokens minted # by the admin API. Use the access token ID instead. else: - assert ( - requester.access_token_id is not None - ), "Requester must have an access_token_id" + assert requester.access_token_id is not None, ( + "Requester must have an access_token_id" + ) return (path, "user_admin", requester.access_token_id) def fetch_or_execute_request( diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index ba1141bbe5..1b8efd98cd 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -112,6 +112,7 @@ class VersionsRestServlet(RestServlet): "v1.9", "v1.10", "v1.11", + "v1.12", ], # as per MSC1497: "unstable_features": { @@ -170,10 +171,15 @@ class VersionsRestServlet(RestServlet): ), # MSC4140: Delayed events "org.matrix.msc4140": bool(self.config.server.max_event_delay_ms), - # MSC4151: Report room API (Client-Server API) - "org.matrix.msc4151": self.config.experimental.msc4151_enabled, # Simplified sliding sync "org.matrix.simplified_msc3575": msc3575_enabled, + # Arbitrary key-value profile fields. + "uk.tcpip.msc4133": self.config.experimental.msc4133_enabled, + "uk.tcpip.msc4133.stable": True, + # MSC4155: Invite filtering + "org.matrix.msc4155": self.config.experimental.msc4155_enabled, + # MSC4306: Support for thread subscriptions + "org.matrix.msc4306": self.config.experimental.msc4306_enabled, }, }, ) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index ff5dc51a01..3961f82894 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -81,7 +81,7 @@ class ConsentResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self.hs = hs self.store = hs.get_datastores().main diff --git a/synapse/rest/media/config_resource.py b/synapse/rest/media/config_resource.py index 80462d65d3..b014e91bdb 100644 --- a/synapse/rest/media/config_resource.py +++ b/synapse/rest/media/config_resource.py @@ -40,7 +40,14 @@ class MediaConfigResource(RestServlet): self.clock = hs.get_clock() self.auth = hs.get_auth() self.limits_dict = {"m.upload.size": config.media.max_upload_size} + self.media_repository_callbacks = hs.get_module_api_callbacks().media_repository async def on_GET(self, request: SynapseRequest) -> None: - await self.auth.get_user_by_req(request) - respond_with_json(request, 200, self.limits_dict, send_cors=True) + requester = await self.auth.get_user_by_req(request) + user_specific_config = ( + await self.media_repository_callbacks.get_media_config_for_user( + requester.user.to_string() + ) + ) + response = user_specific_config if user_specific_config else self.limits_dict + respond_with_json(request, 200, response, send_cors=True) diff --git a/synapse/rest/media/upload_resource.py b/synapse/rest/media/upload_resource.py index 359d006f04..74d8280582 100644 --- a/synapse/rest/media/upload_resource.py +++ b/synapse/rest/media/upload_resource.py @@ -50,9 +50,12 @@ class BaseUploadServlet(RestServlet): self.server_name = hs.hostname self.auth = hs.get_auth() self.max_upload_size = hs.config.media.max_upload_size + self._media_repository_callbacks = ( + hs.get_module_api_callbacks().media_repository + ) - def _get_file_metadata( - self, request: SynapseRequest + async def _get_file_metadata( + self, request: SynapseRequest, user_id: str ) -> Tuple[int, Optional[str], str]: raw_content_length = request.getHeader("Content-Length") if raw_content_length is None: @@ -67,7 +70,14 @@ class BaseUploadServlet(RestServlet): code=413, errcode=Codes.TOO_LARGE, ) - + if not await self._media_repository_callbacks.is_user_allowed_to_upload_media_of_size( + user_id, content_length + ): + raise SynapseError( + msg="Upload request body is too large", + code=413, + errcode=Codes.TOO_LARGE, + ) args: Dict[bytes, List[bytes]] = request.args # type: ignore upload_name_bytes = parse_bytes_from_args(args, "filename") if upload_name_bytes: @@ -104,11 +114,13 @@ class UploadServlet(BaseUploadServlet): async def on_POST(self, request: SynapseRequest) -> None: requester = await self.auth.get_user_by_req(request) - content_length, upload_name, media_type = self._get_file_metadata(request) + content_length, upload_name, media_type = await self._get_file_metadata( + request, requester.user.to_string() + ) try: content: IO = request.content # type: ignore - content_uri = await self.media_repo.create_content( + content_uri = await self.media_repo.create_or_update_content( media_type, upload_name, content, content_length, requester.user ) except SpamMediaException: @@ -152,17 +164,19 @@ class AsyncUploadServlet(BaseUploadServlet): async with lock: await self.media_repo.verify_can_upload(media_id, requester.user) - content_length, upload_name, media_type = self._get_file_metadata(request) + content_length, upload_name, media_type = await self._get_file_metadata( + request, requester.user.to_string() + ) try: content: IO = request.content # type: ignore - await self.media_repo.update_content( - media_id, + await self.media_repo.create_or_update_content( media_type, upload_name, content, content_length, requester.user, + media_id=media_id, ) except SpamMediaException: # For uploading of media we want to respond with a 400, instead of diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py index 7b5bfc0421..665ce77dd7 100644 --- a/synapse/rest/synapse/client/__init__.py +++ b/synapse/rest/synapse/client/__init__.py @@ -30,6 +30,7 @@ from synapse.rest.synapse.client.pick_username import pick_username_resource from synapse.rest.synapse.client.rendezvous import MSC4108RendezvousSessionResource from synapse.rest.synapse.client.sso_register import SsoRegisterResource from synapse.rest.synapse.client.unsubscribe import UnsubscribeResource +from synapse.rest.synapse.mas import MasResource if TYPE_CHECKING: from synapse.server import HomeServer @@ -55,11 +56,13 @@ def build_synapse_client_resource_tree(hs: "HomeServer") -> Mapping[str, Resourc "/_synapse/client/unsubscribe": UnsubscribeResource(hs), } - # Expose the JWKS endpoint if OAuth2 delegation is enabled - if hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: + resources["/_synapse/mas"] = MasResource(hs) + elif hs.config.experimental.msc3861.enabled: from synapse.rest.synapse.client.jwks import JwksResource resources["/_synapse/jwks"] = JwksResource(hs) + resources["/_synapse/mas"] = MasResource(hs) # provider-specific SSO bits. Only load these if they are enabled, since they # rely on optional dependencies. diff --git a/synapse/rest/synapse/client/federation_whitelist.py b/synapse/rest/synapse/client/federation_whitelist.py index 2b8f0320e0..f59daf8428 100644 --- a/synapse/rest/synapse/client/federation_whitelist.py +++ b/synapse/rest/synapse/client/federation_whitelist.py @@ -44,7 +44,7 @@ class FederationWhitelistResource(DirectServeJsonResource): PATH = "/_synapse/client/v1/config/federation_whitelist" def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._federation_whitelist = hs.config.federation.federation_domain_whitelist diff --git a/synapse/rest/synapse/client/jwks.py b/synapse/rest/synapse/client/jwks.py index 5f581d3445..e9a7c24e3b 100644 --- a/synapse/rest/synapse/client/jwks.py +++ b/synapse/rest/synapse/client/jwks.py @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) class JwksResource(DirectServeJsonResource): def __init__(self, hs: "HomeServer"): - super().__init__(extract_context=True) + super().__init__(clock=hs.get_clock(), extract_context=True) # Parameters that are allowed to be exposed in the public key. # This is done manually, because authlib's private to public key conversion diff --git a/synapse/rest/synapse/client/new_user_consent.py b/synapse/rest/synapse/client/new_user_consent.py index 8b00b8c012..c7bd8de482 100644 --- a/synapse/rest/synapse/client/new_user_consent.py +++ b/synapse/rest/synapse/client/new_user_consent.py @@ -48,7 +48,7 @@ class NewUserConsentResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() self._server_name = hs.hostname self._consent_version = hs.config.consent.user_consent_version diff --git a/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py b/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py index 3f4cf16934..114c8c2d82 100644 --- a/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py +++ b/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py @@ -35,7 +35,7 @@ class OIDCBackchannelLogoutResource(DirectServeJsonResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._oidc_handler = hs.get_oidc_handler() async def _async_render_POST(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/synapse/client/oidc/callback_resource.py b/synapse/rest/synapse/client/oidc/callback_resource.py index 84077684d4..cc0213c50c 100644 --- a/synapse/rest/synapse/client/oidc/callback_resource.py +++ b/synapse/rest/synapse/client/oidc/callback_resource.py @@ -35,7 +35,7 @@ class OIDCCallbackResource(DirectServeHtmlResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._oidc_handler = hs.get_oidc_handler() async def _async_render_GET(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/synapse/client/password_reset.py b/synapse/rest/synapse/client/password_reset.py index 29e4b2d07a..377578ef8a 100644 --- a/synapse/rest/synapse/client/password_reset.py +++ b/synapse/rest/synapse/client/password_reset.py @@ -47,7 +47,7 @@ class PasswordResetSubmitTokenResource(DirectServeHtmlResource): Args: hs: server """ - super().__init__() + super().__init__(clock=hs.get_clock()) self.clock = hs.get_clock() self.store = hs.get_datastores().main diff --git a/synapse/rest/synapse/client/pick_idp.py b/synapse/rest/synapse/client/pick_idp.py index f26929bd60..15c1b3ab49 100644 --- a/synapse/rest/synapse/client/pick_idp.py +++ b/synapse/rest/synapse/client/pick_idp.py @@ -21,6 +21,7 @@ import logging from typing import TYPE_CHECKING +from synapse.api.urls import LoginSSORedirectURIBuilder from synapse.http.server import ( DirectServeHtmlResource, finish_request, @@ -43,12 +44,14 @@ class PickIdpResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() self._sso_login_idp_picker_template = ( hs.config.sso.sso_login_idp_picker_template ) self._server_name = hs.hostname + self._public_baseurl = hs.config.server.public_baseurl + self._login_sso_redirect_url_builder = LoginSSORedirectURIBuilder(hs.config) async def _async_render_GET(self, request: SynapseRequest) -> None: client_redirect_url = parse_string( @@ -56,11 +59,17 @@ class PickIdpResource(DirectServeHtmlResource): ) idp = parse_string(request, "idp", required=False) - # if we need to pick an IdP, do so + # If we need to pick an IdP, do so if not idp: return await self._serve_id_picker(request, client_redirect_url) - # otherwise, redirect to the IdP's redirect URI + # Validate the `idp` query parameter. We should only be working with known IdPs. + # No need waste further effort if we don't know about it. + # + # Although, we primarily prevent open redirect attacks by URL encoding all of + # the parameters we use in the redirect URL below, this validation also helps + # prevent Synapse from crafting arbitrary URLs and being used in open redirect + # attacks (defense in depth). providers = self._sso_handler.get_identity_providers() auth_provider = providers.get(idp) if not auth_provider: @@ -70,11 +79,19 @@ class PickIdpResource(DirectServeHtmlResource): ) return - sso_url = await auth_provider.handle_redirect_request( - request, client_redirect_url.encode("utf8") + # Otherwise, redirect to the login SSO redirect endpoint for the given IdP + # (which will in turn take us to the the IdP's redirect URI). + # + # We could go directly to the IdP's redirect URI, but this way we ensure that + # the user goes through the same logic as normal flow. Additionally, if a proxy + # needs to intercept the request, it only needs to intercept the one endpoint. + sso_login_redirect_url = ( + self._login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id=idp, client_redirect_url=client_redirect_url + ) ) - logger.info("Redirecting to %s", sso_url) - request.redirect(sso_url) + logger.info("Redirecting to %s", sso_login_redirect_url) + request.redirect(sso_login_redirect_url) finish_request(request) async def _serve_id_picker( diff --git a/synapse/rest/synapse/client/pick_username.py b/synapse/rest/synapse/client/pick_username.py index 7d16b796d4..1727bb63b7 100644 --- a/synapse/rest/synapse/client/pick_username.py +++ b/synapse/rest/synapse/client/pick_username.py @@ -62,7 +62,7 @@ def pick_username_resource(hs: "HomeServer") -> Resource: class AvailabilityCheckResource(DirectServeJsonResource): def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() async def _async_render_GET(self, request: Request) -> Tuple[int, JsonDict]: @@ -78,7 +78,7 @@ class AvailabilityCheckResource(DirectServeJsonResource): class AccountDetailsResource(DirectServeHtmlResource): def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() def template_search_dirs() -> Generator[str, None, None]: diff --git a/synapse/rest/synapse/client/rendezvous.py b/synapse/rest/synapse/client/rendezvous.py index 5216d30d1f..5278c35572 100644 --- a/synapse/rest/synapse/client/rendezvous.py +++ b/synapse/rest/synapse/client/rendezvous.py @@ -30,7 +30,7 @@ class MSC4108RendezvousSessionResource(DirectServeJsonResource): isLeaf = True def __init__(self, hs: "HomeServer") -> None: - super().__init__() + super().__init__(clock=hs.get_clock()) self._handler = hs.get_rendezvous_handler() async def _async_render_GET(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/synapse/client/saml2/response_resource.py b/synapse/rest/synapse/client/saml2/response_resource.py index 7b8667e04c..d2cf4f21f1 100644 --- a/synapse/rest/synapse/client/saml2/response_resource.py +++ b/synapse/rest/synapse/client/saml2/response_resource.py @@ -35,7 +35,7 @@ class SAML2ResponseResource(DirectServeHtmlResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._saml_handler = hs.get_saml_handler() self._sso_handler = hs.get_sso_handler() diff --git a/synapse/rest/synapse/client/sso_register.py b/synapse/rest/synapse/client/sso_register.py index be562ad8dc..74d9bdee87 100644 --- a/synapse/rest/synapse/client/sso_register.py +++ b/synapse/rest/synapse/client/sso_register.py @@ -43,7 +43,7 @@ class SsoRegisterResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() async def _async_render_GET(self, request: Request) -> None: diff --git a/synapse/rest/synapse/client/unsubscribe.py b/synapse/rest/synapse/client/unsubscribe.py index 6d4bd9f2ed..90837611dd 100644 --- a/synapse/rest/synapse/client/unsubscribe.py +++ b/synapse/rest/synapse/client/unsubscribe.py @@ -38,7 +38,7 @@ class UnsubscribeResource(DirectServeHtmlResource): SUCCESS_HTML = b"You have been unsubscribed" def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self.notifier = hs.get_notifier() self.auth = hs.get_auth() self.pusher_pool = hs.get_pusherpool() diff --git a/synapse/rest/synapse/mas/__init__.py b/synapse/rest/synapse/mas/__init__.py new file mode 100644 index 0000000000..8115c563d2 --- /dev/null +++ b/synapse/rest/synapse/mas/__init__.py @@ -0,0 +1,71 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + + +import logging +from typing import TYPE_CHECKING + +from twisted.web.resource import Resource + +from synapse.rest.synapse.mas.devices import ( + MasDeleteDeviceResource, + MasSyncDevicesResource, + MasUpdateDeviceDisplayNameResource, + MasUpsertDeviceResource, +) +from synapse.rest.synapse.mas.users import ( + MasAllowCrossSigningResetResource, + MasDeleteUserResource, + MasIsLocalpartAvailableResource, + MasProvisionUserResource, + MasQueryUserResource, + MasReactivateUserResource, + MasSetDisplayNameResource, + MasUnsetDisplayNameResource, +) + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +logger = logging.getLogger(__name__) + + +class MasResource(Resource): + """ + Provides endpoints for MAS to manage user accounts and devices. + + All endpoints are mounted under the path `/_synapse/mas/` and only work + using the MAS admin token. + """ + + def __init__(self, hs: "HomeServer"): + Resource.__init__(self) + self.putChild(b"query_user", MasQueryUserResource(hs)) + self.putChild(b"provision_user", MasProvisionUserResource(hs)) + self.putChild(b"is_localpart_available", MasIsLocalpartAvailableResource(hs)) + self.putChild(b"delete_user", MasDeleteUserResource(hs)) + self.putChild(b"upsert_device", MasUpsertDeviceResource(hs)) + self.putChild(b"delete_device", MasDeleteDeviceResource(hs)) + self.putChild( + b"update_device_display_name", MasUpdateDeviceDisplayNameResource(hs) + ) + self.putChild(b"sync_devices", MasSyncDevicesResource(hs)) + self.putChild(b"reactivate_user", MasReactivateUserResource(hs)) + self.putChild(b"set_displayname", MasSetDisplayNameResource(hs)) + self.putChild(b"unset_displayname", MasUnsetDisplayNameResource(hs)) + self.putChild( + b"allow_cross_signing_reset", MasAllowCrossSigningResetResource(hs) + ) diff --git a/synapse/rest/synapse/mas/_base.py b/synapse/rest/synapse/mas/_base.py new file mode 100644 index 0000000000..7346198b75 --- /dev/null +++ b/synapse/rest/synapse/mas/_base.py @@ -0,0 +1,55 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + + +from typing import TYPE_CHECKING, cast + +from synapse.api.auth.mas import MasDelegatedAuth +from synapse.api.errors import SynapseError +from synapse.http.server import DirectServeJsonResource + +if TYPE_CHECKING: + from synapse.app.generic_worker import GenericWorkerStore + from synapse.http.site import SynapseRequest + from synapse.server import HomeServer + + +class MasBaseResource(DirectServeJsonResource): + def __init__(self, hs: "HomeServer"): + auth = hs.get_auth() + if hs.config.mas.enabled: + assert isinstance(auth, MasDelegatedAuth) + + self._is_request_from_mas = auth.is_request_using_the_shared_secret + else: + # Importing this module requires authlib, which is an optional + # dependency but required if msc3861 is enabled + from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth + + assert isinstance(auth, MSC3861DelegatedAuth) + + self._is_request_from_mas = auth.is_request_using_the_admin_token + + DirectServeJsonResource.__init__(self, extract_context=True) + self.store = cast("GenericWorkerStore", hs.get_datastores().main) + self.hostname = hs.hostname + + def assert_request_is_from_mas(self, request: "SynapseRequest") -> None: + """Assert that the request is coming from MAS itself, not a regular user. + + Throws a 403 if the request is not coming from MAS. + """ + if not self._is_request_from_mas(request): + raise SynapseError(403, "This endpoint must only be called by MAS") diff --git a/synapse/rest/synapse/mas/devices.py b/synapse/rest/synapse/mas/devices.py new file mode 100644 index 0000000000..6cc1153590 --- /dev/null +++ b/synapse/rest/synapse/mas/devices.py @@ -0,0 +1,238 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING, Optional, Tuple + +from synapse._pydantic_compat import StrictStr +from synapse.api.errors import NotFoundError +from synapse.http.servlet import parse_and_validate_json_object_from_request +from synapse.types import JsonDict, UserID +from synapse.types.rest import RequestBodyModel + +if TYPE_CHECKING: + from synapse.http.site import SynapseRequest + from synapse.server import HomeServer + + +from ._base import MasBaseResource + +logger = logging.getLogger(__name__) + + +class MasUpsertDeviceResource(MasBaseResource): + """ + Endpoint for MAS to create or update user devices. + + Takes a localpart, device ID, and optional display name to create new devices + or update existing ones. + + POST /_synapse/mas/upsert_device + {"localpart": "alice", "device_id": "DEVICE123", "display_name": "Alice's Phone"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + device_id: StrictStr + display_name: Optional[StrictStr] + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + inserted = await self.device_handler.upsert_device( + user_id=str(user_id), + device_id=body.device_id, + display_name=body.display_name, + ) + + return HTTPStatus.CREATED if inserted else HTTPStatus.OK, {} + + +class MasDeleteDeviceResource(MasBaseResource): + """ + Endpoint for MAS to delete user devices. + + Takes a localpart and device ID to remove the specified device from the user's account. + + POST /_synapse/mas/delete_device + {"localpart": "alice", "device_id": "DEVICE123"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + device_id: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.device_handler.delete_devices( + user_id=str(user_id), + device_ids=[body.device_id], + ) + + return HTTPStatus.NO_CONTENT, {} + + +class MasUpdateDeviceDisplayNameResource(MasBaseResource): + """ + Endpoint for MAS to update a device's display name. + + Takes a localpart, device ID, and new display name to update the device's name. + + POST /_synapse/mas/update_device_display_name + {"localpart": "alice", "device_id": "DEVICE123", "display_name": "Alice's New Phone"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + device_id: StrictStr + display_name: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.device_handler.update_device( + user_id=str(user_id), + device_id=body.device_id, + content={"display_name": body.display_name}, + ) + + return HTTPStatus.OK, {} + + +class MasSyncDevicesResource(MasBaseResource): + """ + Endpoint for MAS to synchronize a user's complete device list. + + Takes a localpart and a set of device IDs to ensure the user's device list + matches the provided set by adding missing devices and removing extra ones. + + POST /_synapse/mas/sync_devices + {"localpart": "alice", "devices": ["DEVICE123", "DEVICE456"]} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + devices: set[StrictStr] + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + current_devices = await self.store.get_devices_by_user(user_id=str(user_id)) + current_devices_list = set(current_devices.keys()) + target_device_list = set(body.devices) + + to_add = target_device_list - current_devices_list + to_delete = current_devices_list - target_device_list + + # Log what we're about to do to make it easier to debug if it stops + # mid-way, as this can be a long operation if there are a lot of devices + # to delete or to add. + if to_add and to_delete: + logger.info( + "Syncing %d devices for user %s will add %d devices and delete %d devices", + len(target_device_list), + user_id, + len(to_add), + len(to_delete), + ) + elif to_add: + logger.info( + "Syncing %d devices for user %s will add %d devices", + len(target_device_list), + user_id, + len(to_add), + ) + elif to_delete: + logger.info( + "Syncing %d devices for user %s will delete %d devices", + len(target_device_list), + user_id, + len(to_delete), + ) + + if to_delete: + await self.device_handler.delete_devices( + user_id=str(user_id), device_ids=to_delete + ) + + for device_id in to_add: + await self.device_handler.upsert_device( + user_id=str(user_id), + device_id=device_id, + ) + + return 200, {} diff --git a/synapse/rest/synapse/mas/users.py b/synapse/rest/synapse/mas/users.py new file mode 100644 index 0000000000..09aa13bebb --- /dev/null +++ b/synapse/rest/synapse/mas/users.py @@ -0,0 +1,467 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, Optional, Tuple, TypedDict + +from synapse._pydantic_compat import StrictBool, StrictStr, root_validator +from synapse.api.errors import NotFoundError, SynapseError +from synapse.http.servlet import ( + parse_and_validate_json_object_from_request, + parse_string, +) +from synapse.types import JsonDict, UserID, UserInfo, create_requester +from synapse.types.rest import RequestBodyModel + +if TYPE_CHECKING: + from synapse.http.site import SynapseRequest + from synapse.server import HomeServer + + +from ._base import MasBaseResource + +logger = logging.getLogger(__name__) + + +class MasQueryUserResource(MasBaseResource): + """ + Endpoint for MAS to query user information by localpart. + + Takes a localpart parameter and returns user profile data including display name, + avatar URL, and account status (suspended/deactivated). + + GET /_synapse/mas/query_user?localpart=alice + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + class Response(TypedDict): + user_id: str + display_name: Optional[str] + avatar_url: Optional[str] + is_suspended: bool + is_deactivated: bool + + async def _async_render_GET( + self, request: "SynapseRequest" + ) -> Tuple[int, Response]: + self.assert_request_is_from_mas(request) + + localpart = parse_string(request, "localpart", required=True) + user_id = UserID(localpart, self.hostname) + + user: Optional[UserInfo] = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + profile = await self.store.get_profileinfo(user_id=user_id) + + return HTTPStatus.OK, self.Response( + user_id=user_id.to_string(), + display_name=profile.display_name, + avatar_url=profile.avatar_url, + is_suspended=user.suspended, + is_deactivated=user.is_deactivated, + ) + + +class MasProvisionUserResource(MasBaseResource): + """ + Endpoint for MAS to create or update user accounts and their profile data. + + Takes a localpart and optional profile fields (display name, avatar URL, email addresses). + Can create new users or update existing ones by setting or unsetting profile fields. + + POST /_synapse/mas/provision_user + {"localpart": "alice", "set_displayname": "Alice", "set_emails": ["alice@example.com"]} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + self.registration_handler = hs.get_registration_handler() + self.identity_handler = hs.get_identity_handler() + self.auth_handler = hs.get_auth_handler() + self.profile_handler = hs.get_profile_handler() + self.clock = hs.get_clock() + self.auth = hs.get_auth() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + unset_displayname: StrictBool = False + set_displayname: Optional[StrictStr] = None + + unset_avatar_url: StrictBool = False + set_avatar_url: Optional[StrictStr] = None + + unset_emails: StrictBool = False + set_emails: Optional[list[StrictStr]] = None + + @root_validator(pre=True) + def validate_exclusive(cls, values: Any) -> Any: + if "unset_displayname" in values and "set_displayname" in values: + raise ValueError( + "Cannot specify both unset_displayname and set_displayname" + ) + if "unset_avatar_url" in values and "set_avatar_url" in values: + raise ValueError( + "Cannot specify both unset_avatar_url and set_avatar_url" + ) + if "unset_emails" in values and "set_emails" in values: + raise ValueError("Cannot specify both unset_emails and set_emails") + + return values + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + + localpart = body.localpart + user_id = UserID(localpart, self.hostname) + + requester = create_requester(user_id=user_id) + existing_user = await self.store.get_user_by_id(user_id=str(user_id)) + if existing_user is None: + created = True + await self.registration_handler.register_user( + localpart=localpart, + default_display_name=body.set_displayname, + bind_emails=body.set_emails, + by_admin=True, + ) + else: + created = False + if body.unset_displayname: + await self.profile_handler.set_displayname( + target_user=user_id, + requester=requester, + new_displayname="", + by_admin=True, + ) + elif body.set_displayname is not None: + await self.profile_handler.set_displayname( + target_user=user_id, + requester=requester, + new_displayname=body.set_displayname, + by_admin=True, + ) + + new_email_list: Optional[set[str]] = None + if body.unset_emails: + new_email_list = set() + elif body.set_emails is not None: + new_email_list = set(body.set_emails) + + if new_email_list is not None: + medium = "email" + current_threepid_list = await self.store.user_get_threepids( + user_id=user_id.to_string() + ) + current_email_list = { + t.address for t in current_threepid_list if t.medium == medium + } + + to_delete = current_email_list - new_email_list + to_add = new_email_list - current_email_list + + for address in to_delete: + await self.identity_handler.try_unbind_threepid( + mxid=user_id.to_string(), + medium=medium, + address=address, + id_server=None, + ) + + await self.auth_handler.delete_local_threepid( + user_id=user_id.to_string(), + medium=medium, + address=address, + ) + + current_time = self.clock.time_msec() + for address in to_add: + await self.auth_handler.add_threepid( + user_id=user_id.to_string(), + medium=medium, + address=address, + validated_at=current_time, + ) + + if body.unset_avatar_url: + await self.profile_handler.set_avatar_url( + target_user=user_id, + requester=requester, + new_avatar_url="", + by_admin=True, + ) + elif body.set_avatar_url is not None: + await self.profile_handler.set_avatar_url( + target_user=user_id, + requester=requester, + new_avatar_url=body.set_avatar_url, + by_admin=True, + ) + + return HTTPStatus.CREATED if created else HTTPStatus.OK, {} + + +class MasIsLocalpartAvailableResource(MasBaseResource): + """ + Endpoint for MAS to check if a localpart is available for user registration. + + Takes a localpart parameter and validates its format and availability, + checking for conflicts with existing users or application service namespaces. + + GET /_synapse/mas/is_localpart_available?localpart=alice + """ + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.registration_handler = hs.get_registration_handler() + + async def _async_render_GET( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + localpart = parse_string(request, "localpart") + if localpart is None: + raise SynapseError(400, "Missing localpart") + + await self.registration_handler.check_username(localpart) + + return HTTPStatus.OK, {} + + +class MasDeleteUserResource(MasBaseResource): + """ + Endpoint for MAS to delete/deactivate user accounts. + + Takes a localpart and an erase flag to determine whether to deactivate + the account and optionally erase user data for compliance purposes. + + POST /_synapse/mas/delete_user + {"localpart": "alice", "erase": true} + """ + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.deactivate_account_handler = hs.get_deactivate_account_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + erase: StrictBool + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.deactivate_account_handler.deactivate_account( + user_id=user_id.to_string(), + erase_data=body.erase, + requester=create_requester(user_id=user_id), + ) + + return HTTPStatus.OK, {} + + +class MasReactivateUserResource(MasBaseResource): + """ + Endpoint for MAS to reactivate previously deactivated user accounts. + + Takes a localpart parameter to restore access to deactivated accounts. + + POST /_synapse/mas/reactivate_user + {"localpart": "alice"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.deactivate_account_handler = hs.get_deactivate_account_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.deactivate_account_handler.activate_account(user_id=str(user_id)) + + return HTTPStatus.OK, {} + + +class MasSetDisplayNameResource(MasBaseResource): + """ + Endpoint for MAS to set a user's display name. + + Takes a localpart and display name to update the user's profile. + + POST /_synapse/mas/set_displayname + {"localpart": "alice", "displayname": "Alice"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.profile_handler = hs.get_profile_handler() + self.auth_handler = hs.get_auth_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + displayname: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + requester = create_requester(user_id=user_id) + + await self.profile_handler.set_displayname( + target_user=requester.user, + requester=requester, + new_displayname=body.displayname, + by_admin=True, + ) + + return HTTPStatus.OK, {} + + +class MasUnsetDisplayNameResource(MasBaseResource): + """ + Endpoint for MAS to clear a user's display name. + + Takes a localpart parameter to remove the display name for the specified user. + + POST /_synapse/mas/unset_displayname + {"localpart": "alice"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.profile_handler = hs.get_profile_handler() + self.auth_handler = hs.get_auth_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + requester = create_requester(user_id=user_id) + + await self.profile_handler.set_displayname( + target_user=requester.user, + requester=requester, + new_displayname="", + by_admin=True, + ) + + return HTTPStatus.OK, {} + + +class MasAllowCrossSigningResetResource(MasBaseResource): + """ + Endpoint for MAS to allow cross-signing key reset without user interaction. + + Takes a localpart parameter to temporarily allow cross-signing key replacement + without requiring User-Interactive Authentication (UIA). + + POST /_synapse/mas/allow_cross_signing_reset + {"localpart": "alice"} + """ + + REPLACEMENT_PERIOD_MS = 10 * 60 * 1000 # 10 minutes + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.auth_handler = hs.get_auth_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + timestamp = ( + await self.store.allow_master_cross_signing_key_replacement_without_uia( + user_id=str(user_id), + duration_ms=self.REPLACEMENT_PERIOD_MS, + ) + ) + + if timestamp is None: + # If there are no cross-signing keys, this is a no-op, but we should log + logger.warning( + "User %s has no master cross-signing key", user_id.to_string() + ) + + return HTTPStatus.OK, {} diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py index d336d60c93..e4fe4c45ef 100644 --- a/synapse/rest/well_known.py +++ b/synapse/rest/well_known.py @@ -18,11 +18,12 @@ # # import logging -from typing import TYPE_CHECKING, Optional, Tuple, cast +from typing import TYPE_CHECKING, Optional, Tuple from twisted.web.resource import Resource from twisted.web.server import Request +from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.errors import NotFoundError from synapse.http.server import DirectServeJsonResource from synapse.http.site import SynapseRequest @@ -52,18 +53,25 @@ class WellKnownBuilder: "base_url": self._config.registration.default_identity_server } - # We use the MSC3861 values as they are used by multiple MSCs - if self._config.experimental.msc3861.enabled: + if self._config.mas.enabled: + assert isinstance(self._auth, MasDelegatedAuth) + + result["org.matrix.msc2965.authentication"] = { + "issuer": await self._auth.issuer(), + "account": await self._auth.account_management_url(), + } + + elif self._config.experimental.msc3861.enabled: # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth # We import lazily here because of the authlib requirement from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - auth = cast(MSC3861DelegatedAuth, self._auth) + assert isinstance(self._auth, MSC3861DelegatedAuth) result["org.matrix.msc2965.authentication"] = { - "issuer": await auth.issuer(), + "issuer": await self._auth.issuer(), } - account_management_url = await auth.account_management_url() + account_management_url = await self._auth.account_management_url() if account_management_url is not None: result["org.matrix.msc2965.authentication"]["account"] = ( account_management_url @@ -86,7 +94,7 @@ class ClientWellKnownResource(DirectServeJsonResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(clock=hs.get_clock()) self._well_known_builder = WellKnownBuilder(hs) async def _async_render_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: diff --git a/synapse/server.py b/synapse/server.py index c7b4918813..3fb29a7817 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -40,6 +40,7 @@ from twisted.web.resource import Resource from synapse.api.auth import Auth from synapse.api.auth.internal import InternalAuth +from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.auth_blocking import AuthBlocking from synapse.api.filtering import Filtering from synapse.api.ratelimiting import Ratelimiter, RequestRatelimiter @@ -69,7 +70,7 @@ from synapse.handlers.auth import AuthHandler, PasswordAuthProvider from synapse.handlers.cas import CasHandler from synapse.handlers.deactivate_account import DeactivateAccountHandler from synapse.handlers.delayed_events import DelayedEventsHandler -from synapse.handlers.device import DeviceHandler, DeviceWorkerHandler +from synapse.handlers.device import DeviceHandler, DeviceWriterHandler from synapse.handlers.devicemessage import DeviceMessageHandler from synapse.handlers.directory import DirectoryHandler from synapse.handlers.e2e_keys import E2eKeysHandler @@ -94,6 +95,7 @@ from synapse.handlers.read_marker import ReadMarkerHandler from synapse.handlers.receipts import ReceiptsHandler from synapse.handlers.register import RegistrationHandler from synapse.handlers.relations import RelationsHandler +from synapse.handlers.reports import ReportsHandler from synapse.handlers.room import ( RoomContextHandler, RoomCreationHandler, @@ -107,6 +109,7 @@ from synapse.handlers.room_member import ( RoomMemberMasterHandler, ) from synapse.handlers.room_member_worker import RoomMemberWorkerHandler +from synapse.handlers.room_policy import RoomPolicyHandler from synapse.handlers.room_summary import RoomSummaryHandler from synapse.handlers.search import SearchHandler from synapse.handlers.send_email import SendEmailHandler @@ -115,6 +118,7 @@ from synapse.handlers.sliding_sync import SlidingSyncHandler from synapse.handlers.sso import SsoHandler from synapse.handlers.stats import StatsHandler from synapse.handlers.sync import SyncHandler +from synapse.handlers.thread_subscriptions import ThreadSubscriptionsHandler from synapse.handlers.typing import FollowerTypingHandler, TypingWriterHandler from synapse.handlers.user_directory import UserDirectoryHandler from synapse.handlers.worker_lock import WorkerLocksHandler @@ -125,7 +129,10 @@ from synapse.http.client import ( ) from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.media.media_repository import MediaRepository -from synapse.metrics import register_threadpool +from synapse.metrics import ( + all_later_gauges_to_clean_up_on_shutdown, + register_threadpool, +) from synapse.metrics.common_usage_metrics import CommonUsageMetricsManager from synapse.module_api import ModuleApi from synapse.module_api.callbacks import ModuleApiCallbacks @@ -254,6 +261,7 @@ class HomeServer(metaclass=abc.ABCMeta): "auth", "deactivate_account", "delayed_events", + "e2e_keys", # for the `delete_old_otks` scheduled-task handler "message", "pagination", "profile", @@ -358,11 +366,36 @@ class HomeServer(metaclass=abc.ABCMeta): self.datastores = Databases(self.DATASTORE_CLASS, self) logger.info("Finished setting up.") - # Register background tasks required by this server. This must be done - # somewhat manually due to the background tasks not being registered - # unless handlers are instantiated. - if self.config.worker.run_background_tasks: - self.setup_background_tasks() + def __del__(self) -> None: + """ + Called when an the homeserver is garbage collected. + + Make sure we actually do some clean-up, rather than leak data. + """ + self.cleanup() + + def cleanup(self) -> None: + """ + WIP: Clean-up any references to the homeserver and stop any running related + processes, timers, loops, replication stream, etc. + + This should be called wherever you care about the HomeServer being completely + garbage collected like in tests. It's not necessary to call if you plan to just + shut down the whole Python process anyway. + + Can be called multiple times. + """ + logger.info("Received cleanup request for %s.", self.hostname) + + # TODO: Stop background processes, timers, loops, replication stream, etc. + + # Cleanup metrics associated with the homeserver + for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values(): + later_gauge.unregister_hooks_for_homeserver_instance_id( + self.get_instance_id() + ) + + logger.info("Cleanup complete for %s.", self.hostname) def start_listening(self) -> None: # noqa: B027 (no-op by design) """Start the HTTP, manhole, metrics, etc listeners @@ -371,7 +404,7 @@ class HomeServer(metaclass=abc.ABCMeta): appropriate listeners. """ - def setup_background_tasks(self) -> None: + def start_background_tasks(self) -> None: """ Some handlers have side effects on instantiation (like registering background updates). This function causes them to be fetched, and @@ -390,7 +423,7 @@ class HomeServer(metaclass=abc.ABCMeta): def is_mine(self, domain_specific_string: DomainSpecificString) -> bool: return domain_specific_string.domain == self.hostname - def is_mine_id(self, string: str) -> bool: + def is_mine_id(self, user_id: str) -> bool: """Determines whether a user ID or room alias originates from this homeserver. Returns: @@ -398,7 +431,7 @@ class HomeServer(metaclass=abc.ABCMeta): homeserver. `False` otherwise, or if the user ID or room alias is malformed. """ - localpart_hostname = string.split(":", 1) + localpart_hostname = user_id.split(":", 1) if len(localpart_hostname) < 2: return False return localpart_hostname[1] == self.hostname @@ -419,7 +452,7 @@ class HomeServer(metaclass=abc.ABCMeta): @cache_in_self def get_distributor(self) -> Distributor: - return Distributor() + return Distributor(server_name=self.hostname) @cache_in_self def get_registration_ratelimiter(self) -> Ratelimiter: @@ -447,6 +480,8 @@ class HomeServer(metaclass=abc.ABCMeta): @cache_in_self def get_auth(self) -> Auth: + if self.config.mas.enabled: + return MasDelegatedAuth(self) if self.config.experimental.msc3861.enabled: from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth @@ -583,11 +618,11 @@ class HomeServer(metaclass=abc.ABCMeta): ) @cache_in_self - def get_device_handler(self) -> DeviceWorkerHandler: - if self.config.worker.worker_app: - return DeviceWorkerHandler(self) - else: - return DeviceHandler(self) + def get_device_handler(self) -> DeviceHandler: + if self.get_instance_name() in self.config.worker.writers.device_lists: + return DeviceWriterHandler(self) + + return DeviceHandler(self) @cache_in_self def get_device_message_handler(self) -> DeviceMessageHandler: @@ -716,6 +751,10 @@ class HomeServer(metaclass=abc.ABCMeta): def get_receipts_handler(self) -> ReceiptsHandler: return ReceiptsHandler(self) + @cache_in_self + def get_reports_handler(self) -> ReportsHandler: + return ReportsHandler(self) + @cache_in_self def get_read_marker_handler(self) -> ReadMarkerHandler: return ReadMarkerHandler(self) @@ -782,6 +821,10 @@ class HomeServer(metaclass=abc.ABCMeta): def get_timestamp_lookup_handler(self) -> TimestampLookupHandler: return TimestampLookupHandler(self) + @cache_in_self + def get_thread_subscriptions_handler(self) -> ThreadSubscriptionsHandler: + return ThreadSubscriptionsHandler(self) + @cache_in_self def get_registration_handler(self) -> RegistrationHandler: return RegistrationHandler(self) @@ -806,6 +849,10 @@ class HomeServer(metaclass=abc.ABCMeta): return OidcHandler(self) + @cache_in_self + def get_room_policy_handler(self) -> RoomPolicyHandler: + return RoomPolicyHandler(self) + @cache_in_self def get_event_client_serializer(self) -> EventClientSerializer: return EventClientSerializer(self) @@ -833,7 +880,8 @@ class HomeServer(metaclass=abc.ABCMeta): @cache_in_self def get_federation_ratelimiter(self) -> FederationRateLimiter: return FederationRateLimiter( - self.get_clock(), + our_server_name=self.hostname, + clock=self.get_clock(), config=self.config.ratelimiting.rc_federation, metrics_name="federation_servlets", ) @@ -964,7 +1012,10 @@ class HomeServer(metaclass=abc.ABCMeta): ) # Register the threadpool with our metrics. - register_threadpool("media", media_threadpool) + server_name = self.hostname + register_threadpool( + name="media", server_name=server_name, threadpool=media_threadpool + ) return media_threadpool diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 001a290e87..19f86b5a56 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -35,6 +35,7 @@ SERVER_NOTICE_ROOM_TAG = "m.server_notice" class ServerNoticesManager: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @cached self._store = hs.get_datastores().main self._config = hs.config self._account_data_handler = hs.get_account_data_handler() @@ -44,7 +45,6 @@ class ServerNoticesManager: self._message_handler = hs.get_message_handler() self._storage_controllers = hs.get_storage_controllers() self._is_mine_id = hs.is_mine_id - self._server_name = hs.hostname self._notifier = hs.get_notifier() self.server_notices_mxid = self._config.servernotices.server_notices_mxid @@ -77,7 +77,7 @@ class ServerNoticesManager: assert self.server_notices_mxid is not None requester = create_requester( - self.server_notices_mxid, authenticated_entity=self._server_name + self.server_notices_mxid, authenticated_entity=self.server_name ) logger.info("Sending server notice to %s", user_id) @@ -151,7 +151,7 @@ class ServerNoticesManager: assert self._is_mine_id(user_id), "Cannot send server notices to remote users" requester = create_requester( - self.server_notices_mxid, authenticated_entity=self._server_name + self.server_notices_mxid, authenticated_entity=self.server_name ) room_id = await self.maybe_get_notice_room_for_user(user_id) @@ -256,7 +256,7 @@ class ServerNoticesManager: """ assert self.server_notices_mxid is not None requester = create_requester( - self.server_notices_mxid, authenticated_entity=self._server_name + self.server_notices_mxid, authenticated_entity=self.server_name ) # Check whether the user has already joined or been invited to this room. If @@ -279,7 +279,7 @@ class ServerNoticesManager: if self._config.servernotices.server_notices_auto_join: user_requester = create_requester( - user_id, authenticated_entity=self._server_name + user_id, authenticated_entity=self.server_name ) await self._room_member_handler.update_membership( requester=user_requester, diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 72b291889b..3d8016c264 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -51,19 +51,23 @@ from synapse.events.snapshot import ( ) from synapse.logging.context import ContextResourceUsage from synapse.logging.opentracing import tag_args, trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.replication.http.state import ReplicationUpdateCurrentStateRestServlet from synapse.state import v1, v2 +from synapse.storage.databases.main.event_federation import StateDifference from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import StateMap, StrCollection from synapse.types.state import StateFilter from synapse.util.async_helpers import Linearizer from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.metrics import Measure, measure_func +from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer from synapse.storage.controllers import StateStorageController from synapse.storage.databases.main import DataStore + from synapse.storage.databases.state.deletion import StateDeletionDataStore logger = logging.getLogger(__name__) metrics_logger = logging.getLogger("synapse.state.metrics") @@ -72,6 +76,7 @@ metrics_logger = logging.getLogger("synapse.state.metrics") state_groups_histogram = Histogram( "synapse_state_number_state_groups_in_resolution", "Number of state groups used when performing a state resolution", + labelnames=[SERVER_NAME_LABEL], buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), ) @@ -81,6 +86,7 @@ EVICTION_TIMEOUT_SECONDS = 60 * 60 _NEXT_STATE_ID = 1 +CREATE_KEY = (EventTypes.Create, "") POWER_KEY = (EventTypes.PowerLevels, "") @@ -186,7 +192,8 @@ class StateHandler: """ def __init__(self, hs: "HomeServer"): - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.store = hs.get_datastores().main self._state_storage_controller = hs.get_storage_controllers().state self.hs = hs @@ -194,6 +201,8 @@ class StateHandler: self._storage_controllers = hs.get_storage_controllers() self._events_shard_config = hs.config.worker.events_shard_config self._instance_name = hs.get_instance_name() + self._state_store = hs.get_datastores().state + self._state_deletion_store = hs.get_datastores().state_deletion self._update_current_state_client = ( ReplicationUpdateCurrentStateRestServlet.make_client(hs) @@ -355,6 +364,28 @@ class StateHandler: await_full_state=False, ) + # Ensure we still have the state groups we're relying on, and bump + # their usage time to avoid them being deleted from under us. + if entry.state_group: + missing_state_group = await self._state_deletion_store.check_state_groups_and_bump_deletion( + {entry.state_group} + ) + if missing_state_group: + raise Exception(f"Missing state group: {entry.state_group}") + elif entry.prev_group: + # We only rely on the prev group when persisting the event if we + # don't have an `entry.state_group`. + missing_state_group = await self._state_deletion_store.check_state_groups_and_bump_deletion( + {entry.prev_group} + ) + + if missing_state_group: + # If we're missing the prev group then we can just clear the + # entries, and rely on `entry._state` (which must exist if + # `entry.state_group` is None) + entry.prev_group = None + entry.delta_ids = None + state_group_before_event_prev_group = entry.prev_group deltas_to_state_group_before_event = entry.delta_ids state_ids_before_event = None @@ -475,7 +506,10 @@ class StateHandler: @trace @measure_func() async def resolve_state_groups_for_events( - self, room_id: str, event_ids: StrCollection, await_full_state: bool = True + self, + room_id: str, + event_ids: StrCollection, + await_full_state: bool = True, ) -> _StateCacheEntry: """Given a list of event_ids this method fetches the state at each event, resolves conflicts between them and returns them. @@ -511,6 +545,7 @@ class StateHandler: ) = await self._state_storage_controller.get_state_group_delta( state_group_id ) + return _StateCacheEntry( state=None, state_group=state_group_id, @@ -531,7 +566,9 @@ class StateHandler: room_version, state_to_resolve, None, - state_res_store=StateResolutionStore(self.store), + state_res_store=StateResolutionStore( + self.store, self._state_deletion_store + ), ) return result @@ -573,20 +610,24 @@ _biggest_room_by_cpu_counter = Counter( "synapse_state_res_cpu_for_biggest_room_seconds", "CPU time spent performing state resolution for the single most expensive " "room for state resolution", + labelnames=[SERVER_NAME_LABEL], ) _biggest_room_by_db_counter = Counter( "synapse_state_res_db_for_biggest_room_seconds", "Database time spent performing state resolution for the single most " "expensive room for state resolution", + labelnames=[SERVER_NAME_LABEL], ) _cpu_times = Histogram( "synapse_state_res_cpu_for_all_rooms_seconds", "CPU time (utime+stime) spent computing a single state resolution", + labelnames=[SERVER_NAME_LABEL], ) _db_times = Histogram( "synapse_state_res_db_for_all_rooms_seconds", "Database time spent computing a single state resolution", + labelnames=[SERVER_NAME_LABEL], ) @@ -598,6 +639,7 @@ class StateResolutionHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.clock = hs.get_clock() self.resolve_linearizer = Linearizer(name="state_resolve_lock") @@ -606,6 +648,7 @@ class StateResolutionHandler: self._state_cache: ExpiringCache[FrozenSet[int], _StateCacheEntry] = ( ExpiringCache( cache_name="state_cache", + server_name=self.server_name, clock=self.clock, max_len=100000, expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000, @@ -663,7 +706,25 @@ class StateResolutionHandler: async with self.resolve_linearizer.queue(group_names): cache = self._state_cache.get(group_names, None) if cache: - return cache + # Check that the returned cache entry doesn't point to deleted + # state groups. + state_groups_to_check = set() + if cache.state_group is not None: + state_groups_to_check.add(cache.state_group) + + if cache.prev_group is not None: + state_groups_to_check.add(cache.prev_group) + + missing_state_groups = await state_res_store.state_deletion_store.check_state_groups_and_bump_deletion( + state_groups_to_check + ) + + if not missing_state_groups: + return cache + else: + # There are missing state groups, so let's remove the stale + # entry and continue as if it was a cache miss. + self._state_cache.pop(group_names, None) logger.info( "Resolving state for %s with groups %s", @@ -671,7 +732,19 @@ class StateResolutionHandler: list(group_names), ) - state_groups_histogram.observe(len(state_groups_ids)) + # We double check that none of the state groups have been deleted. + # They shouldn't be as all these state groups should be referenced. + missing_state_groups = await state_res_store.state_deletion_store.check_state_groups_and_bump_deletion( + group_names + ) + if missing_state_groups: + raise Exception( + f"State groups have been deleted: {shortstr(missing_state_groups)}" + ) + + state_groups_histogram.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).observe(len(state_groups_ids)) new_state = await self.resolve_events_with_store( room_id, @@ -686,7 +759,9 @@ class StateResolutionHandler: # which will be used as a cache key for future resolutions, but # not get persisted. - with Measure(self.clock, "state.create_group_ids"): + with Measure( + self.clock, name="state.create_group_ids", server_name=self.server_name + ): cache = _make_state_cache_entry(new_state, state_groups_ids) self._state_cache[group_names] = cache @@ -724,7 +799,9 @@ class StateResolutionHandler: a map from (type, state_key) to event_id. """ try: - with Measure(self.clock, "state._resolve_events") as m: + with Measure( + self.clock, name="state._resolve_events", server_name=self.server_name + ) as m: room_version_obj = KNOWN_ROOM_VERSIONS[room_version] if room_version_obj.state_res == StateResolutionVersions.V1: return await v1.resolve_events_with_store( @@ -754,8 +831,12 @@ class StateResolutionHandler: room_metrics.db_time += rusage.db_txn_duration_sec room_metrics.db_events += rusage.evt_db_fetch_count - _cpu_times.observe(rusage.ru_utime + rusage.ru_stime) - _db_times.observe(rusage.db_txn_duration_sec) + _cpu_times.labels(**{SERVER_NAME_LABEL: self.server_name}).observe( + rusage.ru_utime + rusage.ru_stime + ) + _db_times.labels(**{SERVER_NAME_LABEL: self.server_name}).observe( + rusage.db_txn_duration_sec + ) def _report_metrics(self) -> None: if not self._state_res_metrics: @@ -812,7 +893,9 @@ class StateResolutionHandler: # report info on the single biggest to prometheus _, biggest_metrics = biggest[0] - prometheus_counter_metric.inc(extract_key(biggest_metrics)) + prometheus_counter_metric.labels(**{SERVER_NAME_LABEL: self.server_name}).inc( + extract_key(biggest_metrics) + ) def _make_state_cache_entry( @@ -884,7 +967,8 @@ class StateResolutionStore: in well defined way. """ - store: "DataStore" + main_store: "DataStore" + state_deletion_store: "StateDeletionDataStore" def get_events( self, event_ids: StrCollection, allow_rejected: bool = False @@ -899,7 +983,7 @@ class StateResolutionStore: An awaitable which resolves to a dict from event_id to event. """ - return self.store.get_events( + return self.main_store.get_events( event_ids, redact_behaviour=EventRedactBehaviour.as_is, get_prev_content=False, @@ -907,17 +991,35 @@ class StateResolutionStore: ) def get_auth_chain_difference( - self, room_id: str, state_sets: List[Set[str]] - ) -> Awaitable[Set[str]]: - """Given sets of state events figure out the auth chain difference (as + self, + room_id: str, + state_sets: List[Set[str]], + conflicted_state: Optional[Set[str]], + additional_backwards_reachable_conflicted_events: Optional[Set[str]], + ) -> Awaitable[StateDifference]: + """ "Given sets of state events figure out the auth chain difference (as per state res v2 algorithm). - This equivalent to fetching the full auth chain for each set of state + This is equivalent to fetching the full auth chain for each set of state and returning the events that don't appear in each and every auth chain. + If conflicted_state is not None, calculate and return the conflicted sub-graph as per + state res v2.1. The event IDs in the conflicted state MUST be a subset of the event IDs in + state_sets. + + If additional_backwards_reachable_conflicted_events is set, the provided events are included + when calculating the conflicted subgraph. This is primarily useful for calculating the + subgraph across a combination of persisted and unpersisted events. + Returns: - An awaitable that resolves to a set of event IDs. + information on the auth chain difference, and also the conflicted subgraph if + conflicted_state is not None """ - return self.store.get_auth_chain_difference(room_id, state_sets) + return self.main_store.get_auth_chain_difference_extended( + room_id, + state_sets, + conflicted_state, + additional_backwards_reachable_conflicted_events, + ) diff --git a/synapse/state/v2.py b/synapse/state/v2.py index da926ad146..8bf6706434 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -29,20 +29,21 @@ from typing import ( Generator, Iterable, List, + Literal, Optional, + Protocol, Sequence, Set, Tuple, overload, ) -from typing_extensions import Literal, Protocol - from synapse import event_auth -from synapse.api.constants import EventTypes +from synapse.api.constants import CREATOR_POWER_LEVEL, EventTypes from synapse.api.errors import AuthError -from synapse.api.room_versions import RoomVersion -from synapse.events import EventBase +from synapse.api.room_versions import RoomVersion, StateResolutionVersions +from synapse.events import EventBase, is_creator +from synapse.storage.databases.main.event_federation import StateDifference from synapse.types import MutableStateMap, StateMap, StrCollection logger = logging.getLogger(__name__) @@ -52,7 +53,7 @@ class Clock(Protocol): # This is usually synapse.util.Clock, but it's replaced with a FakeClock in tests. # We only ever sleep(0) though, so that other async functions can make forward # progress without waiting for stateres to complete. - def sleep(self, duration_ms: float) -> Awaitable[None]: ... + async def sleep(self, duration_ms: float) -> None: ... class StateResolutionStore(Protocol): @@ -63,8 +64,12 @@ class StateResolutionStore(Protocol): ) -> Awaitable[Dict[str, EventBase]]: ... def get_auth_chain_difference( - self, room_id: str, state_sets: List[Set[str]] - ) -> Awaitable[Set[str]]: ... + self, + room_id: str, + state_sets: List[Set[str]], + conflicted_state: Optional[Set[str]], + additional_backwards_reachable_conflicted_events: Optional[set[str]], + ) -> Awaitable[StateDifference]: ... # We want to await to the reactor occasionally during state res when dealing @@ -123,12 +128,17 @@ async def resolve_events_with_store( logger.debug("%d conflicted state entries", len(conflicted_state)) logger.debug("Calculating auth chain difference") - # Also fetch all auth events that appear in only some of the state sets' - # auth chains. + conflicted_set: Optional[Set[str]] = None + if room_version.state_res == StateResolutionVersions.V2_1: + # calculate the conflicted subgraph + conflicted_set = set(itertools.chain.from_iterable(conflicted_state.values())) auth_diff = await _get_auth_chain_difference( - room_id, state_sets, event_map, state_res_store + room_id, + state_sets, + event_map, + state_res_store, + conflicted_set, ) - full_conflicted_set = set( itertools.chain( itertools.chain.from_iterable(conflicted_state.values()), auth_diff @@ -168,15 +178,26 @@ async def resolve_events_with_store( logger.debug("sorted %d power events", len(sorted_power_events)) + # v2.1 starts iterative auth checks from the empty set and not the unconflicted state. + # It relies on IAC behaviour which populates the base state with the events from auth_events + # if the state tuple is missing from the base state. This ensures the base state is only + # populated from auth_events rather than whatever the unconflicted state is (which could be + # completely bogus). + base_state = ( + {} + if room_version.state_res == StateResolutionVersions.V2_1 + else unconflicted_state + ) + # Now sequentially auth each one resolved_state = await _iterative_auth_checks( clock, room_id, room_version, - sorted_power_events, - unconflicted_state, - event_map, - state_res_store, + event_ids=sorted_power_events, + base_state=base_state, + event_map=event_map, + state_res_store=state_res_store, ) logger.debug("resolved power events") @@ -239,13 +260,23 @@ async def _get_power_level_for_sender( event = await _get_event(room_id, event_id, event_map, state_res_store) pl = None + create = None for aid in event.auth_event_ids(): aev = await _get_event( room_id, aid, event_map, state_res_store, allow_none=True ) if aev and (aev.type, aev.state_key) == (EventTypes.PowerLevels, ""): pl = aev - break + if aev and (aev.type, aev.state_key) == (EventTypes.Create, ""): + create = aev + + if event.type != EventTypes.Create: + # we should always have a create event + assert create is not None + + if create and create.room_version.msc4289_creator_power_enabled: + if is_creator(create, event.sender): + return CREATOR_POWER_LEVEL if pl is None: # Couldn't find power level. Check if they're the creator of the room @@ -254,7 +285,19 @@ async def _get_power_level_for_sender( room_id, aid, event_map, state_res_store, allow_none=True ) if aev and (aev.type, aev.state_key) == (EventTypes.Create, ""): - if aev.content.get("creator") == event.sender: + creator = ( + aev.sender + if event.room_version.implicit_room_creator + else aev.content.get("creator") + ) + if not creator: + logger.warning( + "_get_power_level_for_sender: event %s has no PL in auth_events and " + "creator is missing from create event %s", + event_id, + aev.event_id, + ) + if creator == event.sender: return 100 break return 0 @@ -274,6 +317,7 @@ async def _get_auth_chain_difference( state_sets: Sequence[StateMap[str]], unpersisted_events: Dict[str, EventBase], state_res_store: StateResolutionStore, + conflicted_state: Optional[Set[str]], ) -> Set[str]: """Compare the auth chains of each state set and return the set of events that only appear in some, but not all of the auth chains. @@ -282,11 +326,18 @@ async def _get_auth_chain_difference( state_sets: The input state sets we are trying to resolve across. unpersisted_events: A map from event ID to EventBase containing all unpersisted events involved in this resolution. - state_res_store: + state_res_store: A way to retrieve events and extract graph information on the auth chains. + conflicted_state: which event IDs are conflicted. Used in v2.1 for calculating the conflicted + subgraph. Returns: - The auth difference of the given state sets, as a set of event IDs. + The auth difference of the given state sets, as a set of event IDs. Also includes the + conflicted subgraph if `conflicted_state` is set. """ + is_state_res_v21 = conflicted_state is not None + num_conflicted_state = ( + len(conflicted_state) if conflicted_state is not None else None + ) # The `StateResolutionStore.get_auth_chain_difference` function assumes that # all events passed to it (and their auth chains) have been persisted @@ -306,14 +357,19 @@ async def _get_auth_chain_difference( # the event's auth chain with the events in `unpersisted_events` *plus* their # auth event IDs. events_to_auth_chain: Dict[str, Set[str]] = {} + # remember the forward links when doing the graph traversal, we'll need it for v2.1 checks + # This is a map from an event to the set of events that contain it as an auth event. + event_to_next_event: Dict[str, Set[str]] = {} for event in unpersisted_events.values(): chain = {event.event_id} events_to_auth_chain[event.event_id] = chain to_search = [event] while to_search: - for auth_id in to_search.pop().auth_event_ids(): + next_event = to_search.pop() + for auth_id in next_event.auth_event_ids(): chain.add(auth_id) + event_to_next_event.setdefault(auth_id, set()).add(next_event.event_id) auth_event = unpersisted_events.get(auth_id) if auth_event: to_search.append(auth_event) @@ -323,6 +379,8 @@ async def _get_auth_chain_difference( # # Note: If there are no `unpersisted_events` (which is the common case), we can do a # much simpler calculation. + additional_backwards_reachable_conflicted_events: Set[str] = set() + unpersisted_conflicted_events: Set[str] = set() if unpersisted_events: # The list of state sets to pass to the store, where each state set is a set # of the event ids making up the state. This is similar to `state_sets`, @@ -360,7 +418,16 @@ async def _get_auth_chain_difference( ) else: set_ids.add(event_id) - + if conflicted_state: + for conflicted_event_id in conflicted_state: + # presence in this map means it is unpersisted. + event_chain = events_to_auth_chain.get(conflicted_event_id) + if event_chain is not None: + unpersisted_conflicted_events.add(conflicted_event_id) + # tell the DB layer that we have some unpersisted conflicted events + additional_backwards_reachable_conflicted_events.update( + e for e in event_chain if e not in unpersisted_events + ) # The auth chain difference of the unpersisted events of the state sets # is calculated by taking the difference between the union and # intersections. @@ -372,12 +439,89 @@ async def _get_auth_chain_difference( auth_difference_unpersisted_part = () state_sets_ids = [set(state_set.values()) for state_set in state_sets] - difference = await state_res_store.get_auth_chain_difference( - room_id, state_sets_ids - ) - difference.update(auth_difference_unpersisted_part) + if conflicted_state: + # to ensure that conflicted state is a subset of state set IDs, we need to remove UNPERSISTED + # conflicted state set ids as we removed them above. + conflicted_state = conflicted_state - unpersisted_conflicted_events - return difference + difference = await state_res_store.get_auth_chain_difference( + room_id, + state_sets_ids, + conflicted_state, + additional_backwards_reachable_conflicted_events, + ) + difference.auth_difference.update(auth_difference_unpersisted_part) + + # if we're doing v2.1 we may need to add or expand the conflicted subgraph + if ( + is_state_res_v21 + and difference.conflicted_subgraph is not None + and unpersisted_events + ): + # we always include the conflicted events themselves in the subgraph. + if conflicted_state: + difference.conflicted_subgraph.update(conflicted_state) + # we may need to expand the subgraph in the case where the subgraph starts in the DB and + # ends in unpersisted events. To do this, we first need to see where the subgraph got up to, + # which we can do by finding the intersection between the additional backwards reachable + # conflicted events and the conflicted subgraph. Events in both sets mean A) some unpersisted + # conflicted event could backwards reach it and B) some persisted conflicted event could forward + # reach it. + subgraph_frontier = difference.conflicted_subgraph.intersection( + additional_backwards_reachable_conflicted_events + ) + # we can now combine the 2 scenarios: + # - subgraph starts in DB and ends in unpersisted + # - subgraph starts in unpersisted and ends in unpersisted + # by expanding the frontier into unpersisted events. + # The frontier is currently all persisted events. We want to expand this into unpersisted + # events. Mark every forwards reachable event from the frontier in the forwards_conflicted_set + # but NOT the backwards conflicted set. This mirrors what the DB layer does but in reverse: + # we supplied events which are backwards reachable to the DB and now the DB is providing + # forwards reachable events from the DB. + forwards_conflicted_set: Set[str] = set() + # we include unpersisted conflicted events here to process exclusive unpersisted subgraphs + search_queue = subgraph_frontier.union(unpersisted_conflicted_events) + while search_queue: + frontier_event = search_queue.pop() + next_event_ids = event_to_next_event.get(frontier_event, set()) + search_queue.update(next_event_ids) + forwards_conflicted_set.add(frontier_event) + + # we've already calculated the backwards form as this is the auth chain for each + # unpersisted conflicted event. + backwards_conflicted_set: Set[str] = set() + for uce in unpersisted_conflicted_events: + backwards_conflicted_set.update(events_to_auth_chain.get(uce, [])) + + # the unpersisted conflicted subgraph is the intersection of the backwards/forwards sets + conflicted_subgraph_unpersisted_part = backwards_conflicted_set.intersection( + forwards_conflicted_set + ) + # print(f"event_to_next_event={event_to_next_event}") + # print(f"unpersisted_conflicted_events={unpersisted_conflicted_events}") + # print(f"unperssited backwards_conflicted_set={backwards_conflicted_set}") + # print(f"unperssited forwards_conflicted_set={forwards_conflicted_set}") + difference.conflicted_subgraph.update(conflicted_subgraph_unpersisted_part) + + if difference.conflicted_subgraph: + old_events = difference.auth_difference.union( + conflicted_state if conflicted_state else set() + ) + additional_events = difference.conflicted_subgraph.difference(old_events) + + logger.debug( + "v2.1 %s additional events replayed=%d num_conflicts=%d conflicted_subgraph=%d auth_difference=%d", + room_id, + len(additional_events), + num_conflicted_state, + len(difference.conflicted_subgraph), + len(difference.auth_difference), + ) + # State res v2.1 includes the conflicted subgraph in the difference + return difference.auth_difference.union(difference.conflicted_subgraph) + + return difference.auth_difference def _seperate( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e14d711c76..d55c9e18ed 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -55,6 +55,7 @@ class SQLBaseStore(metaclass=ABCMeta): hs: "HomeServer", ): self.hs = hs + self.server_name = hs.hostname # nb must be called this for @cached self._clock = hs.get_clock() self.database_engine = database.engine self.db_pool = database @@ -86,7 +87,9 @@ class SQLBaseStore(metaclass=ABCMeta): """ def _invalidate_state_caches( - self, room_id: str, members_changed: Collection[str] + self, + room_id: str, + members_changed: Collection[str], ) -> None: """Invalidates caches that are based on the current state, but does not stream invalidations down replication. @@ -128,7 +131,7 @@ class SQLBaseStore(metaclass=ABCMeta): "_get_rooms_for_local_user_where_membership_is_inner", (user_id,) ) self._attempt_to_invalidate_cache( - "get_sliding_sync_rooms_for_user", (user_id,) + "get_sliding_sync_rooms_for_user_from_membership_snapshots", (user_id,) ) # Purge other caches based on room state. @@ -136,7 +139,9 @@ class SQLBaseStore(metaclass=ABCMeta): self._attempt_to_invalidate_cache("get_partial_current_state_ids", (room_id,)) self._attempt_to_invalidate_cache("get_room_type", (room_id,)) self._attempt_to_invalidate_cache("get_room_encryption", (room_id,)) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) def _invalidate_state_caches_all(self, room_id: str) -> None: """Invalidates caches that are based on the current state, but does @@ -166,7 +171,9 @@ class SQLBaseStore(metaclass=ABCMeta): self._attempt_to_invalidate_cache("get_room_summary", (room_id,)) self._attempt_to_invalidate_cache("get_room_type", (room_id,)) self._attempt_to_invalidate_cache("get_room_encryption", (room_id,)) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) def _attempt_to_invalidate_cache( self, cache_name: str, key: Optional[Collection[Any]] @@ -234,5 +241,5 @@ def db_to_json(db_content: Union[memoryview, bytes, bytearray, str]) -> Any: try: return json_decoder.decode(db_content) except Exception: - logging.warning("Tried to decode '%r' as JSON and failed", db_content) + logger.warning("Tried to decode '%r' as JSON and failed", db_content) raise diff --git a/synapse/storage/admin_client_config.py b/synapse/storage/admin_client_config.py new file mode 100644 index 0000000000..07acddc660 --- /dev/null +++ b/synapse/storage/admin_client_config.py @@ -0,0 +1,26 @@ +import logging +from typing import Optional + +from synapse.types import JsonMapping + +logger = logging.getLogger(__name__) + + +class AdminClientConfig: + """Class to track various Synapse-specific admin-only client-impacting config options.""" + + def __init__(self, account_data: Optional[JsonMapping]): + # Allow soft-failed events to be returned down `/sync` and other + # client APIs. `io.element.synapse.soft_failed: true` is added to the + # `unsigned` portion of the event to inform clients that the event + # is soft-failed. + self.return_soft_failed_events: bool = False + self.return_policy_server_spammy_events: bool = False + + if account_data: + self.return_soft_failed_events = account_data.get( + "return_soft_failed_events", False + ) + self.return_policy_server_spammy_events = account_data.get( + "return_policy_server_spammy_events", self.return_soft_failed_events + ) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 34139f580d..acc0abee63 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -249,6 +249,7 @@ class BackgroundUpdater: self._clock = hs.get_clock() self.db_pool = database self.hs = hs + self.server_name = hs.hostname self._database_name = database.name() @@ -395,7 +396,10 @@ class BackgroundUpdater: self._all_done = False sleep = self.sleep_enabled run_as_background_process( - "background_updates", self.run_background_updates, sleep + "background_updates", + self.server_name, + self.run_background_updates, + sleep, ) async def run_background_updates(self, sleep: bool) -> None: @@ -739,9 +743,9 @@ class BackgroundUpdater: c.execute(sql) async def updater(progress: JsonDict, batch_size: int) -> int: - assert isinstance( - self.db_pool.engine, engines.PostgresEngine - ), "validate constraint background update registered for non-Postres database" + assert isinstance(self.db_pool.engine, engines.PostgresEngine), ( + "validate constraint background update registered for non-Postres database" + ) logger.info("Validating constraint %s to %s", constraint_name, table) await self.db_pool.runWithConnection(runner) @@ -789,7 +793,7 @@ class BackgroundUpdater: # we may already have a half-built index. Let's just drop it # before trying to create it again. - sql = "DROP INDEX IF EXISTS %s" % (index_name,) + sql = "DROP INDEX CONCURRENTLY IF EXISTS %s" % (index_name,) logger.debug("[SQL] %s", sql) c.execute(sql) @@ -814,7 +818,7 @@ class BackgroundUpdater: if replaces_index is not None: # We drop the old index as the new index has now been created. - sql = f"DROP INDEX IF EXISTS {replaces_index}" + sql = f"DROP INDEX CONCURRENTLY IF EXISTS {replaces_index}" logger.debug("[SQL] %s", sql) c.execute(sql) finally: @@ -900,9 +904,9 @@ class BackgroundUpdater: on the table. Used to iterate over the table. """ - assert isinstance( - self.db_pool.engine, engines.PostgresEngine - ), "validate constraint background update registered for non-Postres database" + assert isinstance(self.db_pool.engine, engines.PostgresEngine), ( + "validate constraint background update registered for non-Postres database" + ) async def updater(progress: JsonDict, batch_size: int) -> int: return await self.validate_constraint_and_delete_in_background( diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 879ee9039e..120934af57 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -51,7 +51,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.events import EventBase -from synapse.events.snapshot import EventContext +from synapse.events.snapshot import EventContext, EventPersistencePair from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable from synapse.logging.opentracing import ( @@ -61,6 +61,7 @@ from synapse.logging.opentracing import ( start_active_span_follows_from, trace, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.controllers.state import StateStorageController from synapse.storage.databases import Databases @@ -82,25 +83,30 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) # The number of times we are recalculating the current state -state_delta_counter = Counter("synapse_storage_events_state_delta", "") +state_delta_counter = Counter( + "synapse_storage_events_state_delta", "", labelnames=[SERVER_NAME_LABEL] +) # The number of times we are recalculating state when there is only a # single forward extremity state_delta_single_event_counter = Counter( - "synapse_storage_events_state_delta_single_event", "" + "synapse_storage_events_state_delta_single_event", + "", + labelnames=[SERVER_NAME_LABEL], ) # The number of times we are reculating state when we could have resonably # calculated the delta when we calculated the state for an event we were # persisting. state_delta_reuse_delta_counter = Counter( - "synapse_storage_events_state_delta_reuse_delta", "" + "synapse_storage_events_state_delta_reuse_delta", "", labelnames=[SERVER_NAME_LABEL] ) # The number of forward extremities for each new event. forward_extremities_counter = Histogram( "synapse_storage_events_forward_extremities_persisted", "Number of forward extremities for each new event", + labelnames=[SERVER_NAME_LABEL], buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), ) @@ -109,22 +115,26 @@ forward_extremities_counter = Histogram( stale_forward_extremities_counter = Histogram( "synapse_storage_events_stale_forward_extremities_persisted", "Number of unchanged forward extremities for each new event", + labelnames=[SERVER_NAME_LABEL], buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), ) state_resolutions_during_persistence = Counter( "synapse_storage_events_state_resolutions_during_persistence", "Number of times we had to do state res to calculate new current state", + labelnames=[SERVER_NAME_LABEL], ) potential_times_prune_extremities = Counter( "synapse_storage_events_potential_times_prune_extremities", "Number of times we might be able to prune extremities", + labelnames=[SERVER_NAME_LABEL], ) times_pruned_extremities = Counter( "synapse_storage_events_times_pruned_extremities", "Number of times we were actually be able to prune extremities", + labelnames=[SERVER_NAME_LABEL], ) @@ -134,7 +144,7 @@ class _PersistEventsTask: name: ClassVar[str] = "persist_event_batch" # used for opentracing - events_and_contexts: List[Tuple[EventBase, EventContext]] + events_and_contexts: List[EventPersistencePair] backfilled: bool def try_merge(self, task: "_EventPersistQueueTask") -> bool: @@ -185,6 +195,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]): def __init__( self, + server_name: str, per_item_callback: Callable[ [str, _EventPersistQueueTask], Awaitable[_PersistResult], @@ -195,6 +206,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]): The per_item_callback will be called for each item added via add_to_queue, and its result will be returned via the Deferreds returned from add_to_queue. """ + self.server_name = server_name self._event_persist_queues: Dict[str, Deque[_EventPersistQueueItem]] = {} self._currently_persisting_rooms: Set[str] = set() self._per_item_callback = per_item_callback @@ -299,7 +311,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]): self._currently_persisting_rooms.discard(room_id) # set handle_queue_loop off in the background - run_as_background_process("persist_events", handle_queue_loop) + run_as_background_process("persist_events", self.server_name, handle_queue_loop) def _get_drainining_queue( self, room_id: str @@ -332,15 +344,17 @@ class EventsPersistenceStorageController: # store for now. self.main_store = stores.main self.state_store = stores.state + self._state_deletion_store = stores.state_deletion assert stores.persist_events self.persist_events_store = stores.persist_events + self.server_name = hs.hostname self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() self.is_mine_id = hs.is_mine_id self._event_persist_queue = _EventPeristenceQueue( - self._process_event_persist_queue_task + self.server_name, self._process_event_persist_queue_task ) self._state_resolution_handler = hs.get_state_resolution_handler() self._state_controller = state_controller @@ -377,7 +391,7 @@ class EventsPersistenceStorageController: @trace async def persist_events( self, - events_and_contexts: Iterable[Tuple[EventBase, EventContext]], + events_and_contexts: Iterable[EventPersistencePair], backfilled: bool = False, ) -> Tuple[List[EventBase], RoomStreamToken]: """ @@ -400,7 +414,7 @@ class EventsPersistenceStorageController: a room that has been un-partial stated. """ event_ids: List[str] = [] - partitioned: Dict[str, List[Tuple[EventBase, EventContext]]] = {} + partitioned: Dict[str, List[EventPersistencePair]] = {} for event, ctx in events_and_contexts: partitioned.setdefault(event.room_id, []).append((event, ctx)) event_ids.append(event.event_id) @@ -416,7 +430,7 @@ class EventsPersistenceStorageController: set_tag(SynapseTags.FUNC_ARG_PREFIX + "backfilled", str(backfilled)) async def enqueue( - item: Tuple[str, List[Tuple[EventBase, EventContext]]], + item: Tuple[str, List[EventPersistencePair]], ) -> Dict[str, str]: room_id, evs_ctxs = item return await self._event_persist_queue.add_to_queue( @@ -549,7 +563,9 @@ class EventsPersistenceStorageController: room_version, state_maps_by_state_group, event_map=None, - state_res_store=StateResolutionStore(self.main_store), + state_res_store=StateResolutionStore( + self.main_store, self._state_deletion_store + ), ) return await res.get_state(self._state_controller, StateFilter.all()) @@ -613,7 +629,11 @@ class EventsPersistenceStorageController: state_delta_for_room = None if not backfilled: - with Measure(self._clock, "_calculate_state_and_extrem"): + with Measure( + self._clock, + name="_calculate_state_and_extrem", + server_name=self.server_name, + ): # Work out the new "current state" for the room. # We do this by working out what the new extremities are and then # calculating the state from that. @@ -624,7 +644,11 @@ class EventsPersistenceStorageController: room_id, chunk ) - with Measure(self._clock, "calculate_chain_cover_index_for_events"): + with Measure( + self._clock, + name="calculate_chain_cover_index_for_events", + server_name=self.server_name, + ): # We now calculate chain ID/sequence numbers for any state events we're # persisting. We ignore out of band memberships as we're not in the room # and won't have their auth chain (we'll fix it up later if we join the @@ -635,20 +659,25 @@ class EventsPersistenceStorageController: room_id, [e for e, _ in chunk] ) - await self.persist_events_store._persist_events_and_state_updates( - room_id, - chunk, - state_delta_for_room=state_delta_for_room, - new_forward_extremities=new_forward_extremities, - use_negative_stream_ordering=backfilled, - inhibit_local_membership_updates=backfilled, - new_event_links=new_event_links, - ) + # Stop the state groups from being deleted while we're persisting + # them. + async with self._state_deletion_store.persisting_state_group_references( + events_and_contexts + ): + await self.persist_events_store._persist_events_and_state_updates( + room_id, + chunk, + state_delta_for_room=state_delta_for_room, + new_forward_extremities=new_forward_extremities, + use_negative_stream_ordering=backfilled, + inhibit_local_membership_updates=backfilled, + new_event_links=new_event_links, + ) return replaced_events async def _calculate_new_forward_extremities_and_state_delta( - self, room_id: str, ev_ctx_rm: List[Tuple[EventBase, EventContext]] + self, room_id: str, ev_ctx_rm: List[EventPersistencePair] ) -> Tuple[Optional[Set[str]], Optional[DeltaState]]: """Calculates the new forward extremities and state delta for a room given events to persist. @@ -690,9 +719,11 @@ class EventsPersistenceStorageController: if all_single_prev_not_state: return (new_forward_extremities, None) - state_delta_counter.inc() + state_delta_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc() if len(new_latest_event_ids) == 1: - state_delta_single_event_counter.inc() + state_delta_single_event_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() # This is a fairly handwavey check to see if we could # have guessed what the delta would have been when @@ -707,11 +738,17 @@ class EventsPersistenceStorageController: for ev, _ in ev_ctx_rm: prev_event_ids = set(ev.prev_event_ids()) if latest_event_ids == prev_event_ids: - state_delta_reuse_delta_counter.inc() + state_delta_reuse_delta_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() break logger.debug("Calculating state delta for room %s", room_id) - with Measure(self._clock, "persist_events.get_new_state_after_events"): + with Measure( + self._clock, + name="persist_events.get_new_state_after_events", + server_name=self.server_name, + ): res = await self._get_new_state_after_events( room_id, ev_ctx_rm, @@ -738,7 +775,11 @@ class EventsPersistenceStorageController: # removed keys entirely. delta = DeltaState([], delta_ids) elif current_state is not None: - with Measure(self._clock, "persist_events.calculate_state_delta"): + with Measure( + self._clock, + name="persist_events.calculate_state_delta", + server_name=self.server_name, + ): delta = await self._calculate_state_delta(room_id, current_state) if delta: @@ -761,7 +802,7 @@ class EventsPersistenceStorageController: async def _calculate_new_extremities( self, room_id: str, - event_contexts: List[Tuple[EventBase, EventContext]], + event_contexts: List[EventPersistencePair], latest_event_ids: AbstractSet[str], ) -> Set[str]: """Calculates the new forward extremities for a room given events to @@ -808,16 +849,20 @@ class EventsPersistenceStorageController: # We only update metrics for events that change forward extremities # (e.g. we ignore backfill/outliers/etc) if result != latest_event_ids: - forward_extremities_counter.observe(len(result)) + forward_extremities_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).observe(len(result)) stale = latest_event_ids & result - stale_forward_extremities_counter.observe(len(stale)) + stale_forward_extremities_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).observe(len(stale)) return result async def _get_new_state_after_events( self, room_id: str, - events_context: List[Tuple[EventBase, EventContext]], + events_context: List[EventPersistencePair], old_latest_event_ids: AbstractSet[str], new_latest_event_ids: Set[str], ) -> Tuple[Optional[StateMap[str]], Optional[StateMap[str]], Set[str]]: @@ -862,8 +907,7 @@ class EventsPersistenceStorageController: # This should only happen for outlier events. if not ev.internal_metadata.is_outlier(): raise Exception( - "Context for new event %s has no state " - "group" % (ev.event_id,) + "Context for new event %s has no state group" % (ev.event_id,) ) continue if ctx.state_group_deltas: @@ -965,10 +1009,14 @@ class EventsPersistenceStorageController: room_version, state_groups, events_map, - state_res_store=StateResolutionStore(self.main_store), + state_res_store=StateResolutionStore( + self.main_store, self._state_deletion_store + ), ) - state_resolutions_during_persistence.inc() + state_resolutions_during_persistence.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() # If the returned state matches the state group of one of the new # forward extremities then we check if we are able to prune some state @@ -991,12 +1039,14 @@ class EventsPersistenceStorageController: new_latest_event_ids: Set[str], resolved_state_group: int, event_id_to_state_group: Dict[str, int], - events_context: List[Tuple[EventBase, EventContext]], + events_context: List[EventPersistencePair], ) -> Set[str]: """See if we can prune any of the extremities after calculating the resolved state. """ - potential_times_prune_extremities.inc() + potential_times_prune_extremities.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() # We keep all the extremities that have the same state group, and # see if we can drop the others. @@ -1094,7 +1144,7 @@ class EventsPersistenceStorageController: return new_latest_event_ids - times_pruned_extremities.inc() + times_pruned_extremities.labels(**{SERVER_NAME_LABEL: self.server_name}).inc() logger.info( "Pruning forward extremities in room %s: from %s -> %s", @@ -1126,7 +1176,7 @@ class EventsPersistenceStorageController: async def _is_server_still_joined( self, room_id: str, - ev_ctx_rm: List[Tuple[EventBase, EventContext]], + ev_ctx_rm: List[EventPersistencePair], delta: DeltaState, ) -> bool: """Check if the server will still be joined after the given events have diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index e794b370c2..14b37ac543 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -21,10 +21,20 @@ import itertools import logging -from typing import TYPE_CHECKING, Set +from typing import ( + TYPE_CHECKING, + Collection, + Mapping, + Optional, + Set, +) from synapse.logging.context import nested_logging_context +from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.storage.database import LoggingTransaction from synapse.storage.databases import Databases +from synapse.types.storage import _BackgroundUpdates +from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer @@ -36,14 +46,27 @@ class PurgeEventsStorageController: """High level interface for purging rooms and event history.""" def __init__(self, hs: "HomeServer", stores: Databases): + self.server_name = ( + hs.hostname + ) # nb must be called this for @wrap_as_background_process self.stores = stores + if hs.config.worker.run_background_tasks: + self._delete_state_loop_call = hs.get_clock().looping_call( + self._delete_state_groups_loop, 60 * 1000 + ) + + self.stores.state.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE, + self._background_delete_unrefereneced_state_groups, + ) + async def purge_room(self, room_id: str) -> None: """Deletes all record of a room""" with nested_logging_context(room_id): - state_groups_to_delete = await self.stores.main.purge_room(room_id) - await self.stores.state.purge_room_state(room_id, state_groups_to_delete) + await self.stores.main.purge_room(room_id) + await self.stores.state.purge_room_state(room_id) async def purge_history( self, room_id: str, token: str, delete_local_events: bool @@ -68,11 +91,16 @@ class PurgeEventsStorageController: logger.info("[purge] finding state groups that can be deleted") sg_to_delete = await self._find_unreferenced_groups(state_groups) - await self.stores.state.purge_unreferenced_state_groups( - room_id, sg_to_delete + # Mark these state groups as pending deletion, they will actually + # get deleted automatically later. + await self.stores.state_deletion.mark_state_groups_as_pending_deletion( + sg_to_delete ) - async def _find_unreferenced_groups(self, state_groups: Set[int]) -> Set[int]: + async def _find_unreferenced_groups( + self, + state_groups: Collection[int], + ) -> Set[int]: """Used when purging history to figure out which state groups can be deleted. @@ -118,6 +146,313 @@ class PurgeEventsStorageController: next_to_search |= prevs state_groups_seen |= prevs + # We also check to see if anything referencing the state groups are + # also unreferenced. This helps ensure that we delete unreferenced + # state groups, if we don't then we will de-delta them when we + # delete the other state groups leading to increased DB usage. + next_edges = await self.stores.state.get_next_state_groups(current_search) + nexts = set(next_edges.keys()) + nexts -= state_groups_seen + next_to_search |= nexts + state_groups_seen |= nexts + to_delete = state_groups_seen - referenced_groups return to_delete + + @wrap_as_background_process("_delete_state_groups_loop") + async def _delete_state_groups_loop(self) -> None: + """Background task that deletes any state groups that may be pending + deletion.""" + + while True: + next_to_delete = await self.stores.state_deletion.get_next_state_group_collection_to_delete() + if next_to_delete is None: + break + + (room_id, groups_to_sequences) = next_to_delete + + logger.info( + "[purge] deleting state groups for room %s: %s", + room_id, + shortstr(groups_to_sequences.keys(), maxitems=10), + ) + made_progress = await self._delete_state_groups( + room_id, groups_to_sequences + ) + + # If no progress was made in deleting the state groups, then we + # break to allow a pause before trying again next time we get + # called. + if not made_progress: + break + + async def _delete_state_groups( + self, room_id: str, groups_to_sequences: Mapping[int, int] + ) -> bool: + """Tries to delete the given state groups. + + Returns: + Whether we made progress in deleting the state groups (or marking + them as referenced). + """ + + # We double check if any of the state groups have become referenced. + # This shouldn't happen, as any usages should cause the state group to + # be removed as pending deletion. + referenced_state_groups = await self.stores.main.get_referenced_state_groups( + groups_to_sequences + ) + + if referenced_state_groups: + # We mark any state groups that have become referenced as being + # used. + await self.stores.state_deletion.mark_state_groups_as_used( + referenced_state_groups + ) + + # Update list of state groups to remove referenced ones + groups_to_sequences = { + state_group: sequence_number + for state_group, sequence_number in groups_to_sequences.items() + if state_group not in referenced_state_groups + } + + if not groups_to_sequences: + # We made progress here as long as we marked some state groups as + # now referenced. + return len(referenced_state_groups) > 0 + + return await self.stores.state.purge_unreferenced_state_groups( + room_id, + groups_to_sequences, + ) + + async def _background_delete_unrefereneced_state_groups( + self, progress: dict, batch_size: int + ) -> int: + """This background update will slowly delete any unreferenced state groups""" + + last_checked_state_group = progress.get("last_checked_state_group") + + if last_checked_state_group is None: + # This is the first run. + last_checked_state_group = ( + await self.stores.state.db_pool.simple_select_one_onecol( + table="state_groups", + keyvalues={}, + retcol="MAX(id)", + allow_none=True, + desc="get_max_state_group", + ) + ) + if last_checked_state_group is None: + # There are no state groups so the background process is finished. + await self.stores.state.db_pool.updates._end_background_update( + _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE + ) + return batch_size + last_checked_state_group += 1 + + ( + last_checked_state_group, + final_batch, + ) = await self._delete_unreferenced_state_groups_batch( + last_checked_state_group, + batch_size, + ) + + if not final_batch: + # There are more state groups to check. + progress = { + "last_checked_state_group": last_checked_state_group, + } + await self.stores.state.db_pool.updates._background_update_progress( + _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE, + progress, + ) + else: + # This background process is finished. + await self.stores.state.db_pool.updates._end_background_update( + _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE + ) + + return batch_size + + async def _delete_unreferenced_state_groups_batch( + self, + last_checked_state_group: int, + batch_size: int, + ) -> tuple[int, bool]: + """Looks for unreferenced state groups starting from the last state group + checked and marks them for deletion. + + Args: + last_checked_state_group: The last state group that was checked. + batch_size: How many state groups to process in this iteration. + + Returns: + (last_checked_state_group, final_batch) + """ + + # Find all state groups that can be deleted if any of the original set are deleted. + ( + to_delete, + last_checked_state_group, + final_batch, + ) = await self._find_unreferenced_groups_for_background_deletion( + last_checked_state_group, batch_size + ) + + if len(to_delete) == 0: + return last_checked_state_group, final_batch + + await self.stores.state_deletion.mark_state_groups_as_pending_deletion( + to_delete + ) + + return last_checked_state_group, final_batch + + async def _find_unreferenced_groups_for_background_deletion( + self, + last_checked_state_group: int, + batch_size: int, + ) -> tuple[Set[int], int, bool]: + """Used when deleting unreferenced state groups in the background to figure out + which state groups can be deleted. + To avoid increased DB usage due to de-deltaing state groups, this returns only + state groups which are free standing (ie. no shared edges with referenced groups) or + state groups which do not share edges which result in a future referenced group. + + The following scenarios outline the possibilities based on state group data in + the DB. + + ie. Free standing -> state groups 1-N would be returned: + SG_1 + | + ... + | + SG_N + + ie. Previous reference -> state groups 2-N would be returned: + SG_1 <- referenced by event + | + SG_2 + | + ... + | + SG_N + + ie. Future reference -> none of the following state groups would be returned: + SG_1 + | + SG_2 + | + ... + | + SG_N <- referenced by event + + Args: + last_checked_state_group: The last state group that was checked. + batch_size: How many state groups to process in this iteration. + + Returns: + (to_delete, last_checked_state_group, final_batch) + """ + + # If a state group's next edge is not pending deletion then we don't delete the state group. + # If there is no next edge or the next edges are all marked for deletion, then delete + # the state group. + # This holds since we walk backwards from the latest state groups, ensuring that + # we've already checked newer state groups for event references along the way. + def get_next_state_groups_marked_for_deletion_txn( + txn: LoggingTransaction, + ) -> tuple[dict[int, bool], dict[int, int]]: + state_group_sql = """ + SELECT s.id, e.state_group, d.state_group + FROM ( + SELECT id FROM state_groups + WHERE id < ? ORDER BY id DESC LIMIT ? + ) as s + LEFT JOIN state_group_edges AS e ON (s.id = e.prev_state_group) + LEFT JOIN state_groups_pending_deletion AS d ON (e.state_group = d.state_group) + """ + txn.execute(state_group_sql, (last_checked_state_group, batch_size)) + + # Mapping from state group to whether we should delete it. + state_groups_to_deletion: dict[int, bool] = {} + + # Mapping from state group to prev state group. + state_groups_to_prev: dict[int, int] = {} + + for row in txn: + state_group = row[0] + next_edge = row[1] + pending_deletion = row[2] + + if next_edge is not None: + state_groups_to_prev[next_edge] = state_group + + if next_edge is not None and not pending_deletion: + # We have found an edge not marked for deletion. + # Check previous results to see if this group is part of a chain + # within this batch that qualifies for deletion. + # ie. batch contains: + # SG_1 -> SG_2 -> SG_3 + # If SG_3 is a candidate for deletion, then SG_2 & SG_1 should also + # be, even though they have edges which may not be marked for + # deletion. + # This relies on SQL results being sorted in DESC order to work. + next_is_deletion_candidate = state_groups_to_deletion.get(next_edge) + if ( + next_is_deletion_candidate is None + or not next_is_deletion_candidate + ): + state_groups_to_deletion[state_group] = False + else: + state_groups_to_deletion.setdefault(state_group, True) + else: + # This state group may be a candidate for deletion + state_groups_to_deletion.setdefault(state_group, True) + + return state_groups_to_deletion, state_groups_to_prev + + ( + state_groups_to_deletion, + state_group_edges, + ) = await self.stores.state.db_pool.runInteraction( + "get_next_state_groups_marked_for_deletion", + get_next_state_groups_marked_for_deletion_txn, + ) + deletion_candidates = { + state_group + for state_group, deletion in state_groups_to_deletion.items() + if deletion + } + + final_batch = False + state_groups = state_groups_to_deletion.keys() + if len(state_groups) < batch_size: + final_batch = True + else: + last_checked_state_group = min(state_groups) + + if len(state_groups) == 0: + return set(), last_checked_state_group, final_batch + + # Determine if any of the remaining state groups are directly referenced. + referenced = await self.stores.main.get_referenced_state_groups( + deletion_candidates + ) + + # Remove state groups from deletion_candidates which are directly referenced or share a + # future edge with a referenced state group within this batch. + def filter_reference_chains(group: Optional[int]) -> None: + while group is not None: + deletion_candidates.discard(group) + group = state_group_edges.get(group) + + for referenced_group in referenced: + filter_reference_chains(referenced_group) + + return deletion_candidates, last_checked_state_group, final_batch diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index f28f5d7e03..8997f4526f 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -68,6 +68,7 @@ class StateStorageController: """ def __init__(self, hs: "HomeServer", stores: "Databases"): + self.server_name = hs.hostname # nb must be called this for @cached self._is_mine_id = hs.is_mine_id self._clock = hs.get_clock() self.stores = stores @@ -812,7 +813,9 @@ class StateStorageController: state_group = object() assert state_group is not None - with Measure(self._clock, "get_joined_hosts"): + with Measure( + self._clock, name="get_joined_hosts", server_name=self.server_name + ): return await self._get_joined_hosts( room_id, state_group, state_entry=state_entry ) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index cb4a5857be..aae029f910 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -35,6 +35,7 @@ from typing import ( Iterable, Iterator, List, + Literal, Mapping, Optional, Sequence, @@ -47,7 +48,7 @@ from typing import ( import attr from prometheus_client import Counter, Histogram -from typing_extensions import Concatenate, Literal, ParamSpec +from typing_extensions import Concatenate, ParamSpec from twisted.enterprise import adbapi from twisted.internet.interfaces import IReactorCore @@ -60,7 +61,7 @@ from synapse.logging.context import ( current_context, make_deferred_yieldable, ) -from synapse.metrics import LaterGauge, register_threadpool +from synapse.metrics import SERVER_NAME_LABEL, register_threadpool from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.background_updates import BackgroundUpdater from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine @@ -81,11 +82,23 @@ sql_logger = logging.getLogger("synapse.storage.SQL") transaction_logger = logging.getLogger("synapse.storage.txn") perf_logger = logging.getLogger("synapse.storage.TIME") -sql_scheduling_timer = Histogram("synapse_storage_schedule_time", "sec") +sql_scheduling_timer = Histogram( + "synapse_storage_schedule_time", "sec", labelnames=[SERVER_NAME_LABEL] +) -sql_query_timer = Histogram("synapse_storage_query_time", "sec", ["verb"]) -sql_txn_count = Counter("synapse_storage_transaction_time_count", "sec", ["desc"]) -sql_txn_duration = Counter("synapse_storage_transaction_time_sum", "sec", ["desc"]) +sql_query_timer = Histogram( + "synapse_storage_query_time", "sec", labelnames=["verb", SERVER_NAME_LABEL] +) +sql_txn_count = Counter( + "synapse_storage_transaction_time_count", + "sec", + labelnames=["desc", SERVER_NAME_LABEL], +) +sql_txn_duration = Counter( + "synapse_storage_transaction_time_sum", + "sec", + labelnames=["desc", SERVER_NAME_LABEL], +) # Unique indexes which have been added in background updates. Maps from table name @@ -117,9 +130,11 @@ class _PoolConnection(Connection): def make_pool( + *, reactor: IReactorCore, db_config: DatabaseConnectionConfig, engine: BaseDatabaseEngine, + server_name: str, ) -> adbapi.ConnectionPool: """Get the connection pool for the database.""" @@ -133,7 +148,12 @@ def make_pool( # etc. with LoggingContext("db.on_new_connection"): engine.on_new_connection( - LoggingDatabaseConnection(conn, engine, "on_new_connection") + LoggingDatabaseConnection( + conn=conn, + engine=engine, + default_txn_name="on_new_connection", + server_name=server_name, + ) ) connection_pool = adbapi.ConnectionPool( @@ -143,15 +163,21 @@ def make_pool( **db_args, ) - register_threadpool(f"database-{db_config.name}", connection_pool.threadpool) + register_threadpool( + name=f"database-{db_config.name}", + server_name=server_name, + threadpool=connection_pool.threadpool, + ) return connection_pool def make_conn( + *, db_config: DatabaseConnectionConfig, engine: BaseDatabaseEngine, default_txn_name: str, + server_name: str, ) -> "LoggingDatabaseConnection": """Make a new connection to the database and return it. @@ -165,13 +191,18 @@ def make_conn( if not k.startswith("cp_") } native_db_conn = engine.module.connect(**db_params) - db_conn = LoggingDatabaseConnection(native_db_conn, engine, default_txn_name) + db_conn = LoggingDatabaseConnection( + conn=native_db_conn, + engine=engine, + default_txn_name=default_txn_name, + server_name=server_name, + ) engine.on_new_connection(db_conn) return db_conn -@attr.s(slots=True, auto_attribs=True) +@attr.s(slots=True, auto_attribs=True, kw_only=True) class LoggingDatabaseConnection: """A wrapper around a database connection that returns `LoggingTransaction` as its cursor class. @@ -182,6 +213,7 @@ class LoggingDatabaseConnection: conn: Connection engine: BaseDatabaseEngine default_txn_name: str + server_name: str def cursor( self, @@ -195,8 +227,9 @@ class LoggingDatabaseConnection: txn_name = self.default_txn_name return LoggingTransaction( - self.conn.cursor(), + txn=self.conn.cursor(), name=txn_name, + server_name=self.server_name, database_engine=self.engine, after_callbacks=after_callbacks, async_after_callbacks=async_after_callbacks, @@ -265,6 +298,7 @@ class LoggingTransaction: __slots__ = [ "txn", "name", + "server_name", "database_engine", "after_callbacks", "async_after_callbacks", @@ -273,8 +307,10 @@ class LoggingTransaction: def __init__( self, + *, txn: Cursor, name: str, + server_name: str, database_engine: BaseDatabaseEngine, after_callbacks: Optional[List[_CallbackListEntry]] = None, async_after_callbacks: Optional[List[_AsyncCallbackListEntry]] = None, @@ -282,6 +318,7 @@ class LoggingTransaction: ): self.txn = txn self.name = name + self.server_name = server_name self.database_engine = database_engine self.after_callbacks = after_callbacks self.async_after_callbacks = async_after_callbacks @@ -492,7 +529,9 @@ class LoggingTransaction: finally: secs = time.time() - start sql_logger.debug("[SQL time] {%s} %f sec", self.name, secs) - sql_query_timer.labels(sql.split()[0]).observe(secs) + sql_query_timer.labels( + verb=sql.split()[0], **{SERVER_NAME_LABEL: self.server_name} + ).observe(secs) def close(self) -> None: self.txn.close() @@ -560,18 +599,18 @@ class DatabasePool: engine: BaseDatabaseEngine, ): self.hs = hs + self.server_name = hs.hostname self._clock = hs.get_clock() self._txn_limit = database_config.config.get("txn_limit", 0) self._database_config = database_config - self._db_pool = make_pool(hs.get_reactor(), database_config, engine) + self._db_pool = make_pool( + reactor=hs.get_reactor(), + db_config=database_config, + engine=engine, + server_name=self.server_name, + ) self.updates = BackgroundUpdater(hs, self) - LaterGauge( - "synapse_background_update_status", - "Background update status", - [], - self.updates.get_status, - ) self._previous_txn_total_time = 0.0 self._current_txn_total_time = 0.0 @@ -601,6 +640,7 @@ class DatabasePool: 0.0, run_as_background_process, "upsert_safety_check", + self.server_name, self._check_safe_to_upsert, ) @@ -643,6 +683,7 @@ class DatabasePool: 15.0, run_as_background_process, "upsert_safety_check", + self.server_name, self._check_safe_to_upsert, ) @@ -865,8 +906,14 @@ class DatabasePool: self._current_txn_total_time += duration self._txn_perf_counters.update(desc, duration) - sql_txn_count.labels(desc).inc(1) - sql_txn_duration.labels(desc).inc(duration) + sql_txn_count.labels( + desc=desc, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(1) + sql_txn_duration.labels( + desc=desc, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(duration) async def runInteraction( self, @@ -1002,7 +1049,9 @@ class DatabasePool: operation_name="db.connection", ): sched_duration_sec = monotonic_time() - start_time - sql_scheduling_timer.observe(sched_duration_sec) + sql_scheduling_timer.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).observe(sched_duration_sec) context.add_database_scheduled(sched_duration_sec) if self._txn_limit > 0: @@ -1035,7 +1084,10 @@ class DatabasePool: ) db_conn = LoggingDatabaseConnection( - conn, self.engine, "runWithConnection" + conn=conn, + engine=self.engine, + default_txn_name="runWithConnection", + server_name=self.server_name, ) return func(db_conn, *args, **kwargs) finally: @@ -1477,13 +1529,49 @@ class DatabasePool: """ Upsert, many times. + This executes a query equivalent to `INSERT INTO ... ON CONFLICT DO UPDATE`, + with multiple value rows. + The query may use emulated upserts if the database engine does not support upserts, + or if the table is currently unsafe to upsert. + + If there are no value columns, this instead generates a `ON CONFLICT DO NOTHING`. + Args: table: The table to upsert into - key_names: The key column names. - key_values: A list of each row's key column values. - value_names: The value column names - value_values: A list of each row's value column values. + key_names: The unique key column names. These are the columns used in the ON CONFLICT clause. + key_values: A list of each row's key column values, in the same order as `key_names`. + value_names: The non-unique value column names + value_values: A list of each row's value column values, in the same order as `value_names`. Ignored if value_names is empty. + + Example: + ```python + simple_upsert_many( + "mytable", + key_names=("room_id", "user_id"), + key_values=[ + ("!room1:example.org", "@user1:example.org"), + ("!room2:example.org", "@user2:example.org"), + ], + value_names=("wombat_count", "is_updated"), + value_values=[ + (42, True), + (7, False) + ], + ) + ``` + + gives something equivalent to: + + ```sql + INSERT INTO mytable (room_id, user_id, wombat_count, is_updated) + VALUES + ('!room1:example.org', '@user1:example.org', 42, True), + ('!room2:example.org', '@user2:example.org', 7, False) + ON CONFLICT DO UPDATE SET + wombat_count = EXCLUDED.wombat_count, + is_updated = EXCLUDED.is_updated + ``` """ # We can autocommit if it safe to upsert @@ -1512,6 +1600,8 @@ class DatabasePool: """ Upsert, many times. + See the documentation for `simple_upsert_many` for examples. + Args: table: The table to upsert into key_names: The key column names. @@ -2159,10 +2249,26 @@ class DatabasePool: if rowcount > 1: raise StoreError(500, "More than one row matched (%s)" % (table,)) - # Ideally we could use the overload decorator here to specify that the - # return type is only optional if allow_none is True, but this does not work - # when you call a static method from an instance. - # See https://github.com/python/mypy/issues/7781 + @overload + @staticmethod + def simple_select_one_txn( + txn: LoggingTransaction, + table: str, + keyvalues: Dict[str, Any], + retcols: Collection[str], + allow_none: Literal[False] = False, + ) -> Tuple[Any, ...]: ... + + @overload + @staticmethod + def simple_select_one_txn( + txn: LoggingTransaction, + table: str, + keyvalues: Dict[str, Any], + retcols: Collection[str], + allow_none: Literal[True] = True, + ) -> Optional[Tuple[Any, ...]]: ... + @staticmethod def simple_select_one_txn( txn: LoggingTransaction, @@ -2547,8 +2653,7 @@ def make_in_list_sql_clause( # These overloads ensure that `columns` and `iterable` values have the same length. -# Suppress "Single overload definition, multiple required" complaint. -@overload # type: ignore[misc] +@overload def make_tuple_in_list_sql_clause( database_engine: BaseDatabaseEngine, columns: Tuple[str, str], @@ -2556,6 +2661,14 @@ def make_tuple_in_list_sql_clause( ) -> Tuple[str, list]: ... +@overload +def make_tuple_in_list_sql_clause( + database_engine: BaseDatabaseEngine, + columns: Tuple[str, str, str], + iterable: Collection[Tuple[Any, Any, Any]], +) -> Tuple[str, list]: ... + + def make_tuple_in_list_sql_clause( database_engine: BaseDatabaseEngine, columns: Tuple[str, ...], diff --git a/synapse/storage/databases/__init__.py b/synapse/storage/databases/__init__.py index dd9fc01fb0..a4aba96686 100644 --- a/synapse/storage/databases/__init__.py +++ b/synapse/storage/databases/__init__.py @@ -22,10 +22,12 @@ import logging from typing import TYPE_CHECKING, Generic, List, Optional, Type, TypeVar +from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.storage._base import SQLBaseStore from synapse.storage.database import DatabasePool, make_conn from synapse.storage.databases.main.events import PersistEventsStore from synapse.storage.databases.state import StateGroupDataStore +from synapse.storage.databases.state.deletion import StateDeletionDataStore from synapse.storage.engines import create_engine from synapse.storage.prepare_database import prepare_database @@ -39,6 +41,13 @@ logger = logging.getLogger(__name__) DataStoreT = TypeVar("DataStoreT", bound=SQLBaseStore, covariant=True) +background_update_status = LaterGauge( + name="synapse_background_update_status", + desc="Background update status", + labelnames=["database_name", SERVER_NAME_LABEL], +) + + class Databases(Generic[DataStoreT]): """The various databases. @@ -49,12 +58,14 @@ class Databases(Generic[DataStoreT]): main state persist_events + state_deletion """ databases: List[DatabasePool] main: "DataStore" # FIXME: https://github.com/matrix-org/synapse/issues/11165: actually an instance of `main_store_class` state: StateGroupDataStore persist_events: Optional[PersistEventsStore] + state_deletion: StateDeletionDataStore def __init__(self, main_store_class: Type[DataStoreT], hs: "HomeServer"): # Note we pass in the main store class here as workers use a different main @@ -63,13 +74,21 @@ class Databases(Generic[DataStoreT]): self.databases = [] main: Optional[DataStoreT] = None state: Optional[StateGroupDataStore] = None + state_deletion: Optional[StateDeletionDataStore] = None persist_events: Optional[PersistEventsStore] = None + server_name = hs.hostname + for database_config in hs.config.database.databases: db_name = database_config.name engine = create_engine(database_config.config) - with make_conn(database_config, engine, "startup") as db_conn: + with make_conn( + db_config=database_config, + engine=engine, + default_txn_name="startup", + server_name=server_name, + ) as db_conn: logger.info("[database config %r]: Checking database server", db_name) engine.check_database(db_conn) @@ -114,7 +133,8 @@ class Databases(Generic[DataStoreT]): if state: raise Exception("'state' data store already configured") - state = StateGroupDataStore(database, db_conn, hs) + state_deletion = StateDeletionDataStore(database, db_conn, hs) + state = StateGroupDataStore(database, db_conn, hs, state_deletion) db_conn.commit() @@ -131,11 +151,20 @@ class Databases(Generic[DataStoreT]): db_conn.close() + # Track the background update status for each database + background_update_status.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: { + (database.name(), server_name): database.updates.get_status() + for database in self.databases + }, + ) + # Sanity check that we have actually configured all the required stores. if not main: raise Exception("No 'main' database configured") - if not state: + if not state or not state_deletion: raise Exception("No 'state' database configured") # We use local variables here to ensure that the databases do not have @@ -143,3 +172,4 @@ class Databases(Generic[DataStoreT]): self.main = main # type: ignore[assignment] self.state = state self.persist_events = persist_events + self.state_deletion = state_deletion diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 86431f6e40..de55c452ae 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -19,7 +19,6 @@ # [This file includes modifications made by New Vector Limited] # # - import logging from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast @@ -35,6 +34,9 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.sliding_sync import SlidingSyncStore from synapse.storage.databases.main.stats import UserSortOrder +from synapse.storage.databases.main.thread_subscriptions import ( + ThreadSubscriptionsWorkerStore, +) from synapse.storage.engines import BaseDatabaseEngine from synapse.storage.types import Cursor from synapse.types import get_domain_from_id @@ -141,6 +143,7 @@ class DataStore( SearchStore, TagsStore, AccountDataStore, + ThreadSubscriptionsWorkerStore, PushRulesWorkerStore, StreamWorkerStore, OpenIdStore, diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index e583c182ba..c049789e44 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -34,8 +34,10 @@ from typing import ( ) from synapse.api.constants import AccountDataTypes +from synapse.api.errors import Codes, SynapseError from synapse.replication.tcp.streams import AccountDataStream from synapse.storage._base import db_to_json +from synapse.storage.admin_client_config import AdminClientConfig from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, @@ -43,6 +45,7 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.push_rule import PushRulesWorkerStore +from synapse.storage.invite_rule import InviteRulesConfig from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, JsonMapping from synapse.util import json_encoder @@ -75,6 +78,7 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) db=database, notifier=hs.get_replication_notifier(), stream_name="account_data", + server_name=self.server_name, instance_name=self._instance_name, tables=[ ("room_account_data", "instance_name", "stream_id"), @@ -87,7 +91,9 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) account_max = self.get_max_account_data_stream_id() self._account_data_stream_cache = StreamChangeCache( - "AccountDataAndTagsChangeCache", account_max + name="AccountDataAndTagsChangeCache", + server_name=self.server_name, + current_stream_pos=account_max, ) self.db_pool.updates.register_background_index_update( @@ -102,6 +108,8 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) self._delete_account_data_for_deactivated_users, ) + self._msc4155_enabled = hs.config.experimental.msc4155_enabled + def get_max_account_data_stream_id(self) -> int: """Get the current max stream ID for account data stream @@ -557,6 +565,38 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) ) ) + async def get_invite_config_for_user(self, user_id: str) -> InviteRulesConfig: + """ + Get the invite configuration for the current user. + + Args: + user_id: + """ + + if not self._msc4155_enabled: + # This equates to allowing all invites, as if the setting was off. + return InviteRulesConfig(None) + + data = await self.get_global_account_data_by_type_for_user( + user_id, AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG + ) + return InviteRulesConfig(data) + + async def get_admin_client_config_for_user(self, user_id: str) -> AdminClientConfig: + """ + Get the admin client configuration for the specified user. + + The admin client config contains Synapse-specific settings that clients running + server admin accounts can use. They have no effect on non-admin users. + + Args: + user_id: The user ID to get config for. + """ + data = await self.get_global_account_data_by_type_for_user( + user_id, AccountDataTypes.SYNAPSE_ADMIN_CLIENT_CONFIG + ) + return AdminClientConfig(data) + def process_replication_rows( self, stream_name: str, @@ -760,6 +800,9 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) else: currently_ignored_users = set() + if user_id in currently_ignored_users: + raise SynapseError(400, "You cannot ignore yourself", Codes.INVALID_PARAM) + # If the data has not changed, nothing to do. if previously_ignored_users == currently_ignored_users: return diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py index 766c94fc14..9862e574fd 100644 --- a/synapse/storage/databases/main/appservice.py +++ b/synapse/storage/databases/main/appservice.py @@ -126,7 +126,7 @@ class ApplicationServiceWorkerStore(RoomMemberWorkerStore): The application service or None. """ for service in self.services_cache: - if service.sender == user_id: + if service.sender.to_string() == user_id: return service return None diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index 32c3472e58..cad26fefa4 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -21,6 +21,7 @@ import itertools +import json import logging from typing import TYPE_CHECKING, Any, Collection, Iterable, List, Optional, Tuple @@ -41,7 +42,6 @@ from synapse.storage.database import ( LoggingDatabaseConnection, LoggingTransaction, ) -from synapse.storage.databases.main.events import SLIDING_SYNC_RELEVANT_STATE_SET from synapse.storage.engines import PostgresEngine from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.util.caches.descriptors import CachedFunction @@ -63,6 +63,12 @@ PURGE_HISTORY_CACHE_NAME = "ph_cache_fake" # As above, but for invalidating room caches on room deletion DELETE_ROOM_CACHE_NAME = "dr_cache_fake" +# This cache takes a list of tuples as its first argument, which requires +# special handling. +GET_E2E_CROSS_SIGNING_SIGNATURES_FOR_DEVICE_CACHE_NAME = ( + "_get_e2e_cross_signing_signatures_for_device" +) + # How long between cache invalidation table cleanups, once we have caught up # with the backlog. REGULAR_CLEANUP_INTERVAL_MS = Config.parse_duration("1h") @@ -105,10 +111,11 @@ class CacheInvalidationWorkerStore(SQLBaseStore): # caches to invalidate. (This reduces the amount of writes to the DB # that happen). self._cache_id_gen = MultiWriterIdGenerator( - db_conn, - database, + db_conn=db_conn, + db=database, notifier=hs.get_replication_notifier(), stream_name="caches", + server_name=self.server_name, instance_name=hs.get_instance_name(), tables=[ ( @@ -219,6 +226,11 @@ class CacheInvalidationWorkerStore(SQLBaseStore): room_id = row.keys[0] members_changed = set(row.keys[1:]) self._invalidate_state_caches(room_id, members_changed) + self._curr_state_delta_stream_cache.entity_has_changed( # type: ignore[attr-defined] + room_id, token + ) + for user_id in members_changed: + self._membership_stream_cache.entity_has_changed(user_id, token) # type: ignore[attr-defined] elif row.cache_func == PURGE_HISTORY_CACHE_NAME: if row.keys is None: raise Exception( @@ -236,6 +248,62 @@ class CacheInvalidationWorkerStore(SQLBaseStore): room_id = row.keys[0] self._invalidate_caches_for_room_events(room_id) self._invalidate_caches_for_room(room_id) + self._curr_state_delta_stream_cache.entity_has_changed( # type: ignore[attr-defined] + room_id, token + ) + # Note: This code is commented out to improve cache performance. + # While uncommenting would provide complete correctness, our + # automatic forgotten room purge logic (see + # `forgotten_room_retention_period`) means this would frequently + # clear the entire cache (effectively) and probably have a noticable + # impact on the cache hit ratio. + # + # Not updating the cache here is safe because: + # + # 1. `_membership_stream_cache` is only used to indicate the + # *absence* of changes, i.e. "nothing has changed between tokens + # X and Y and so return early and don't query the database". + # 2. `_membership_stream_cache` is used when we query data from + # `current_state_delta_stream` and `room_memberships` but since + # nothing new is written to the database for those tables when + # purging/deleting a room (only deleting rows), there is nothing + # changed to care about. + # + # At worst, the cache might indicate a change at token X, at which + # point, we will query the database and discover nothing is there. + # + # Ideally, we would make it so that we could clear the cache on a + # more granular level but that's a bit complex and fiddly to do with + # room membership. + # + # self._membership_stream_cache.all_entities_changed(token) # type: ignore[attr-defined] + elif ( + row.cache_func + == GET_E2E_CROSS_SIGNING_SIGNATURES_FOR_DEVICE_CACHE_NAME + ): + # "keys" is a list of strings, where each string is a + # JSON-encoded representation of the tuple keys, i.e. + # keys: ['["@userid:domain", "DEVICEID"]','["@userid2:domain", "DEVICEID2"]'] + # + # This is a side-effect of not being able to send nested + # information over replication. + for json_str in row.keys: + try: + user_id, device_id = json.loads(json_str) + except (json.JSONDecodeError, TypeError): + logger.error( + "Failed to deserialise cache key as valid JSON: %s", + json_str, + ) + continue + + # Invalidate each key. + # + # Note: .invalidate takes a tuple of arguments, hence the need + # to nest our tuple in another tuple. + self._get_e2e_cross_signing_signatures_for_device.invalidate( # type: ignore[attr-defined] + ((user_id, device_id),) + ) else: self._attempt_to_invalidate_cache(row.cache_func, row.keys) @@ -250,6 +318,11 @@ class CacheInvalidationWorkerStore(SQLBaseStore): super().process_replication_position(stream_name, instance_name, token) def _process_event_stream_row(self, token: int, row: EventsStreamRow) -> None: + # This is needed to avoid a circular import. + from synapse.storage.databases.main.events import ( + SLIDING_SYNC_RELEVANT_STATE_SET, + ) + data = row.data if row.type == EventsStreamEventRow.TypeId: @@ -273,8 +346,9 @@ class CacheInvalidationWorkerStore(SQLBaseStore): "get_rooms_for_user", (data.state_key,) ) self._attempt_to_invalidate_cache( - "get_sliding_sync_rooms_for_user", None + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None ) + self._membership_stream_cache.entity_has_changed(data.state_key, token) # type: ignore[attr-defined] elif data.type == EventTypes.RoomEncryption: self._attempt_to_invalidate_cache( "get_room_encryption", (data.room_id,) @@ -284,17 +358,20 @@ class CacheInvalidationWorkerStore(SQLBaseStore): if (data.type, data.state_key) in SLIDING_SYNC_RELEVANT_STATE_SET: self._attempt_to_invalidate_cache( - "get_sliding_sync_rooms_for_user", None + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None ) elif row.type == EventsStreamAllStateRow.TypeId: assert isinstance(data, EventsStreamAllStateRow) # Similar to the above, but the entire caches are invalidated. This is # unfortunate for the membership caches, but should recover quickly. self._curr_state_delta_stream_cache.entity_has_changed(data.room_id, token) # type: ignore[attr-defined] + self._membership_stream_cache.all_entities_changed(token) # type: ignore[attr-defined] self._attempt_to_invalidate_cache("get_rooms_for_user", None) self._attempt_to_invalidate_cache("get_room_type", (data.room_id,)) self._attempt_to_invalidate_cache("get_room_encryption", (data.room_id,)) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) else: raise Exception("Unknown events stream row type %s" % (row.type,)) @@ -309,6 +386,11 @@ class CacheInvalidationWorkerStore(SQLBaseStore): relates_to: Optional[str], backfilled: bool, ) -> None: + # This is needed to avoid a circular import. + from synapse.storage.databases.main.events import ( + SLIDING_SYNC_RELEVANT_STATE_SET, + ) + # XXX: If you add something to this function make sure you add it to # `_invalidate_caches_for_room_events` as well. @@ -322,6 +404,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache( "get_unread_event_push_actions_by_room_for_user", (room_id,) ) + self._attempt_to_invalidate_cache("get_metadata_for_event", (room_id, event_id)) self._attempt_to_invalidate_cache("_get_max_event_pos", (room_id,)) @@ -357,7 +440,8 @@ class CacheInvalidationWorkerStore(SQLBaseStore): "_get_rooms_for_local_user_where_membership_is_inner", (state_key,) ) self._attempt_to_invalidate_cache( - "get_sliding_sync_rooms_for_user", (state_key,) + "get_sliding_sync_rooms_for_user_from_membership_snapshots", + (state_key,), ) self._attempt_to_invalidate_cache( @@ -376,7 +460,9 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache("get_room_encryption", (room_id,)) if (etype, state_key) in SLIDING_SYNC_RELEVANT_STATE_SET: - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) if relates_to: self._attempt_to_invalidate_cache( @@ -433,7 +519,9 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache( "_get_rooms_for_local_user_where_membership_is_inner", None ) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) self._attempt_to_invalidate_cache("did_forget", None) self._attempt_to_invalidate_cache("get_forgotten_rooms_for_user", None) self._attempt_to_invalidate_cache("get_references_for_event", None) @@ -446,6 +534,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache("_get_state_group_for_event", None) self._attempt_to_invalidate_cache("get_event_ordering", None) + self._attempt_to_invalidate_cache("get_metadata_for_event", (room_id,)) self._attempt_to_invalidate_cache("is_partial_state_event", None) self._attempt_to_invalidate_cache("_get_joined_profile_from_event_id", None) @@ -491,7 +580,9 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache( "get_current_hosts_in_room_ordered", (room_id,) ) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) self._attempt_to_invalidate_cache("did_forget", None) self._attempt_to_invalidate_cache("get_forgotten_rooms_for_user", None) self._attempt_to_invalidate_cache("_get_membership_from_event_id", None) diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index bf6cfcbfd9..cf7bc4ac69 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -20,10 +20,19 @@ # import logging -from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Dict, + List, + Mapping, + Optional, + Tuple, + TypedDict, + Union, + cast, +) import attr -from typing_extensions import TypedDict from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore @@ -412,6 +421,7 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke hs: "HomeServer", ): super().__init__(database, db_conn, hs) + self.server_name = hs.hostname if hs.config.redis.redis_enabled: # If we're using Redis, we can shift this update process off to @@ -425,7 +435,9 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke # (user_id, access_token, ip,) -> last_seen self.client_ip_last_seen = LruCache[Tuple[str, str, str], int]( - cache_name="client_ip_last_seen", max_size=50000 + cache_name="client_ip_last_seen", + server_name=self.server_name, + max_size=50000, ) if hs.config.worker.run_background_tasks and self.user_ips_max_age: @@ -641,9 +653,9 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke @wrap_as_background_process("update_client_ips") async def _update_client_ips_batch(self) -> None: - assert ( - self._update_on_this_worker - ), "This worker is not designated to update client IPs" + assert self._update_on_this_worker, ( + "This worker is not designated to update client IPs" + ) # If the DB pool has already terminated, don't try updating if not self.db_pool.is_running(): @@ -662,9 +674,9 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke txn: LoggingTransaction, to_update: Mapping[Tuple[str, str, str], Tuple[str, Optional[str], int]], ) -> None: - assert ( - self._update_on_this_worker - ), "This worker is not designated to update client IPs" + assert self._update_on_this_worker, ( + "This worker is not designated to update client IPs" + ) # Keys and values for the `user_ips` upsert. user_ips_keys = [] diff --git a/synapse/storage/databases/main/delayed_events.py b/synapse/storage/databases/main/delayed_events.py index 1616e30e22..c88682d55c 100644 --- a/synapse/storage/databases/main/delayed_events.py +++ b/synapse/storage/databases/main/delayed_events.py @@ -424,25 +424,37 @@ class DelayedEventsStore(SQLBaseStore): room_id: str, event_type: str, state_key: str, + not_from_localpart: str, ) -> Optional[Timestamp]: """ Cancels all matching delayed state events, i.e. remove them as long as they haven't been processed. + Args: + room_id: The room ID to match against. + event_type: The event type to match against. + state_key: The state key to match against. + not_from_localpart: The localpart of a user whose delayed events to not cancel. + If set to the empty string, any users' delayed events may be cancelled. + Returns: The send time of the next delayed event to be sent, if any. """ def cancel_delayed_state_events_txn( txn: LoggingTransaction, ) -> Optional[Timestamp]: - self.db_pool.simple_delete_txn( - txn, - table="delayed_events", - keyvalues={ - "room_id": room_id, - "event_type": event_type, - "state_key": state_key, - "is_processed": False, - }, + txn.execute( + """ + DELETE FROM delayed_events + WHERE room_id = ? AND event_type = ? AND state_key = ? + AND user_localpart <> ? + AND NOT is_processed + """, + ( + room_id, + event_type, + state_key, + not_from_localpart, + ), ) return self._get_next_delayed_event_send_ts_txn(txn) diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 0612b82b9b..c10e2d2611 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -42,6 +42,7 @@ from synapse.logging.opentracing import ( start_active_span, trace, ) +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.streams import ToDeviceStream from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage.database import ( @@ -51,10 +52,11 @@ from synapse.storage.database import ( make_in_list_sql_clause, ) from synapse.storage.util.id_generators import MultiWriterIdGenerator -from synapse.types import JsonDict -from synapse.util import json_encoder +from synapse.types import JsonDict, StrCollection +from synapse.util import Duration, json_encoder from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.caches.stream_change_cache import StreamChangeCache +from synapse.util.iterutils import batch_iter from synapse.util.stringutils import parse_and_validate_server_name if TYPE_CHECKING: @@ -63,6 +65,18 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# How long to keep messages in the device federation inbox before deleting them. +DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS = 7 * Duration.DAY_MS + +# How often to run the task to clean up old device_federation_inbox rows. +DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS = 5 * Duration.MINUTE_MS + +# Update name for the device federation inbox received timestamp index. +DEVICE_FEDERATION_INBOX_RECEIVED_INDEX_UPDATE = ( + "device_federation_inbox_received_ts_index" +) + + class DeviceInboxWorkerStore(SQLBaseStore): def __init__( self, @@ -80,6 +94,7 @@ class DeviceInboxWorkerStore(SQLBaseStore): Tuple[str, Optional[str]], int ] = ExpiringCache( cache_name="last_device_delete_cache", + server_name=self.server_name, clock=self._clock, max_len=10000, expiry_ms=30 * 60 * 1000, @@ -94,6 +109,7 @@ class DeviceInboxWorkerStore(SQLBaseStore): db=database, notifier=hs.get_replication_notifier(), stream_name="to_device", + server_name=self.server_name, instance_name=self._instance_name, tables=[ ("device_inbox", "instance_name", "stream_id"), @@ -113,8 +129,9 @@ class DeviceInboxWorkerStore(SQLBaseStore): limit=1000, ) self._device_inbox_stream_cache = StreamChangeCache( - "DeviceInboxStreamChangeCache", - min_device_inbox_id, + name="DeviceInboxStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_inbox_id, prefilled_cache=device_inbox_prefill, ) @@ -129,11 +146,21 @@ class DeviceInboxWorkerStore(SQLBaseStore): limit=1000, ) self._device_federation_outbox_stream_cache = StreamChangeCache( - "DeviceFederationOutboxStreamChangeCache", - min_device_outbox_id, + name="DeviceFederationOutboxStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_outbox_id, prefilled_cache=device_outbox_prefill, ) + if hs.config.worker.run_background_tasks: + self._clock.looping_call( + run_as_background_process, + DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS, + "_delete_old_federation_inbox_rows", + self.server_name, + self._delete_old_federation_inbox_rows, + ) + def process_replication_rows( self, stream_name: str, @@ -200,9 +227,9 @@ class DeviceInboxWorkerStore(SQLBaseStore): to_stream_id=to_stream_id, ) - assert ( - last_processed_stream_id == to_stream_id - ), "Expected _get_device_messages to process all to-device messages up to `to_stream_id`" + assert last_processed_stream_id == to_stream_id, ( + "Expected _get_device_messages to process all to-device messages up to `to_stream_id`" + ) return user_id_device_id_to_messages @@ -960,6 +987,86 @@ class DeviceInboxWorkerStore(SQLBaseStore): ], ) + async def _delete_old_federation_inbox_rows(self, batch_size: int = 1000) -> None: + """Delete old rows from the device_federation_inbox table.""" + + # We wait until we have the index on `received_ts`, otherwise the query + # will take a very long time. + if not await self.db_pool.updates.has_completed_background_update( + DEVICE_FEDERATION_INBOX_RECEIVED_INDEX_UPDATE + ): + return + + def _delete_old_federation_inbox_rows_txn(txn: LoggingTransaction) -> bool: + # We delete at most 100 rows that are older than + # DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS + delete_before_ts = ( + self._clock.time_msec() - DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS + ) + sql = """ + WITH to_delete AS ( + SELECT origin, message_id + FROM device_federation_inbox + WHERE received_ts < ? + ORDER BY received_ts ASC + LIMIT ? + ) + DELETE FROM device_federation_inbox + WHERE + (origin, message_id) IN ( + SELECT origin, message_id FROM to_delete + ) + """ + txn.execute(sql, (delete_before_ts, batch_size)) + return txn.rowcount < batch_size + + while True: + finished = await self.db_pool.runInteraction( + "_delete_old_federation_inbox_rows", + _delete_old_federation_inbox_rows_txn, + db_autocommit=True, # We don't need to run in a transaction + ) + if finished: + return + + # We sleep a bit so that we don't hammer the database in a tight + # loop first time we run this. + await self._clock.sleep(1) + + async def get_devices_with_messages( + self, user_id: str, device_ids: StrCollection + ) -> StrCollection: + """Get the matching device IDs that have messages in the device inbox.""" + + def get_devices_with_messages_txn( + txn: LoggingTransaction, + batch_device_ids: StrCollection, + ) -> StrCollection: + clause, args = make_in_list_sql_clause( + self.database_engine, "device_id", batch_device_ids + ) + sql = f""" + SELECT DISTINCT device_id FROM device_inbox + WHERE {clause} AND user_id = ? + """ + args.append(user_id) + txn.execute(sql, args) + return {row[0] for row in txn} + + results: Set[str] = set() + for batch_device_ids in batch_iter(device_ids, 1000): + batch_results = await self.db_pool.runInteraction( + "get_devices_with_messages", + get_devices_with_messages_txn, + batch_device_ids, + # We don't need to run in a transaction as it's a single query + db_autocommit=True, + ) + + results.update(batch_results) + + return results + class DeviceInboxBackgroundUpdateStore(SQLBaseStore): DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop" @@ -995,6 +1102,13 @@ class DeviceInboxBackgroundUpdateStore(SQLBaseStore): self._cleanup_device_federation_outbox, ) + self.db_pool.updates.register_background_index_update( + update_name=DEVICE_FEDERATION_INBOX_RECEIVED_INDEX_UPDATE, + index_name="device_federation_inbox_received_ts_index", + table="device_federation_inbox", + columns=["received_ts"], + ) + async def _background_drop_index_device_inbox( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 8088943253..a28cc40a95 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -35,7 +35,6 @@ from typing import ( ) from canonicaljson import encode_canonical_json -from typing_extensions import Literal from synapse.api.constants import EduTypes from synapse.api.errors import Codes, StoreError @@ -61,12 +60,12 @@ from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import ( JsonDict, JsonMapping, + MultiWriterStreamToken, StrCollection, get_verify_key_from_cross_signing_key, ) from synapse.util import json_decoder, json_encoder from synapse.util.caches.descriptors import cached, cachedList -from synapse.util.caches.lrucache import LruCache from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.cancellation import cancellable from synapse.util.iterutils import batch_iter @@ -86,6 +85,9 @@ BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES = "remove_dup_outbound_pokes" class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): + _device_list_id_gen: MultiWriterIdGenerator + _instance_name: str + def __init__( self, database: DatabasePool, @@ -101,6 +103,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): db=database, notifier=hs.get_replication_notifier(), stream_name="device_lists_stream", + server_name=self.server_name, instance_name=self._instance_name, tables=[ ("device_lists_stream", "instance_name", "stream_id"), @@ -115,7 +118,11 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): ), ], sequence_name="device_lists_sequence", - writers=["master"], + writers=hs.config.worker.writers.device_lists, + ) + + self._is_device_list_writer = ( + self._instance_name in hs.config.worker.writers.device_lists ) device_list_max = self._device_list_id_gen.get_current_token() @@ -128,8 +135,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=10000, ) self._device_list_stream_cache = StreamChangeCache( - "DeviceListStreamChangeCache", - min_device_list_id, + name="DeviceListStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_list_id, prefilled_cache=device_list_prefill, ) @@ -142,8 +150,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=10000, ) self._device_list_room_stream_cache = StreamChangeCache( - "DeviceListRoomStreamChangeCache", - min_device_list_room_id, + name="DeviceListRoomStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_list_room_id, prefilled_cache=device_list_room_prefill, ) @@ -159,8 +168,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=1000, ) self._user_signature_stream_cache = StreamChangeCache( - "UserSignatureStreamChangeCache", - user_signature_stream_list_id, + name="UserSignatureStreamChangeCache", + server_name=self.server_name, + current_stream_pos=user_signature_stream_list_id, prefilled_cache=user_signature_stream_prefill, ) @@ -178,8 +188,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=10000, ) self._device_list_federation_stream_cache = StreamChangeCache( - "DeviceListFederationStreamChangeCache", - device_list_federation_list_id, + name="DeviceListFederationStreamChangeCache", + server_name=self.server_name, + current_stream_pos=device_list_federation_list_id, prefilled_cache=device_list_federation_prefill, ) @@ -240,8 +251,8 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): for room_id in room_ids: self._device_list_room_stream_cache.entity_has_changed(room_id, token) - def get_device_stream_token(self) -> int: - return self._device_list_id_gen.get_current_token() + def get_device_stream_token(self) -> MultiWriterStreamToken: + return MultiWriterStreamToken.from_generator(self._device_list_id_gen) def get_device_stream_id_generator(self) -> MultiWriterIdGenerator: return self._device_list_id_gen @@ -282,9 +293,187 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): "count_devices_by_users", count_devices_by_users_txn, user_ids ) + async def store_device( + self, + user_id: str, + device_id: str, + initial_device_display_name: Optional[str], + auth_provider_id: Optional[str] = None, + auth_provider_session_id: Optional[str] = None, + ) -> bool: + """Ensure the given device is known; add it to the store if not + + Args: + user_id: id of user associated with the device + device_id: id of device + initial_device_display_name: initial displayname of the device. + Ignored if device exists. + auth_provider_id: The SSO IdP the user used, if any. + auth_provider_session_id: The session ID (sid) got from a OIDC login. + + Returns: + Whether the device was inserted or an existing device existed with that ID. + + Raises: + StoreError: if the device is already in use + """ + try: + inserted = await self.db_pool.simple_upsert( + "devices", + keyvalues={ + "user_id": user_id, + "device_id": device_id, + }, + values={}, + insertion_values={ + "display_name": initial_device_display_name, + "hidden": False, + }, + desc="store_device", + ) + await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) + + if not inserted: + # if the device already exists, check if it's a real device, or + # if the device ID is reserved by something else + hidden = await self.db_pool.simple_select_one_onecol( + "devices", + keyvalues={"user_id": user_id, "device_id": device_id}, + retcol="hidden", + ) + if hidden: + raise StoreError(400, "The device ID is in use", Codes.FORBIDDEN) + + if auth_provider_id and auth_provider_session_id: + await self.db_pool.simple_insert( + "device_auth_providers", + values={ + "user_id": user_id, + "device_id": device_id, + "auth_provider_id": auth_provider_id, + "auth_provider_session_id": auth_provider_session_id, + }, + desc="store_device_auth_provider", + ) + + return inserted + except StoreError: + raise + except Exception as e: + logger.error( + "store_device with device_id=%s(%r) user_id=%s(%r)" + " display_name=%s(%r) failed: %s", + type(device_id).__name__, + device_id, + type(user_id).__name__, + user_id, + type(initial_device_display_name).__name__, + initial_device_display_name, + e, + ) + raise StoreError(500, "Problem storing device.") + + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: + """Deletes several devices. + + Args: + user_id: The ID of the user which owns the devices + device_ids: The IDs of the devices to delete + """ + + def _delete_devices_txn(txn: LoggingTransaction, device_ids: List[str]) -> None: + self.db_pool.simple_delete_many_txn( + txn, + table="devices", + column="device_id", + values=device_ids, + keyvalues={"user_id": user_id, "hidden": False}, + ) + + self.db_pool.simple_delete_many_txn( + txn, + table="device_auth_providers", + column="device_id", + values=device_ids, + keyvalues={"user_id": user_id}, + ) + + # Also delete associated e2e keys. + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_device_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_one_time_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="dehydrated_devices", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_fallback_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + + # We're bulk deleting potentially many devices at once, so + # let's not invalidate the cache for each device individually. + # Instead, we will invalidate the cache for the user as a whole. + self._invalidate_cache_and_stream(txn, self.get_device, (user_id,)) + self._invalidate_cache_and_stream( + txn, self.count_e2e_one_time_keys, (user_id,) + ) + self._invalidate_cache_and_stream( + txn, self.get_e2e_unused_fallback_key_types, (user_id,) + ) + + for batch in batch_iter(device_ids, 1000): + await self.db_pool.runInteraction( + "delete_devices", _delete_devices_txn, batch + ) + + async def update_device( + self, user_id: str, device_id: str, new_display_name: Optional[str] = None + ) -> None: + """Update a device. Only updates the device if it is not marked as + hidden. + + Args: + user_id: The ID of the user which owns the device + device_id: The ID of the device to update + new_display_name: new displayname for device; None to leave unchanged + Raises: + StoreError: if the device is not found + """ + updates = {} + if new_display_name is not None: + updates["display_name"] = new_display_name + if not updates: + return None + await self.db_pool.simple_update_one( + table="devices", + keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False}, + updatevalues=updates, + desc="update_device", + ) + await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) + + @cached(tree=True) async def get_device( self, user_id: str, device_id: str - ) -> Optional[Dict[str, Any]]: + ) -> Optional[Mapping[str, Any]]: """Retrieve a device. Only returns devices that are not marked as hidden. @@ -374,7 +563,11 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): - The list of updates, where each update is a pair of EDU type and EDU contents. """ - now_stream_id = self.get_device_stream_token() + # Here, we don't use the individual instances positions, as we only + # record the last stream position we've sent to a destination. This + # means we have to wait for all the writers to catch up before sending + # device list updates, which is fine. + now_stream_id = self.get_device_stream_token().stream if from_stream_id == now_stream_id: return now_stream_id, [] @@ -751,6 +944,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): Returns: The new stream ID. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") async with self._device_list_id_gen.get_next() as stream_id: await self.db_pool.runInteraction( @@ -873,8 +1069,8 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): @cancellable async def get_all_devices_changed( self, - from_key: int, - to_key: int, + from_key: MultiWriterStreamToken, + to_key: MultiWriterStreamToken, ) -> Set[str]: """Get all users whose devices have changed in the given range. @@ -889,7 +1085,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): (exclusive) until `to_key` (inclusive). """ - result = self._device_list_stream_cache.get_all_entities_changed(from_key) + result = self._device_list_stream_cache.get_all_entities_changed( + from_key.stream + ) if result.hit: # We know which users might have changed devices. @@ -905,24 +1103,34 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): # If the cache didn't tell us anything, we just need to query the full # range. sql = """ - SELECT DISTINCT user_id FROM device_lists_stream + SELECT user_id, stream_id, instance_name + FROM device_lists_stream WHERE ? < stream_id AND stream_id <= ? """ rows = await self.db_pool.execute( "get_all_devices_changed", sql, - from_key, - to_key, + from_key.stream, + to_key.get_max_stream_pos(), ) - return {u for (u,) in rows} + return { + user_id + for (user_id, stream_id, instance_name) in rows + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_key, + high=to_key, + instance_name=instance_name, + pos=stream_id, + ) + } @cancellable async def get_users_whose_devices_changed( self, - from_key: int, + from_key: MultiWriterStreamToken, user_ids: Collection[str], - to_key: Optional[int] = None, + to_key: Optional[MultiWriterStreamToken] = None, ) -> Set[str]: """Get set of users whose devices have changed since `from_key` that are in the given list of user_ids. @@ -942,7 +1150,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): # Get set of users who *may* have changed. Users not in the returned # list have definitely not changed. user_ids_to_check = self._device_list_stream_cache.get_entities_changed( - user_ids, from_key + user_ids, from_key.stream ) # If an empty set was returned, there's nothing to do. @@ -950,11 +1158,16 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): return set() if to_key is None: - to_key = self._device_list_id_gen.get_current_token() + to_key = self.get_device_stream_token() - def _get_users_whose_devices_changed_txn(txn: LoggingTransaction) -> Set[str]: + def _get_users_whose_devices_changed_txn( + txn: LoggingTransaction, + from_key: MultiWriterStreamToken, + to_key: MultiWriterStreamToken, + ) -> Set[str]: sql = """ - SELECT DISTINCT user_id FROM device_lists_stream + SELECT user_id, stream_id, instance_name + FROM device_lists_stream WHERE ? < stream_id AND stream_id <= ? AND %s """ @@ -965,17 +1178,32 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): clause, args = make_in_list_sql_clause( txn.database_engine, "user_id", chunk ) - txn.execute(sql % (clause,), [from_key, to_key] + args) - changes.update(user_id for (user_id,) in txn) + txn.execute( + sql % (clause,), + [from_key.stream, to_key.get_max_stream_pos()] + args, + ) + changes.update( + user_id + for (user_id, stream_id, instance_name) in txn + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_key, + high=to_key, + instance_name=instance_name, + pos=stream_id, + ) + ) return changes return await self.db_pool.runInteraction( - "get_users_whose_devices_changed", _get_users_whose_devices_changed_txn + "get_users_whose_devices_changed", + _get_users_whose_devices_changed_txn, + from_key, + to_key, ) async def get_users_whose_signatures_changed( - self, user_id: str, from_key: int + self, user_id: str, from_key: MultiWriterStreamToken ) -> Set[str]: """Get the users who have new cross-signing signatures made by `user_id` since `from_key`. @@ -988,18 +1216,31 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): A set of user IDs with updated signatures. """ - if self._user_signature_stream_cache.has_entity_changed(user_id, from_key): - sql = """ - SELECT DISTINCT user_ids FROM user_signature_stream - WHERE from_user_id = ? AND stream_id > ? - """ - rows = await self.db_pool.execute( - "get_users_whose_signatures_changed", sql, user_id, from_key - ) - return {user for row in rows for user in db_to_json(row[0])} - else: + if not self._user_signature_stream_cache.has_entity_changed( + user_id, from_key.stream + ): return set() + sql = """ + SELECT user_ids, stream_id, instance_name + FROM user_signature_stream + WHERE from_user_id = ? AND stream_id > ? + """ + rows = await self.db_pool.execute( + "get_users_whose_signatures_changed", sql, user_id, from_key.stream + ) + return { + user + for (user_ids, stream_id, instance_name) in rows + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_key, + high=None, + instance_name=instance_name, + pos=stream_id, + ) + for user in db_to_json(user_ids) + } + async def get_all_device_list_changes_for_remotes( self, instance_name: str, last_id: int, current_id: int, limit: int ) -> Tuple[List[Tuple[int, tuple]], int, bool]: @@ -1091,7 +1332,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): ), ) - results: Dict[str, Optional[str]] = {user_id: None for user_id in user_ids} + results: Dict[str, Optional[str]] = dict.fromkeys(user_ids) results.update(rows) return results @@ -1253,9 +1494,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): if keys: device_keys = keys.get("device_keys", None) if device_keys: - # Type ignore - this function is defined on EndToEndKeyStore which we do - # have access to due to hs.get_datastore() "magic" - self._set_e2e_device_keys_txn( # type: ignore[attr-defined] + self._set_e2e_device_keys_txn( txn, user_id, device_id, time, device_keys ) @@ -1485,7 +1724,10 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): @cancellable async def get_device_list_changes_in_rooms( - self, room_ids: Collection[str], from_id: int, to_id: int + self, + room_ids: Collection[str], + from_token: MultiWriterStreamToken, + to_token: MultiWriterStreamToken, ) -> Optional[Set[str]]: """Return the set of users whose devices have changed in the given rooms since the given stream ID. @@ -1498,41 +1740,50 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): min_stream_id = await self._get_min_device_lists_changes_in_room() - if min_stream_id > from_id: + # Return early if there are no rows to process in device_lists_changes_in_room + if min_stream_id > from_token.stream: return None changed_room_ids = self._device_list_room_stream_cache.get_entities_changed( - room_ids, from_id + room_ids, from_token.stream ) if not changed_room_ids: return set() sql = """ - SELECT DISTINCT user_id FROM device_lists_changes_in_room + SELECT user_id, stream_id, instance_name + FROM device_lists_changes_in_room WHERE {clause} AND stream_id > ? AND stream_id <= ? """ def _get_device_list_changes_in_rooms_txn( txn: LoggingTransaction, - clause: str, - args: List[Any], + chunk: list[str], ) -> Set[str]: - txn.execute(sql.format(clause=clause), args) - return {user_id for (user_id,) in txn} - - changes = set() - for chunk in batch_iter(changed_room_ids, 1000): clause, args = make_in_list_sql_clause( self.database_engine, "room_id", chunk ) - args.append(from_id) - args.append(to_id) + args.append(from_token.stream) + args.append(to_token.get_max_stream_pos()) + txn.execute(sql.format(clause=clause), args) + return { + user_id + for (user_id, stream_id, instance_name) in txn + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_token, + high=to_token, + instance_name=instance_name, + pos=stream_id, + ) + } + + changes = set() + for chunk in batch_iter(changed_room_ids, 1000): changes |= await self.db_pool.runInteraction( "get_device_list_changes_in_rooms", _get_device_list_changes_in_rooms_txn, - clause, - args, + chunk, ) return changes @@ -1600,322 +1851,6 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): desc="get_destinations_for_device", ) - -class DeviceBackgroundUpdateStore(SQLBaseStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - self._instance_name = hs.get_instance_name() - - self.db_pool.updates.register_background_index_update( - "device_lists_stream_idx", - index_name="device_lists_stream_user_id", - table="device_lists_stream", - columns=["user_id", "device_id"], - ) - - # create a unique index on device_lists_remote_cache - self.db_pool.updates.register_background_index_update( - "device_lists_remote_cache_unique_idx", - index_name="device_lists_remote_cache_unique_id", - table="device_lists_remote_cache", - columns=["user_id", "device_id"], - unique=True, - ) - - # And one on device_lists_remote_extremeties - self.db_pool.updates.register_background_index_update( - "device_lists_remote_extremeties_unique_idx", - index_name="device_lists_remote_extremeties_unique_idx", - table="device_lists_remote_extremeties", - columns=["user_id"], - unique=True, - ) - - # once they complete, we can remove the old non-unique indexes. - self.db_pool.updates.register_background_update_handler( - DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES, - self._drop_device_list_streams_non_unique_indexes, - ) - - # clear out duplicate device list outbound pokes - self.db_pool.updates.register_background_update_handler( - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, - self._remove_duplicate_outbound_pokes, - ) - - self.db_pool.updates.register_background_index_update( - "device_lists_changes_in_room_by_room_index", - index_name="device_lists_changes_in_room_by_room_idx", - table="device_lists_changes_in_room", - columns=["room_id", "stream_id"], - ) - - async def _drop_device_list_streams_non_unique_indexes( - self, progress: JsonDict, batch_size: int - ) -> int: - def f(conn: LoggingDatabaseConnection) -> None: - txn = conn.cursor() - txn.execute("DROP INDEX IF EXISTS device_lists_remote_cache_id") - txn.execute("DROP INDEX IF EXISTS device_lists_remote_extremeties_id") - txn.close() - - await self.db_pool.runWithConnection(f) - await self.db_pool.updates._end_background_update( - DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES - ) - return 1 - - async def _remove_duplicate_outbound_pokes( - self, progress: JsonDict, batch_size: int - ) -> int: - # for some reason, we have accumulated duplicate entries in - # device_lists_outbound_pokes, which makes prune_outbound_device_list_pokes less - # efficient. - # - # For each duplicate, we delete all the existing rows and put one back. - - last_row = progress.get( - "last_row", - {"stream_id": 0, "destination": "", "user_id": "", "device_id": ""}, - ) - - def _txn(txn: LoggingTransaction) -> int: - clause, args = make_tuple_comparison_clause( - [ - ("stream_id", last_row["stream_id"]), - ("destination", last_row["destination"]), - ("user_id", last_row["user_id"]), - ("device_id", last_row["device_id"]), - ] - ) - sql = f""" - SELECT stream_id, destination, user_id, device_id, MAX(ts) AS ts - FROM device_lists_outbound_pokes - WHERE {clause} - GROUP BY stream_id, destination, user_id, device_id - HAVING count(*) > 1 - ORDER BY stream_id, destination, user_id, device_id - LIMIT ? - """ - txn.execute(sql, args + [batch_size]) - rows = txn.fetchall() - - stream_id, destination, user_id, device_id = None, None, None, None - for stream_id, destination, user_id, device_id, _ in rows: - self.db_pool.simple_delete_txn( - txn, - "device_lists_outbound_pokes", - { - "stream_id": stream_id, - "destination": destination, - "user_id": user_id, - "device_id": device_id, - }, - ) - - self.db_pool.simple_insert_txn( - txn, - "device_lists_outbound_pokes", - { - "stream_id": stream_id, - "instance_name": self._instance_name, - "destination": destination, - "user_id": user_id, - "device_id": device_id, - "sent": False, - }, - ) - - if rows: - self.db_pool.updates._background_update_progress_txn( - txn, - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, - { - "last_row": { - "stream_id": stream_id, - "destination": destination, - "user_id": user_id, - "device_id": device_id, - } - }, - ) - - return len(rows) - - rows = await self.db_pool.runInteraction( - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, _txn - ) - - if not rows: - await self.db_pool.updates._end_background_update( - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES - ) - - return rows - - -class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - # Map of (user_id, device_id) -> bool. If there is an entry that implies - # the device exists. - self.device_id_exists_cache: LruCache[Tuple[str, str], Literal[True]] = ( - LruCache(cache_name="device_id_exists", max_size=10000) - ) - - async def store_device( - self, - user_id: str, - device_id: str, - initial_device_display_name: Optional[str], - auth_provider_id: Optional[str] = None, - auth_provider_session_id: Optional[str] = None, - ) -> bool: - """Ensure the given device is known; add it to the store if not - - Args: - user_id: id of user associated with the device - device_id: id of device - initial_device_display_name: initial displayname of the device. - Ignored if device exists. - auth_provider_id: The SSO IdP the user used, if any. - auth_provider_session_id: The session ID (sid) got from a OIDC login. - - Returns: - Whether the device was inserted or an existing device existed with that ID. - - Raises: - StoreError: if the device is already in use - """ - key = (user_id, device_id) - if self.device_id_exists_cache.get(key, None): - return False - - try: - inserted = await self.db_pool.simple_upsert( - "devices", - keyvalues={ - "user_id": user_id, - "device_id": device_id, - }, - values={}, - insertion_values={ - "display_name": initial_device_display_name, - "hidden": False, - }, - desc="store_device", - ) - if not inserted: - # if the device already exists, check if it's a real device, or - # if the device ID is reserved by something else - hidden = await self.db_pool.simple_select_one_onecol( - "devices", - keyvalues={"user_id": user_id, "device_id": device_id}, - retcol="hidden", - ) - if hidden: - raise StoreError(400, "The device ID is in use", Codes.FORBIDDEN) - - if auth_provider_id and auth_provider_session_id: - await self.db_pool.simple_insert( - "device_auth_providers", - values={ - "user_id": user_id, - "device_id": device_id, - "auth_provider_id": auth_provider_id, - "auth_provider_session_id": auth_provider_session_id, - }, - desc="store_device_auth_provider", - ) - - self.device_id_exists_cache.set(key, True) - return inserted - except StoreError: - raise - except Exception as e: - logger.error( - "store_device with device_id=%s(%r) user_id=%s(%r)" - " display_name=%s(%r) failed: %s", - type(device_id).__name__, - device_id, - type(user_id).__name__, - user_id, - type(initial_device_display_name).__name__, - initial_device_display_name, - e, - ) - raise StoreError(500, "Problem storing device.") - - async def delete_devices(self, user_id: str, device_ids: List[str]) -> None: - """Deletes several devices. - - Args: - user_id: The ID of the user which owns the devices - device_ids: The IDs of the devices to delete - """ - - def _delete_devices_txn(txn: LoggingTransaction, device_ids: List[str]) -> None: - self.db_pool.simple_delete_many_txn( - txn, - table="devices", - column="device_id", - values=device_ids, - keyvalues={"user_id": user_id, "hidden": False}, - ) - - self.db_pool.simple_delete_many_txn( - txn, - table="device_auth_providers", - column="device_id", - values=device_ids, - keyvalues={"user_id": user_id}, - ) - - for batch in batch_iter(device_ids, 100): - await self.db_pool.runInteraction( - "delete_devices", _delete_devices_txn, batch - ) - - for device_id in device_ids: - self.device_id_exists_cache.invalidate((user_id, device_id)) - - async def update_device( - self, user_id: str, device_id: str, new_display_name: Optional[str] = None - ) -> None: - """Update a device. Only updates the device if it is not marked as - hidden. - - Args: - user_id: The ID of the user which owns the device - device_id: The ID of the device to update - new_display_name: new displayname for device; None to leave unchanged - Raises: - StoreError: if the device is not found - """ - updates = {} - if new_display_name is not None: - updates["display_name"] = new_display_name - if not updates: - return None - await self.db_pool.simple_update_one( - table="devices", - keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False}, - updatevalues=updates, - desc="update_device", - ) - async def update_remote_device_list_cache_entry( self, user_id: str, device_id: str, content: JsonDict, stream_id: str ) -> None: @@ -1954,8 +1889,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): table="device_lists_remote_cache", keyvalues={"user_id": user_id, "device_id": device_id}, ) - - txn.call_after(self.device_id_exists_cache.invalidate, (user_id, device_id)) else: self.db_pool.simple_upsert_txn( txn, @@ -2048,40 +1981,50 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): The maximum stream ID of device list updates that were added to the database, or None if no updates were added. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") + if not device_ids: return None context = get_active_span_text_map() def add_device_changes_txn( - txn: LoggingTransaction, stream_ids: List[int] - ) -> None: + txn: LoggingTransaction, + batch_device_ids: StrCollection, + ) -> int: + stream_ids = self._device_list_id_gen.get_next_mult_txn( + txn, len(device_ids) + ) + self._add_device_change_to_stream_txn( txn, user_id, - device_ids, + batch_device_ids, stream_ids, ) self._add_device_outbound_room_poke_txn( txn, user_id, - device_ids, + batch_device_ids, room_ids, stream_ids, context, ) - async with self._device_list_id_gen.get_next_mult( - len(device_ids) - ) as stream_ids: - await self.db_pool.runInteraction( + return stream_ids[-1] + + last_stream_id: Optional[int] = None + for batch_device_ids in batch_iter(device_ids, 1000): + last_stream_id = await self.db_pool.runInteraction( "add_device_change_to_stream", add_device_changes_txn, - stream_ids, + batch_device_ids, ) - return stream_ids[-1] + return last_stream_id def _add_device_change_to_stream_txn( self, @@ -2280,7 +2223,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): A list of user ID, device ID, room ID, stream ID and optional opentracing context, in order of ascending (stream ID, room ID). """ - sql = """ SELECT user_id, device_id, room_id, stream_id, opentracing_context FROM device_lists_changes_in_room @@ -2333,6 +2275,10 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): """Queue the device update to be sent to the given set of hosts, calculated from the room ID. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") + if not hosts: return @@ -2361,6 +2307,9 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): """Add a device list update to the table tracking remote device list updates during partial joins. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") async with self._device_list_id_gen.get_next() as stream_id: await self.db_pool.simple_upsert( @@ -2383,6 +2332,14 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): the room. """ + # The device list stream is a multi-writer stream, but when we partially + # join a room, we only record the minimum stream ID. This means that we + # may be returning a device update that was already sent through + # federation here in case of concurrent writes. This is absolutely fine, + # sending a device update multiple times through federation is safe + + # FIXME: record the full multi-writer stream token with individual + # writer positions at the time of the join to avoid this min_device_stream_id = await self.db_pool.simple_select_one_onecol( table="partial_state_rooms", keyvalues={ @@ -2453,3 +2410,176 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): }, desc="set_device_change_last_converted_pos", ) + + +class DeviceBackgroundUpdateStore(SQLBaseStore): + _instance_name: str + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self._instance_name = hs.get_instance_name() + + self.db_pool.updates.register_background_index_update( + "device_lists_stream_idx", + index_name="device_lists_stream_user_id", + table="device_lists_stream", + columns=["user_id", "device_id"], + ) + + # create a unique index on device_lists_remote_cache + self.db_pool.updates.register_background_index_update( + "device_lists_remote_cache_unique_idx", + index_name="device_lists_remote_cache_unique_id", + table="device_lists_remote_cache", + columns=["user_id", "device_id"], + unique=True, + ) + + # And one on device_lists_remote_extremeties + self.db_pool.updates.register_background_index_update( + "device_lists_remote_extremeties_unique_idx", + index_name="device_lists_remote_extremeties_unique_idx", + table="device_lists_remote_extremeties", + columns=["user_id"], + unique=True, + ) + + # once they complete, we can remove the old non-unique indexes. + self.db_pool.updates.register_background_update_handler( + DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES, + self._drop_device_list_streams_non_unique_indexes, + ) + + # clear out duplicate device list outbound pokes + self.db_pool.updates.register_background_update_handler( + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, + self._remove_duplicate_outbound_pokes, + ) + + self.db_pool.updates.register_background_index_update( + "device_lists_changes_in_room_by_room_index", + index_name="device_lists_changes_in_room_by_room_idx", + table="device_lists_changes_in_room", + columns=["room_id", "stream_id"], + ) + + async def _drop_device_list_streams_non_unique_indexes( + self, progress: JsonDict, batch_size: int + ) -> int: + def f(conn: LoggingDatabaseConnection) -> None: + txn = conn.cursor() + txn.execute("DROP INDEX IF EXISTS device_lists_remote_cache_id") + txn.execute("DROP INDEX IF EXISTS device_lists_remote_extremeties_id") + txn.close() + + await self.db_pool.runWithConnection(f) + await self.db_pool.updates._end_background_update( + DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES + ) + return 1 + + async def _remove_duplicate_outbound_pokes( + self, progress: JsonDict, batch_size: int + ) -> int: + # for some reason, we have accumulated duplicate entries in + # device_lists_outbound_pokes, which makes prune_outbound_device_list_pokes less + # efficient. + # + # For each duplicate, we delete all the existing rows and put one back. + + last_row = progress.get( + "last_row", + {"stream_id": 0, "destination": "", "user_id": "", "device_id": ""}, + ) + + def _txn(txn: LoggingTransaction) -> int: + clause, args = make_tuple_comparison_clause( + [ + ("stream_id", last_row["stream_id"]), + ("destination", last_row["destination"]), + ("user_id", last_row["user_id"]), + ("device_id", last_row["device_id"]), + ] + ) + sql = f""" + SELECT stream_id, destination, user_id, device_id, MAX(ts) AS ts + FROM device_lists_outbound_pokes + WHERE {clause} + GROUP BY stream_id, destination, user_id, device_id + HAVING count(*) > 1 + ORDER BY stream_id, destination, user_id, device_id + LIMIT ? + """ + txn.execute(sql, args + [batch_size]) + rows = txn.fetchall() + + stream_id, destination, user_id, device_id = None, None, None, None + for stream_id, destination, user_id, device_id, _ in rows: + self.db_pool.simple_delete_txn( + txn, + "device_lists_outbound_pokes", + { + "stream_id": stream_id, + "destination": destination, + "user_id": user_id, + "device_id": device_id, + }, + ) + + self.db_pool.simple_insert_txn( + txn, + "device_lists_outbound_pokes", + { + "stream_id": stream_id, + "instance_name": self._instance_name, + "destination": destination, + "user_id": user_id, + "device_id": device_id, + "sent": False, + }, + ) + + if rows: + self.db_pool.updates._background_update_progress_txn( + txn, + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, + { + "last_row": { + "stream_id": stream_id, + "destination": destination, + "user_id": user_id, + "device_id": device_id, + } + }, + ) + + return len(rows) + + rows = await self.db_pool.runInteraction( + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, _txn + ) + + if not rows: + await self.db_pool.updates._end_background_update( + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES + ) + + return rows + + +class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): + _instance_name: str + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py index c2c93e12d9..904ae5cb58 100644 --- a/synapse/storage/databases/main/e2e_room_keys.py +++ b/synapse/storage/databases/main/e2e_room_keys.py @@ -19,9 +19,18 @@ # # -from typing import TYPE_CHECKING, Dict, Iterable, List, Mapping, Optional, Tuple, cast - -from typing_extensions import Literal, TypedDict +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + Literal, + Mapping, + Optional, + Tuple, + TypedDict, + cast, +) from synapse.api.errors import StoreError from synapse.logging.opentracing import log_kv, trace @@ -510,19 +519,16 @@ class EndToEndRoomKeyStore(EndToEndRoomKeyBackgroundStore): # it isn't there. raise StoreError(404, "No backup with that version exists") - row = cast( - Tuple[int, str, str, Optional[int]], - self.db_pool.simple_select_one_txn( - txn, - table="e2e_room_keys_versions", - keyvalues={ - "user_id": user_id, - "version": this_version, - "deleted": 0, - }, - retcols=("version", "algorithm", "auth_data", "etag"), - allow_none=False, - ), + row = self.db_pool.simple_select_one_txn( + txn, + table="e2e_room_keys_versions", + keyvalues={ + "user_id": user_id, + "version": this_version, + "deleted": 0, + }, + retcols=("version", "algorithm", "auth_data", "etag"), + allow_none=False, ) return { "auth_data": db_to_json(row[2]), diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 3bb8fccb5e..17ccefe6b5 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -20,6 +20,7 @@ # # import abc +import json from typing import ( TYPE_CHECKING, Any, @@ -27,6 +28,7 @@ from typing import ( Dict, Iterable, List, + Literal, Mapping, Optional, Sequence, @@ -39,7 +41,6 @@ from typing import ( import attr from canonicaljson import encode_canonical_json -from typing_extensions import Literal from synapse.api.constants import DeviceKeyAlgorithms from synapse.appservice import ( @@ -59,7 +60,7 @@ from synapse.storage.database import ( from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.engines import PostgresEngine from synapse.storage.util.id_generators import MultiWriterIdGenerator -from synapse.types import JsonDict, JsonMapping +from synapse.types import JsonDict, JsonMapping, MultiWriterStreamToken from synapse.util import json_decoder, json_encoder from synapse.util.caches.descriptors import cached, cachedList from synapse.util.cancellation import cancellable @@ -120,6 +121,21 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker self.hs.config.federation.allow_device_name_lookup_over_federation ) + self._cross_signing_id_gen = MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name="e2e_cross_signing_keys", + server_name=self.server_name, + instance_name=self._instance_name, + tables=[ + ("e2e_cross_signing_keys", "instance_name", "stream_id"), + ], + sequence_name="e2e_cross_signing_keys_sequence", + # No one reads the stream positions, so we're allowed to have an empty list of writers + writers=[], + ) + def process_replication_rows( self, stream_name: str, @@ -145,7 +161,12 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker Returns: (stream_id, devices) """ - now_stream_id = self.get_device_stream_token() + # Here, we don't use the individual instances positions, as we *need* to + # give out the stream_id as an integer in the federation API. + # This means that we'll potentially return the same data twice with a + # different stream_id, and invalidate cache more often than necessary, + # which is fine overall. + now_stream_id = self.get_device_stream_token().stream # We need to be careful with the caching here, as we need to always # return *all* persisted devices, however there may be a lag between a @@ -164,8 +185,10 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker # have to check for potential invalidations after the # `now_stream_id`. sql = """ - SELECT user_id FROM device_lists_stream + SELECT 1 + FROM device_lists_stream WHERE stream_id >= ? AND user_id = ? + LIMIT 1 """ rows = await self.db_pool.execute( "get_e2e_device_keys_for_federation_query_check", @@ -332,15 +355,17 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker ) for batch in batch_iter(signature_query, 50): - cross_sigs_result = await self.db_pool.runInteraction( - "get_e2e_cross_signing_signatures_for_devices", - self._get_e2e_cross_signing_signatures_for_devices_txn, - batch, + cross_sigs_result = ( + await self._get_e2e_cross_signing_signatures_for_devices(batch) ) # add each cross-signing signature to the correct device in the result dict. - for user_id, key_id, device_id, signature in cross_sigs_result: + for ( + user_id, + device_id, + ), signature_list in cross_sigs_result.items(): target_device_result = result[user_id][device_id] + # We've only looked up cross-signatures for non-deleted devices with key # data. assert target_device_result is not None @@ -351,7 +376,9 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker signing_user_signatures = target_device_signatures.setdefault( user_id, {} ) - signing_user_signatures[key_id] = signature + + for key_id, signature in signature_list: + signing_user_signatures[key_id] = signature log_kv(result) return result @@ -457,41 +484,83 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker return result - def _get_e2e_cross_signing_signatures_for_devices_txn( - self, txn: LoggingTransaction, device_query: Iterable[Tuple[str, str]] - ) -> List[Tuple[str, str, str, str]]: - """Get cross-signing signatures for a given list of devices - - Returns signatures made by the owners of the devices. - - Returns: a list of results; each entry in the list is a tuple of - (user_id, key_id, target_device_id, signature). + @cached() + def _get_e2e_cross_signing_signatures_for_device( + self, + user_id_and_device_id: Tuple[str, str], + ) -> Sequence[Tuple[str, str]]: """ - signature_query_clauses = [] - signature_query_params = [] + The single-item version of `_get_e2e_cross_signing_signatures_for_devices`. + See @cachedList for why a separate method is needed. + """ + raise NotImplementedError() - for user_id, device_id in device_query: - signature_query_clauses.append( - "target_user_id = ? AND target_device_id = ? AND user_id = ?" + @cachedList( + cached_method_name="_get_e2e_cross_signing_signatures_for_device", + list_name="device_query", + ) + async def _get_e2e_cross_signing_signatures_for_devices( + self, device_query: Iterable[Tuple[str, str]] + ) -> Mapping[Tuple[str, str], Sequence[Tuple[str, str]]]: + """Get cross-signing signatures for a given list of user IDs and devices. + + Args: + An iterable containing tuples of (user ID, device ID). + + Returns: + A mapping of results. The keys are the original (user_id, device_id) + tuple, while the value is the matching list of tuples of + (key_id, signature). The value will be an empty list if no + signatures exist for the device. + + Given this method is annotated with `@cachedList`, the return dict's + keys match the tuples within `device_query`, so that cache entries can + be computed from the corresponding values. + + As results are cached, the return type is immutable. + """ + + def _get_e2e_cross_signing_signatures_for_devices_txn( + txn: LoggingTransaction, device_query: Iterable[Tuple[str, str]] + ) -> Mapping[Tuple[str, str], Sequence[Tuple[str, str]]]: + where_clause_sql, where_clause_params = make_tuple_in_list_sql_clause( + self.database_engine, + columns=("target_user_id", "target_device_id", "user_id"), + iterable=[ + (user_id, device_id, user_id) for user_id, device_id in device_query + ], ) - signature_query_params.extend([user_id, device_id, user_id]) - signature_sql = """ - SELECT user_id, key_id, target_device_id, signature - FROM e2e_cross_signing_signatures WHERE %s - """ % (" OR ".join("(" + q + ")" for q in signature_query_clauses)) + signature_sql = f""" + SELECT user_id, key_id, target_device_id, signature + FROM e2e_cross_signing_signatures WHERE {where_clause_sql} + """ - txn.execute(signature_sql, signature_query_params) - return cast( - List[ - Tuple[ - str, - str, - str, - str, - ] - ], - txn.fetchall(), + txn.execute(signature_sql, where_clause_params) + + devices_and_signatures: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + # `@cachedList` requires we return one key for every item in `device_query`. + # Pre-populate `devices_and_signatures` with each key so that none are missing. + # + # If any are missing, they will be cached as `None`, which is not + # what callers expected. + for user_id, device_id in device_query: + devices_and_signatures.setdefault((user_id, device_id), []) + + # Populate the return dictionary with each found key_id and signature. + for user_id, key_id, target_device_id, signature in txn.fetchall(): + signature_tuple = (key_id, signature) + devices_and_signatures[(user_id, target_device_id)].append( + signature_tuple + ) + + return devices_and_signatures + + return await self.db_pool.runInteraction( + "_get_e2e_cross_signing_signatures_for_devices_txn", + _get_e2e_cross_signing_signatures_for_devices_txn, + device_query, ) async def get_e2e_one_time_keys( @@ -593,7 +662,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker txn, self.count_e2e_one_time_keys, (user_id, device_id) ) - @cached(max_entries=10000) + @cached(max_entries=10000, tree=True) async def count_e2e_one_time_keys( self, user_id: str, device_id: str ) -> Mapping[str, int]: @@ -808,7 +877,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker }, ) - @cached(max_entries=10000) + @cached(max_entries=10000, tree=True) async def get_e2e_unused_fallback_key_types( self, user_id: str, device_id: str ) -> Sequence[str]: @@ -1117,7 +1186,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker ) @abc.abstractmethod - def get_device_stream_token(self) -> int: + def get_device_stream_token(self) -> MultiWriterStreamToken: """Get the current stream id from the _device_list_id_gen""" ... @@ -1501,27 +1570,83 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker "delete_old_otks_for_next_user_batch", impl ) + async def allow_master_cross_signing_key_replacement_without_uia( + self, user_id: str, duration_ms: int + ) -> Optional[int]: + """Mark this user's latest master key as being replaceable without UIA. -class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) + Said replacement will only be permitted for a short time after calling this + function. That time period is controlled by the duration argument. - self._cross_signing_id_gen = MultiWriterIdGenerator( - db_conn=db_conn, - db=database, - notifier=hs.get_replication_notifier(), - stream_name="e2e_cross_signing_keys", - instance_name=self._instance_name, - tables=[ - ("e2e_cross_signing_keys", "instance_name", "stream_id"), - ], - sequence_name="e2e_cross_signing_keys_sequence", - writers=["master"], + Returns: + None, if there is no such key. + Otherwise, the timestamp before which replacement is allowed without UIA. + """ + timestamp = self._clock.time_msec() + duration_ms + + def impl(txn: LoggingTransaction) -> Optional[int]: + txn.execute( + """ + UPDATE e2e_cross_signing_keys + SET updatable_without_uia_before_ms = ? + WHERE stream_id = ( + SELECT stream_id + FROM e2e_cross_signing_keys + WHERE user_id = ? AND keytype = 'master' + ORDER BY stream_id DESC + LIMIT 1 + ) + """, + (timestamp, user_id), + ) + if txn.rowcount == 0: + return None + + return timestamp + + return await self.db_pool.runInteraction( + "allow_master_cross_signing_key_replacement_without_uia", + impl, + ) + + async def delete_e2e_keys_by_device(self, user_id: str, device_id: str) -> None: + def delete_e2e_keys_by_device_txn(txn: LoggingTransaction) -> None: + log_kv( + { + "message": "Deleting keys for device", + "device_id": device_id, + "user_id": user_id, + } + ) + self.db_pool.simple_delete_txn( + txn, + table="e2e_device_keys_json", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self.db_pool.simple_delete_txn( + txn, + table="e2e_one_time_keys_json", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self._invalidate_cache_and_stream( + txn, self.count_e2e_one_time_keys, (user_id, device_id) + ) + self.db_pool.simple_delete_txn( + txn, + table="dehydrated_devices", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self.db_pool.simple_delete_txn( + txn, + table="e2e_fallback_keys_json", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self._invalidate_cache_and_stream( + txn, self.get_e2e_unused_fallback_key_types, (user_id, device_id) + ) + + await self.db_pool.runInteraction( + "delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn ) async def set_e2e_device_keys( @@ -1593,46 +1718,6 @@ class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): log_kv({"message": "Device keys stored."}) return True - async def delete_e2e_keys_by_device(self, user_id: str, device_id: str) -> None: - def delete_e2e_keys_by_device_txn(txn: LoggingTransaction) -> None: - log_kv( - { - "message": "Deleting keys for device", - "device_id": device_id, - "user_id": user_id, - } - ) - self.db_pool.simple_delete_txn( - txn, - table="e2e_device_keys_json", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self.db_pool.simple_delete_txn( - txn, - table="e2e_one_time_keys_json", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self._invalidate_cache_and_stream( - txn, self.count_e2e_one_time_keys, (user_id, device_id) - ) - self.db_pool.simple_delete_txn( - txn, - table="dehydrated_devices", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self.db_pool.simple_delete_txn( - txn, - table="e2e_fallback_keys_json", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self._invalidate_cache_and_stream( - txn, self.get_e2e_unused_fallback_key_types, (user_id, device_id) - ) - - await self.db_pool.runInteraction( - "delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn - ) - def _set_e2e_cross_signing_key_txn( self, txn: LoggingTransaction, @@ -1734,63 +1819,79 @@ class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): user_id: the user who made the signatures signatures: signatures to add """ - await self.db_pool.simple_insert_many( - "e2e_cross_signing_signatures", - keys=( - "user_id", - "key_id", - "target_user_id", - "target_device_id", - "signature", - ), - values=[ - ( - user_id, - item.signing_key_id, - item.target_user_id, - item.target_device_id, - item.signature, - ) - for item in signatures - ], - desc="add_e2e_signing_key", - ) - async def allow_master_cross_signing_key_replacement_without_uia( - self, user_id: str, duration_ms: int - ) -> Optional[int]: - """Mark this user's latest master key as being replaceable without UIA. - - Said replacement will only be permitted for a short time after calling this - function. That time period is controlled by the duration argument. - - Returns: - None, if there is no such key. - Otherwise, the timestamp before which replacement is allowed without UIA. - """ - timestamp = self._clock.time_msec() + duration_ms - - def impl(txn: LoggingTransaction) -> Optional[int]: - txn.execute( - """ - UPDATE e2e_cross_signing_keys - SET updatable_without_uia_before_ms = ? - WHERE stream_id = ( - SELECT stream_id - FROM e2e_cross_signing_keys - WHERE user_id = ? AND keytype = 'master' - ORDER BY stream_id DESC - LIMIT 1 - ) - """, - (timestamp, user_id), + def _store_e2e_cross_signing_signatures( + txn: LoggingTransaction, + signatures: "Iterable[SignatureListItem]", + ) -> None: + self.db_pool.simple_insert_many_txn( + txn, + "e2e_cross_signing_signatures", + keys=( + "user_id", + "key_id", + "target_user_id", + "target_device_id", + "signature", + ), + values=[ + ( + user_id, + item.signing_key_id, + item.target_user_id, + item.target_device_id, + item.signature, + ) + for item in signatures + ], ) - if txn.rowcount == 0: - return None - return timestamp + to_invalidate = [ + # Each entry is a tuple of arguments to + # `_get_e2e_cross_signing_signatures_for_device`, which + # itself takes a tuple. Hence the double-tuple. + ((user_id, item.target_device_id),) + for item in signatures + ] - return await self.db_pool.runInteraction( - "allow_master_cross_signing_key_replacement_without_uia", - impl, + if to_invalidate: + # Invalidate the local cache of this worker. + for cache_key in to_invalidate: + txn.call_after( + self._get_e2e_cross_signing_signatures_for_device.invalidate, + cache_key, + ) + + # Stream cache invalidate keys over replication. + # + # We can only send a primitive per function argument across + # replication. + # + # Encode the array of strings as a JSON string, and we'll unpack + # it on the other side. + to_send = [ + (json.dumps([user_id, item.target_device_id]),) + for item in signatures + ] + + self._send_invalidation_to_replication_bulk( + txn, + cache_name=self._get_e2e_cross_signing_signatures_for_device.__name__, + key_tuples=to_send, + ) + + await self.db_pool.runInteraction( + "add_e2e_signing_key", + _store_e2e_cross_signing_signatures, + signatures, ) + + +class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 46aa5902d8..26a91109df 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -45,14 +45,16 @@ from synapse.api.errors import StoreError from synapse.api.room_versions import EventFormatVersions, RoomVersion from synapse.events import EventBase, make_event_from_dict from synapse.logging.opentracing import tag_args, trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import wrap_as_background_process -from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause +from synapse.storage._base import db_to_json, make_in_list_sql_clause from synapse.storage.background_updates import ForeignKeyConstraint from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, LoggingTransaction, ) +from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.databases.main.signatures import SignatureWorkerStore from synapse.storage.engines import PostgresEngine, Sqlite3Engine @@ -69,17 +71,20 @@ if TYPE_CHECKING: oldest_pdu_in_federation_staging = Gauge( "synapse_federation_server_oldest_inbound_pdu_in_staging", "The age in seconds since we received the oldest pdu in the federation staging area", + labelnames=[SERVER_NAME_LABEL], ) number_pdus_in_federation_queue = Gauge( "synapse_federation_server_number_inbound_pdu_in_staging", "The total number of events in the inbound federation staging", + labelnames=[SERVER_NAME_LABEL], ) pdus_pruned_from_federation_queue = Counter( "synapse_federation_server_number_inbound_pdu_pruned", "The number of events in the inbound federation staging that have been " "pruned due to the queue getting too long", + labelnames=[SERVER_NAME_LABEL], ) logger = logging.getLogger(__name__) @@ -109,6 +114,12 @@ _LONGEST_BACKOFF_PERIOD_MILLISECONDS = ( assert 0 < _LONGEST_BACKOFF_PERIOD_MILLISECONDS <= ((2**31) - 1) +# We use 2^53-1 as a "very large number", it has no particular +# importance other than knowing synapse can support it (given canonical json +# requires it). +MAX_CHAIN_LENGTH = (2**53) - 1 + + # All the info we need while iterating the DAG while backfilling @attr.s(frozen=True, slots=True, auto_attribs=True) class BackfillQueueNavigationItem: @@ -118,12 +129,22 @@ class BackfillQueueNavigationItem: type: str +@attr.s(frozen=True, slots=True, auto_attribs=True) +class StateDifference: + # The event IDs in the auth difference. + auth_difference: Set[str] + # The event IDs in the conflicted state subgraph. Used in v2.1 only. + conflicted_subgraph: Optional[Set[str]] + + class _NoChainCoverIndex(Exception): def __init__(self, room_id: str): super().__init__("Unexpectedly no chain cover for events in %s" % (room_id,)) -class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBaseStore): +class EventFederationWorkerStore( + SignatureWorkerStore, EventsWorkerStore, CacheInvalidationWorkerStore +): # TODO: this attribute comes from EventPushActionWorkerStore. Should we inherit from # that store so that mypy can deduce this for itself? stream_ordering_month_ago: Optional[int] @@ -145,7 +166,10 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas # Cache of event ID to list of auth event IDs and their depths. self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache( - 500000, "_event_auth_cache", size_callback=len + max_size=500000, + server_name=self.server_name, + cache_name="_event_auth_cache", + size_callback=len, ) # Flag used by unit tests to disable fallback when there is no chain cover @@ -461,17 +485,41 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas return results async def get_auth_chain_difference( - self, room_id: str, state_sets: List[Set[str]] + self, + room_id: str, + state_sets: List[Set[str]], ) -> Set[str]: - """Given sets of state events figure out the auth chain difference (as + state_diff = await self.get_auth_chain_difference_extended( + room_id, state_sets, None, None + ) + return state_diff.auth_difference + + async def get_auth_chain_difference_extended( + self, + room_id: str, + state_sets: List[Set[str]], + conflicted_set: Optional[Set[str]], + additional_backwards_reachable_conflicted_events: Optional[Set[str]], + ) -> StateDifference: + """ "Given sets of state events figure out the auth chain difference (as per state res v2 algorithm). - This equivalent to fetching the full auth chain for each set of state + This is equivalent to fetching the full auth chain for each set of state and returning the events that don't appear in each and every auth chain. + If conflicted_set is not None, calculate and return the conflicted sub-graph as per + state res v2.1. The event IDs in the conflicted set MUST be a subset of the event IDs in + state_sets. + + If additional_backwards_reachable_conflicted_events is set, the provided events are included + when calculating the conflicted subgraph. This is primarily useful for calculating the + subgraph across a combination of persisted and unpersisted events. The event IDs in this set + MUST be a subset of the event IDs in state_sets. + Returns: - The set of the difference in auth chains. + information on the auth chain difference, and also the conflicted subgraph if + conflicted_set is not None """ # Check if we have indexed the room so we can use the chain cover @@ -485,6 +533,8 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas self._get_auth_chain_difference_using_cover_index_txn, room_id, state_sets, + conflicted_set, + additional_backwards_reachable_conflicted_events, ) except _NoChainCoverIndex: # For whatever reason we don't actually have a chain cover index @@ -493,25 +543,48 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas if not self.tests_allow_no_chain_cover_index: raise - return await self.db_pool.runInteraction( + # It's been 4 years since we added chain cover, so we expect all rooms to have it. + # If they don't, we will error out when trying to do state res v2.1 + if conflicted_set is not None: + raise _NoChainCoverIndex(room_id) + + auth_diff = await self.db_pool.runInteraction( "get_auth_chain_difference", self._get_auth_chain_difference_txn, state_sets, ) + return StateDifference(auth_difference=auth_diff, conflicted_subgraph=None) def _get_auth_chain_difference_using_cover_index_txn( - self, txn: LoggingTransaction, room_id: str, state_sets: List[Set[str]] - ) -> Set[str]: + self, + txn: LoggingTransaction, + room_id: str, + state_sets: List[Set[str]], + conflicted_set: Optional[Set[str]] = None, + additional_backwards_reachable_conflicted_events: Optional[Set[str]] = None, + ) -> StateDifference: """Calculates the auth chain difference using the chain index. See docs/auth_chain_difference_algorithm.md for details """ + is_state_res_v21 = conflicted_set is not None # First we look up the chain ID/sequence numbers for all the events, and # work out the chain/sequence numbers reachable from each state set. initial_events = set(state_sets[0]).union(*state_sets[1:]) + if is_state_res_v21: + # Sanity check v2.1 fields + assert conflicted_set is not None + assert conflicted_set.issubset(initial_events) + # It's possible for the conflicted_set to be empty if all the conflicts are in + # unpersisted events, so we don't assert that conflicted_set has len > 0 + if additional_backwards_reachable_conflicted_events: + assert additional_backwards_reachable_conflicted_events.issubset( + initial_events + ) + # Map from event_id -> (chain ID, seq no) chain_info: Dict[str, Tuple[int, int]] = {} @@ -547,14 +620,14 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas events_missing_chain_info = initial_events.difference(chain_info) # The result set to return, i.e. the auth chain difference. - result: Set[str] = set() + auth_difference_result: Set[str] = set() if events_missing_chain_info: # For some reason we have events we haven't calculated the chain # index for, so we need to handle those separately. This should only # happen for older rooms where the server doesn't have all the auth # events. - result = self._fixup_auth_chain_difference_sets( + auth_difference_result = self._fixup_auth_chain_difference_sets( txn, room_id, state_sets=state_sets, @@ -573,6 +646,45 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas fetch_chain_info(new_events_to_fetch) + # State Res v2.1 needs extra data structures to calculate the conflicted subgraph which + # are outlined below. + + # A subset of chain_info for conflicted events only, as we need to + # loop all conflicted chain positions. Map from event_id -> (chain ID, seq no) + conflicted_chain_positions: Dict[str, Tuple[int, int]] = {} + # For each chain, remember the positions where conflicted events are. + # We need this for calculating the forward reachable events. + conflicted_chain_to_seq: Dict[int, Set[int]] = {} # chain_id => {seq_num} + # A subset of chain_info for additional backwards reachable events only, as we need to + # loop all additional backwards reachable events for calculating backwards reachable events. + additional_backwards_reachable_positions: Dict[ + str, Tuple[int, int] + ] = {} # event_id => (chain_id, seq_num) + # These next two fields are critical as the intersection of them is the conflicted subgraph. + # We'll populate them when we walk the chain links. + # chain_id => max(seq_num) backwards reachable (e.g 4 means 1,2,3,4 are backwards reachable) + conflicted_backwards_reachable: Dict[int, int] = {} + # chain_id => min(seq_num) forwards reachable (e.g 4 means 4,5,6..n are forwards reachable) + conflicted_forwards_reachable: Dict[int, int] = {} + + # populate the v2.1 data structures + if is_state_res_v21: + assert conflicted_set is not None + # provide chain positions for each conflicted event + for conflicted_event_id in conflicted_set: + (chain_id, seq_num) = chain_info[conflicted_event_id] + conflicted_chain_positions[conflicted_event_id] = (chain_id, seq_num) + conflicted_chain_to_seq.setdefault(chain_id, set()).add(seq_num) + if additional_backwards_reachable_conflicted_events: + for ( + additional_event_id + ) in additional_backwards_reachable_conflicted_events: + (chain_id, seq_num) = chain_info[additional_event_id] + additional_backwards_reachable_positions[additional_event_id] = ( + chain_id, + seq_num, + ) + # Corresponds to `state_sets`, except as a map from chain ID to max # sequence number reachable from the state set. set_to_chain: List[Dict[int, int]] = [] @@ -590,6 +702,8 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas # (We need to take a copy of `seen_chains` as the function mutates it) for links in self._get_chain_links(txn, set(seen_chains)): + # `links` encodes the backwards reachable events _from a single chain_ all the way to + # the root of the graph. for chains in set_to_chain: for chain_id in links: if chain_id not in chains: @@ -598,6 +712,87 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas _materialize(chain_id, chains[chain_id], links, chains) seen_chains.update(chains) + if is_state_res_v21: + # Apply v2.1 conflicted event reachability checks. + # + # A <-- B <-- C <-- D <-- E + # + # Backwards reachable from C = {A,B} + # Forwards reachable from C = {D,E} + + # this handles calculating forwards reachable information and updates + # conflicted_forwards_reachable. + accumulate_forwards_reachable_events( + conflicted_forwards_reachable, + links, + conflicted_chain_positions, + ) + + # handle backwards reachable information + for ( + conflicted_chain_id, + conflicted_chain_seq, + ) in conflicted_chain_positions.values(): + if conflicted_chain_id not in links: + # This conflicted event does not lie on the path to the root. + continue + + # The conflicted chain position itself encodes reachability information + # _within_ the chain. Set it now before walking to other links. + conflicted_backwards_reachable[conflicted_chain_id] = max( + conflicted_chain_seq, + conflicted_backwards_reachable.get(conflicted_chain_id, 0), + ) + + # Build backwards reachability paths. This is the same as what the auth difference + # code does. We find which chain the conflicted event + # belongs to then walk it backwards to the root. We store reachability info + # for all conflicted events in the same map 'conflicted_backwards_reachable' + # as we don't care about the paths themselves. + _materialize( + conflicted_chain_id, + conflicted_chain_seq, + links, + conflicted_backwards_reachable, + ) + # Mark some extra events as backwards reachable. This is used when we have some + # unpersisted events and want to know the subgraph across the persisted/unpersisted + # boundary: + # | + # A <-- B <-- C <-|- D <-- E <-- F + # persisted | unpersisted + # + # Assume {B,E} are conflicted, we want to return {B,C,D,E} + # + # The unpersisted code ensures it passes C as an additional backwards reachable + # event. C is NOT a conflicted event, but we do need to consider it as part of + # the backwards reachable set. When we then calculate the forwards reachable set + # from B, C will be in both the backwards and forwards reachable sets and hence + # will be included in the conflicted subgraph. + for ( + additional_chain_id, + additional_chain_seq, + ) in additional_backwards_reachable_positions.values(): + if additional_chain_id not in links: + # The additional backwards reachable event does not lie on the path to the root. + continue + + # the additional event chain position itself encodes reachability information. + # It means that position and all positions earlier in that chain are backwards reachable + # by some unpersisted conflicted event. + conflicted_backwards_reachable[additional_chain_id] = max( + additional_chain_seq, + conflicted_backwards_reachable.get(additional_chain_id, 0), + ) + + # Now walk the chains back, marking backwards reachable events. + # This is the same thing we do for auth difference / conflicted events. + _materialize( + additional_chain_id, # walk all links back, marking them as backwards reachable + additional_chain_seq, + links, + conflicted_backwards_reachable, + ) # Now for each chain we figure out the maximum sequence number reachable # from *any* state set and the minimum sequence number reachable from @@ -606,7 +801,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas # Mapping from chain ID to the range of sequence numbers that should be # pulled from the database. - chain_to_gap: Dict[int, Tuple[int, int]] = {} + auth_diff_chain_to_gap: Dict[int, Tuple[int, int]] = {} for chain_id in seen_chains: min_seq_no = min(chains.get(chain_id, 0) for chains in set_to_chain) @@ -619,15 +814,76 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas for seq_no in range(min_seq_no + 1, max_seq_no + 1): event_id = chain_to_event.get(chain_id, {}).get(seq_no) if event_id: - result.add(event_id) + auth_difference_result.add(event_id) else: - chain_to_gap[chain_id] = (min_seq_no, max_seq_no) + auth_diff_chain_to_gap[chain_id] = (min_seq_no, max_seq_no) break - if not chain_to_gap: - # If there are no gaps to fetch, we're done! - return result + conflicted_subgraph_result: Set[str] = set() + # Mapping from chain ID to the range of sequence numbers that should be + # pulled from the database. + conflicted_subgraph_chain_to_gap: Dict[int, Tuple[int, int]] = {} + if is_state_res_v21: + # also include the conflicted subgraph using backward/forward reachability info from all + # the conflicted events. To calculate this, we want to extract the intersection between + # the backwards and forwards reachability sets, e.g: + # A <- B <- C <- D <- E + # Assume B and D are conflicted so we want {C} as the conflicted subgraph. + # B_backwards={A}, B_forwards={C,D,E} + # D_backwards={A,B,C} D_forwards={E} + # ALL_backwards={A,B,C} ALL_forwards={C,D,E} + # Intersection(ALL_backwards, ALL_forwards) = {C} + # + # It's worth noting that once we have the ALL_ sets, we no longer care about the paths. + # We're dealing with chains and not singular events, but we've already got the ALL_ sets. + # As such, we can inspect each chain in isolation and check for overlapping sequence + # numbers: + # 1,2,3,4,5 Seq Num + # Chain N [A,B,C,D,E] + # + # if (N,4) is in the backwards set and (N,2) is in the forwards set, then the + # intersection is events between 2 < 4. We will include the conflicted events themselves + # in the subgraph, but they will already be, hence the full set of events is {B,C,D}. + for chain_id, backwards_seq_num in conflicted_backwards_reachable.items(): + forwards_seq_num = conflicted_forwards_reachable.get(chain_id) + if forwards_seq_num is None: + continue # this chain isn't in both sets so can't intersect + if forwards_seq_num > backwards_seq_num: + continue # this chain is in both sets but they don't overap + for seq_no in range( + forwards_seq_num, backwards_seq_num + 1 + ): # inclusive of both + event_id = chain_to_event.get(chain_id, {}).get(seq_no) + if event_id: + conflicted_subgraph_result.add(event_id) + else: + conflicted_subgraph_chain_to_gap[chain_id] = ( + # _fetch_event_ids_from_chains_txn is exclusive of the min value + forwards_seq_num - 1, + backwards_seq_num, + ) + break + if auth_diff_chain_to_gap: + auth_difference_result.update( + self._fetch_event_ids_from_chains_txn(txn, auth_diff_chain_to_gap) + ) + if conflicted_subgraph_chain_to_gap: + conflicted_subgraph_result.update( + self._fetch_event_ids_from_chains_txn( + txn, conflicted_subgraph_chain_to_gap + ) + ) + + return StateDifference( + auth_difference=auth_difference_result, + conflicted_subgraph=conflicted_subgraph_result, + ) + + def _fetch_event_ids_from_chains_txn( + self, txn: LoggingTransaction, chains: Dict[int, Tuple[int, int]] + ) -> Set[str]: + result: Set[str] = set() if isinstance(self.database_engine, PostgresEngine): # We can use `execute_values` to efficiently fetch the gaps when # using postgres. @@ -641,7 +897,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas args = [ (chain_id, min_no, max_no) - for chain_id, (min_no, max_no) in chain_to_gap.items() + for chain_id, (min_no, max_no) in chains.items() ] rows = txn.execute_values(sql, args) @@ -652,10 +908,9 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas SELECT event_id FROM event_auth_chains WHERE chain_id = ? AND ? < sequence_number AND sequence_number <= ? """ - for chain_id, (min_no, max_no) in chain_to_gap.items(): + for chain_id, (min_no, max_no) in chains.items(): txn.execute(sql, (chain_id, min_no, max_no)) result.update(r for (r,) in txn) - return result def _fixup_auth_chain_difference_sets( @@ -1997,7 +2252,9 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas if not to_delete: return False - pdus_pruned_from_federation_queue.inc(len(to_delete)) + pdus_pruned_from_federation_queue.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(len(to_delete)) logger.info( "Pruning %d events in room %s from federation queue", len(to_delete), @@ -2050,8 +2307,25 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas "_get_stats_for_federation_staging", _get_stats_for_federation_staging_txn ) - number_pdus_in_federation_queue.set(count) - oldest_pdu_in_federation_staging.set(age) + number_pdus_in_federation_queue.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(count) + oldest_pdu_in_federation_staging.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(age) + + async def clean_room_for_join(self, room_id: str) -> None: + await self.db_pool.runInteraction( + "clean_room_for_join", self._clean_room_for_join_txn, room_id + ) + + def _clean_room_for_join_txn(self, txn: LoggingTransaction, room_id: str) -> None: + query = "DELETE FROM event_forward_extremities WHERE room_id = ?" + + txn.execute(query, (room_id,)) + self._invalidate_cache_and_stream( + txn, self.get_latest_event_ids_in_room, (room_id,) + ) class EventFederationStore(EventFederationWorkerStore): @@ -2078,17 +2352,6 @@ class EventFederationStore(EventFederationWorkerStore): self.EVENT_AUTH_STATE_ONLY, self._background_delete_non_state_event_auth ) - async def clean_room_for_join(self, room_id: str) -> None: - await self.db_pool.runInteraction( - "clean_room_for_join", self._clean_room_for_join_txn, room_id - ) - - def _clean_room_for_join_txn(self, txn: LoggingTransaction, room_id: str) -> None: - query = "DELETE FROM event_forward_extremities WHERE room_id = ?" - - txn.execute(query, (room_id,)) - txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,)) - async def _background_delete_non_state_event_auth( self, progress: JsonDict, batch_size: int ) -> int: @@ -2147,6 +2410,7 @@ def _materialize( origin_sequence_number: int, links: Dict[int, List[Tuple[int, int, int]]], materialized: Dict[int, int], + backwards: bool = True, ) -> None: """Helper function for fetching auth chain links. For a given origin chain ID / sequence number and a dictionary of links, updates the materialized @@ -2163,6 +2427,7 @@ def _materialize( target sequence number. materialized: dict to update with new reachability information, as a map from chain ID to max sequence number reachable. + backwards: If True, walks backwards down the chains. If False, walks forwards from the chains. """ # Do a standard graph traversal. @@ -2177,12 +2442,104 @@ def _materialize( target_chain_id, target_sequence_number, ) in chain_links: - # Ignore any links that are higher up the chain - if sequence_number > s: - continue + if backwards: + # Ignore any links that are higher up the chain + if sequence_number > s: + continue - # Check if we have already visited the target chain before, if so we - # can skip it. - if materialized.get(target_chain_id, 0) < target_sequence_number: - stack.append((target_chain_id, target_sequence_number)) - materialized[target_chain_id] = target_sequence_number + # Check if we have already visited the target chain before, if so we + # can skip it. + if materialized.get(target_chain_id, 0) < target_sequence_number: + stack.append((target_chain_id, target_sequence_number)) + materialized[target_chain_id] = target_sequence_number + else: + # Ignore any links that are lower down the chain. + if sequence_number < s: + continue + # Check if we have already visited the target chain before, if so we + # can skip it. + if ( + materialized.get(target_chain_id, MAX_CHAIN_LENGTH) + > target_sequence_number + ): + stack.append((target_chain_id, target_sequence_number)) + materialized[target_chain_id] = target_sequence_number + + +def _generate_forward_links( + links: Dict[int, List[Tuple[int, int, int]]], +) -> Dict[int, List[Tuple[int, int, int]]]: + """Reverse the input links from the given backwards links""" + new_links: Dict[int, List[Tuple[int, int, int]]] = {} + for origin_chain_id, chain_links in links.items(): + for origin_seq_num, target_chain_id, target_seq_num in chain_links: + new_links.setdefault(target_chain_id, []).append( + (target_seq_num, origin_chain_id, origin_seq_num) + ) + return new_links + + +def accumulate_forwards_reachable_events( + conflicted_forwards_reachable: Dict[int, int], + back_links: Dict[int, List[Tuple[int, int, int]]], + conflicted_chain_positions: Dict[str, Tuple[int, int]], +) -> None: + """Accumulate new forwards reachable events using the back_links provided. + + Accumulating forwards reachable information is quite different from backwards reachable information + because _get_chain_links returns the entire linkage information for backwards reachable events, + but not _forwards_ reachable events. We are only interested in the forwards reachable information + that is encoded in the backwards reachable links, so we can just invert all the operations we do + for backwards reachable events to calculate a subset of forwards reachable information. The + caveat with this approach is that it is a _subset_. This means new back_links may encode new + forwards reachable information which we also need. Consider this scenario: + + A <-- B <-- C <--- D <-- E <-- F Chain 1 + | + `----- G <-- H <-- I Chain 2 + | + `---- J <-- K Chain 3 + + Now consider what happens when B is a conflicted event. _get_chain_links returns the conflicted + chain and ALL links heading towards the root of the graph. This means we will know the + Chain 1 to Chain 2 link via C (as all links for the chain are returned, not strictly ones with + a lower sequence number), but we will NOT know the Chain 2 to Chain 3 link via H. We can be + blissfully unaware of Chain 3 entirely, if and only if there isn't some other conflicted event + on that chain. Consider what happens when K is /also/ conflicted. _get_chain_links will generate + two iterations: one for B and one for K. It's important that we re-evaluate the forwards reachable + information for B to include Chain 3 when we process the K iteration, hence we are "accumulating" + forwards reachability information. + + NB: We don't consider 'additional backwards reachable events' here because they have no effect + on forwards reachability calculations, only backwards. + + Args: + conflicted_forwards_reachable: The materialised dict of forwards reachable information. + The output to this function are stored here. + back_links: One iteration of _get_chain_links which encodes backwards reachable information. + conflicted_chain_positions: The conflicted events. + """ + # links go backwards but we want them to go forwards as well for v2.1 + fwd_links = _generate_forward_links(back_links) + + # for each conflicted event, accumulate forwards reachability information + for ( + conflicted_chain_id, + conflicted_chain_seq, + ) in conflicted_chain_positions.values(): + # the conflicted event itself encodes reachability information + # e.g if D was conflicted, it encodes E,F as forwards reachable. + conflicted_forwards_reachable[conflicted_chain_id] = min( + conflicted_chain_seq, + conflicted_forwards_reachable.get(conflicted_chain_id, MAX_CHAIN_LENGTH), + ) + # Walk from the conflicted event forwards to explore the links. + # This function checks if we've visited the chain before and skips reprocessing, so this + # does not repeatedly traverse the graph. + _materialize( + conflicted_chain_id, + conflicted_chain_seq, + fwd_links, + conflicted_forwards_reachable, + backwards=False, + ) diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index dd6ac909e9..a50e889b9d 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -35,12 +35,12 @@ from typing import ( Sequence, Set, Tuple, + TypedDict, cast, ) import attr from prometheus_client import Counter -from typing_extensions import TypedDict import synapse.metrics from synapse.api.constants import ( @@ -51,10 +51,16 @@ from synapse.api.constants import ( ) from synapse.api.errors import PartialStateConflictError from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, StrippedStateEvent, relation_from_event -from synapse.events.snapshot import EventContext +from synapse.events import ( + EventBase, + StrippedStateEvent, + is_creator, + relation_from_event, +) +from synapse.events.snapshot import EventPersistencePair from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.storage._base import db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, @@ -78,6 +84,7 @@ from synapse.types import ( from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.state import StateFilter from synapse.util import json_encoder +from synapse.util.events import get_plain_text_topic_from_event_content from synapse.util.iterutils import batch_iter, sorted_topologically from synapse.util.stringutils import non_null_str_or_none @@ -88,11 +95,13 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -persist_event_counter = Counter("synapse_storage_events_persisted_events", "") +persist_event_counter = Counter( + "synapse_storage_events_persisted_events", "", labelnames=[SERVER_NAME_LABEL] +) event_counter = Counter( "synapse_storage_events_persisted_events_sep", "", - ["type", "origin_type", "origin_entity"], + labelnames=["type", "origin_type", "origin_entity", SERVER_NAME_LABEL], ) # State event type/key pairs that we need to gather to fill in the @@ -236,6 +245,7 @@ class PersistEventsStore: db_conn: LoggingDatabaseConnection, ): self.hs = hs + self.server_name = hs.hostname self.db_pool = db self.store = main_data_store self.database_engine = db.engine @@ -246,9 +256,9 @@ class PersistEventsStore: self.is_mine_id = hs.is_mine_id # This should only exist on instances that are configured to write - assert ( - hs.get_instance_name() in hs.config.worker.writers.events - ), "Can only instantiate EventsStore on master" + assert hs.get_instance_name() in hs.config.worker.writers.events, ( + "Can only instantiate EventsStore on master" + ) # Since we have been configured to write, we ought to have id generators, # rather than id trackers. @@ -264,7 +274,7 @@ class PersistEventsStore: async def _persist_events_and_state_updates( self, room_id: str, - events_and_contexts: List[Tuple[EventBase, EventContext]], + events_and_contexts: List[EventPersistencePair], *, state_delta_for_room: Optional[DeltaState], new_forward_extremities: Optional[Set[str]], @@ -356,12 +366,16 @@ class PersistEventsStore: new_event_links=new_event_links, sliding_sync_table_changes=sliding_sync_table_changes, ) - persist_event_counter.inc(len(events_and_contexts)) + persist_event_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc( + len(events_and_contexts) + ) if not use_negative_stream_ordering: # we don't want to set the event_persisted_position to a negative # stream_ordering. - synapse.metrics.event_persisted_position.set(stream) + synapse.metrics.event_persisted_position.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(stream) for event, context in events_and_contexts: if context.app_service: @@ -374,17 +388,151 @@ class PersistEventsStore: origin_type = "remote" origin_entity = get_domain_from_id(event.sender) - event_counter.labels(event.type, origin_type, origin_entity).inc() + event_counter.labels( + type=event.type, + origin_type=origin_type, + origin_entity=origin_entity, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() + + if ( + not self.hs.config.experimental.msc4293_enabled + or event.type != EventTypes.Member + or event.state_key is None + ): + continue + + # check if this is an unban/join that will undo a ban/kick redaction for + # a user in the room + if event.membership in [Membership.LEAVE, Membership.JOIN]: + if ( + event.membership == Membership.LEAVE + and event.sender == event.state_key + ): + # self-leave, ignore + continue + + # if there is an existing ban/leave causing redactions for + # this user/room combination update the entry with the stream + # ordering when the redactions should stop - in the case of a backfilled + # event where the stream ordering is negative, use the current max stream + # ordering + stream_ordering = event.internal_metadata.stream_ordering + assert stream_ordering is not None + if stream_ordering < 0: + stream_ordering = self._stream_id_gen.get_current_token() + await self.db_pool.simple_update( + "room_ban_redactions", + {"room_id": event.room_id, "user_id": event.state_key}, + {"redact_end_ordering": stream_ordering}, + desc="room_ban_redactions update redact_end_ordering", + ) + + # check for msc4293 redact_events flag and apply if found + if event.membership not in [Membership.LEAVE, Membership.BAN]: + continue + redact = event.content.get("org.matrix.msc4293.redact_events", False) + if not redact or not isinstance(redact, bool): + continue + # self-bans currently are not authorized so we don't check for that + # case + if ( + event.membership == Membership.BAN + and event.sender == event.state_key + ): + continue + + # check that sender can redact + redact_allowed = await self._can_sender_redact(event) + + # Signal that this user's past events in this room + # should be redacted by adding an entry to + # `room_ban_redactions`. + if redact_allowed: + await self.db_pool.simple_upsert( + "room_ban_redactions", + {"room_id": event.room_id, "user_id": event.state_key}, + { + "redacting_event_id": event.event_id, + "redact_end_ordering": None, + }, + { + "room_id": event.room_id, + "user_id": event.state_key, + "redacting_event_id": event.event_id, + "redact_end_ordering": None, + }, + ) + + # normally the cache entry for a redacted event would be invalidated + # by an arriving redaction event, but since we are not creating redaction + # events we invalidate manually + self.store._invalidate_local_get_event_cache_room_id(event.room_id) + + self.store._invalidate_async_get_event_cache_room_id(event.room_id) if new_forward_extremities: self.store.get_latest_event_ids_in_room.prefill( (room_id,), frozenset(new_forward_extremities) ) + async def _can_sender_redact(self, event: EventBase) -> bool: + state_filter = StateFilter.from_types( + [(EventTypes.PowerLevels, ""), (EventTypes.Create, "")] + ) + state = await self.store.get_partial_filtered_current_state_ids( + event.room_id, state_filter + ) + pl_id = state[(EventTypes.PowerLevels, "")] + pl_event = await self.store.get_event(pl_id, allow_none=True) + + create_id = state[(EventTypes.Create, "")] + create_event = await self.store.get_event(create_id, allow_none=True) + + if create_event is None: + # not sure how this would happen but if it does then just deny the redaction + logger.warning("No create event found for room %s", event.room_id) + return False + + if create_event.room_version.msc4289_creator_power_enabled: + # per the spec, grant the creator infinite power level and all other users 0 + if is_creator(create_event, event.sender): + return True + if pl_event is None: + # per the spec, users other than the room creator have power level + # 0, which is less than the default to redact events (50). + return False + else: + # per the spec, if a power level event isn't in the room, grant the creator + # level 100 (the default redaction level is 50) and all other users 0 + if pl_event is None: + return create_event.sender == event.sender + + assert pl_event is not None + sender_level = pl_event.content.get("users", {}).get(event.sender) + if sender_level is None: + sender_level = pl_event.content.get("users_default", 0) + + redact_level = pl_event.content.get("redact") + if redact_level is None: + redact_level = pl_event.content.get("events_default", 0) + + room_redaction_level = pl_event.content.get("events", {}).get( + "m.room.redaction" + ) + if room_redaction_level is not None: + if sender_level < room_redaction_level: + return False + + if sender_level >= redact_level: + return True + + return False + async def _calculate_sliding_sync_table_changes( self, room_id: str, - events_and_contexts: Sequence[Tuple[EventBase, EventContext]], + events_and_contexts: Sequence[EventPersistencePair], delta_state: DeltaState, ) -> SlidingSyncTableChanges: """ @@ -465,9 +613,9 @@ class PersistEventsStore: missing_membership_event_ids ) # There shouldn't be any missing events - assert ( - remaining_events.keys() == missing_membership_event_ids - ), missing_membership_event_ids.difference(remaining_events.keys()) + assert remaining_events.keys() == missing_membership_event_ids, ( + missing_membership_event_ids.difference(remaining_events.keys()) + ) membership_event_map.update(remaining_events) for ( @@ -534,9 +682,9 @@ class PersistEventsStore: missing_state_event_ids ) # There shouldn't be any missing events - assert ( - remaining_events.keys() == missing_state_event_ids - ), missing_state_event_ids.difference(remaining_events.keys()) + assert remaining_events.keys() == missing_state_event_ids, ( + missing_state_event_ids.difference(remaining_events.keys()) + ) for event in remaining_events.values(): current_state_map[(event.type, event.state_key)] = event @@ -644,9 +792,9 @@ class PersistEventsStore: if missing_event_ids: remaining_events = await self.store.get_events(missing_event_ids) # There shouldn't be any missing events - assert ( - remaining_events.keys() == missing_event_ids - ), missing_event_ids.difference(remaining_events.keys()) + assert remaining_events.keys() == missing_event_ids, ( + missing_event_ids.difference(remaining_events.keys()) + ) for event in remaining_events.values(): current_state_map[(event.type, event.state_key)] = event @@ -868,7 +1016,7 @@ class PersistEventsStore: txn: LoggingTransaction, *, room_id: str, - events_and_contexts: List[Tuple[EventBase, EventContext]], + events_and_contexts: List[EventPersistencePair], inhibit_local_membership_updates: bool, state_delta_for_room: Optional[DeltaState], new_forward_extremities: Optional[Set[str]], @@ -1518,7 +1666,7 @@ class PersistEventsStore: def _persist_transaction_ids_txn( self, txn: LoggingTransaction, - events_and_contexts: List[Tuple[EventBase, EventContext]], + events_and_contexts: List[EventPersistencePair], ) -> None: """Persist the mapping from transaction IDs to event IDs (if defined).""" @@ -1605,7 +1753,13 @@ class PersistEventsStore: room_id delta_state: Deltas that are going to be used to update the `current_state_events` table. Changes to the current state of the room. - stream_id: TODO + stream_id: This is expected to be the minimum `stream_ordering` for the + batch of events that we are persisting; which means we do not end up in a + situation where workers see events before the `current_state_delta` updates. + FIXME: However, this function also gets called with next upcoming + `stream_ordering` when we re-sync the state of a partial stated room (see + `update_current_state(...)`) which may be "correct" but it would be good to + nail down what exactly is the expected value here. sliding_sync_table_changes: Changes to the `sliding_sync_membership_snapshots` and `sliding_sync_joined_rooms` tables derived from the given `delta_state` (see @@ -1908,6 +2062,13 @@ class PersistEventsStore: stream_id, ) + for user_id in members_to_cache_bust: + txn.call_after( + self.store._membership_stream_cache.entity_has_changed, + user_id, + stream_id, + ) + # Invalidate the various caches self.store._invalidate_state_caches_and_stream( txn, room_id, members_to_cache_bust @@ -2155,7 +2316,7 @@ class PersistEventsStore: self, txn: LoggingTransaction, room_id: str, - events_and_contexts: List[Tuple[EventBase, EventContext]], + events_and_contexts: List[EventPersistencePair], ) -> None: """ Update the latest `event_stream_ordering`/`bump_stamp` columns in the @@ -2295,8 +2456,8 @@ class PersistEventsStore: @classmethod def _filter_events_and_contexts_for_duplicates( - cls, events_and_contexts: List[Tuple[EventBase, EventContext]] - ) -> List[Tuple[EventBase, EventContext]]: + cls, events_and_contexts: List[EventPersistencePair] + ) -> List[EventPersistencePair]: """Ensure that we don't have the same event twice. Pick the earliest non-outlier if there is one, else the earliest one. @@ -2307,9 +2468,7 @@ class PersistEventsStore: Returns: filtered list """ - new_events_and_contexts: OrderedDict[str, Tuple[EventBase, EventContext]] = ( - OrderedDict() - ) + new_events_and_contexts: OrderedDict[str, EventPersistencePair] = OrderedDict() for event, context in events_and_contexts: prev_event_context = new_events_and_contexts.get(event.event_id) if prev_event_context: @@ -2327,7 +2486,7 @@ class PersistEventsStore: self, txn: LoggingTransaction, room_id: str, - events_and_contexts: List[Tuple[EventBase, EventContext]], + events_and_contexts: List[EventPersistencePair], ) -> None: """Update min_depth for each room @@ -2369,8 +2528,8 @@ class PersistEventsStore: def _update_outliers_txn( self, txn: LoggingTransaction, - events_and_contexts: List[Tuple[EventBase, EventContext]], - ) -> List[Tuple[EventBase, EventContext]]: + events_and_contexts: List[EventPersistencePair], + ) -> List[EventPersistencePair]: """Update any outliers with new event info. This turns outliers into ex-outliers (unless the new event was rejected), and @@ -2477,7 +2636,7 @@ class PersistEventsStore: def _store_event_txn( self, txn: LoggingTransaction, - events_and_contexts: Collection[Tuple[EventBase, EventContext]], + events_and_contexts: Collection[EventPersistencePair], ) -> None: """Insert new events into the event, event_json, redaction and state_events tables. @@ -2581,8 +2740,8 @@ class PersistEventsStore: def _store_rejected_events_txn( self, txn: LoggingTransaction, - events_and_contexts: List[Tuple[EventBase, EventContext]], - ) -> List[Tuple[EventBase, EventContext]]: + events_and_contexts: List[EventPersistencePair], + ) -> List[EventPersistencePair]: """Add rows to the 'rejections' table for received events which were rejected @@ -2609,8 +2768,8 @@ class PersistEventsStore: self, txn: LoggingTransaction, *, - events_and_contexts: List[Tuple[EventBase, EventContext]], - all_events_and_contexts: List[Tuple[EventBase, EventContext]], + events_and_contexts: List[EventPersistencePair], + all_events_and_contexts: List[EventPersistencePair], inhibit_local_membership_updates: bool = False, ) -> None: """Update all the miscellaneous tables for new events @@ -2704,9 +2863,9 @@ class PersistEventsStore: def _add_to_cache( self, txn: LoggingTransaction, - events_and_contexts: List[Tuple[EventBase, EventContext]], + events_and_contexts: List[EventPersistencePair], ) -> None: - to_prefill = [] + to_prefill: List[EventCacheEntry] = [] ev_map = {e.event_id: e for e, _ in events_and_contexts} if not ev_map: @@ -2972,6 +3131,10 @@ class PersistEventsStore: # Upsert into the threads table, but only overwrite the value if the # new event is of a later topological order OR if the topological # ordering is equal, but the stream ordering is later. + # (Note by definition that the stream ordering will always be later + # unless this is a backfilled event [= negative stream ordering] + # because we are only persisting this event now and stream_orderings + # are strictly monotonically increasing) sql = """ INSERT INTO threads (room_id, thread_id, latest_event_id, topological_ordering, stream_ordering) VALUES (?, ?, ?, ?, ?) @@ -3089,7 +3252,10 @@ class PersistEventsStore: def _store_room_topic_txn(self, txn: LoggingTransaction, event: EventBase) -> None: if isinstance(event.content.get("topic"), str): self.store_event_search_txn( - txn, event, "content.topic", event.content["topic"] + txn, + event, + "content.topic", + get_plain_text_topic_from_event_content(event.content) or "", ) def _store_room_name_txn(self, txn: LoggingTransaction, event: EventBase) -> None: @@ -3170,8 +3336,8 @@ class PersistEventsStore: def _set_push_actions_for_event_and_users_txn( self, txn: LoggingTransaction, - events_and_contexts: List[Tuple[EventBase, EventContext]], - all_events_and_contexts: List[Tuple[EventBase, EventContext]], + events_and_contexts: List[EventPersistencePair], + all_events_and_contexts: List[EventPersistencePair], ) -> None: """Handles moving push actions from staging table to main event_push_actions table for all events in `events_and_contexts`. @@ -3254,7 +3420,7 @@ class PersistEventsStore: def _store_event_state_mappings_txn( self, txn: LoggingTransaction, - events_and_contexts: Collection[Tuple[EventBase, EventContext]], + events_and_contexts: Collection[EventPersistencePair], ) -> None: """ Raises: @@ -3435,8 +3601,7 @@ class PersistEventsStore: # Delete all these events that we've already fetched and now know that their # prev events are the new backwards extremeties. query = ( - "DELETE FROM event_backward_extremities" - " WHERE event_id = ? AND room_id = ?" + "DELETE FROM event_backward_extremities WHERE event_id = ? AND room_id = ?" ) backward_extremity_tuples_to_remove = [ (ev.event_id, ev.room_id) diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 4b0bdd79c6..8a59091da0 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -24,7 +24,12 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, cast import attr -from synapse.api.constants import EventContentFields, Membership, RelationTypes +from synapse.api.constants import ( + MAX_DEPTH, + EventContentFields, + Membership, + RelationTypes, +) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import EventBase, make_event_from_dict from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause @@ -289,6 +294,27 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS where_clause="NOT outlier", ) + # These indices are needed to validate the foreign key constraint + # when events are deleted. + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="current_state_events_stream_ordering_idx", + table="current_state_events", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="room_memberships_stream_ordering_idx", + table="room_memberships", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="local_current_membership_stream_ordering_idx", + table="local_current_membership", + columns=["event_stream_ordering"], + ) + # Handle background updates for Sliding Sync tables # self.db_pool.updates.register_background_update_handler( @@ -311,6 +337,10 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS self._sliding_sync_membership_snapshots_fix_forgotten_column_bg_update, ) + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.FIXUP_MAX_DEPTH_CAP, self.fixup_max_depth_cap_bg_update + ) + # We want this to run on the main database at startup before we start processing # events. # @@ -2547,6 +2577,77 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS return num_rows + async def fixup_max_depth_cap_bg_update( + self, progress: JsonDict, batch_size: int + ) -> int: + """Fixes the topological ordering for events that have a depth greater + than MAX_DEPTH. This should fix /messages ordering oddities.""" + + room_id_bound = progress.get("room_id", "") + + def redo_max_depth_bg_update_txn(txn: LoggingTransaction) -> Tuple[bool, int]: + txn.execute( + """ + SELECT room_id, room_version FROM rooms + WHERE room_id > ? + ORDER BY room_id + LIMIT ? + """, + (room_id_bound, batch_size), + ) + + # Find the next room ID to process, with a relevant room version. + room_ids: List[str] = [] + max_room_id: Optional[str] = None + for room_id, room_version_str in txn: + max_room_id = room_id + + # We only want to process rooms with a known room version that + # has strict canonical json validation enabled. + room_version = KNOWN_ROOM_VERSIONS.get(room_version_str) + if room_version and room_version.strict_canonicaljson: + room_ids.append(room_id) + + if max_room_id is None: + # The query did not return any rooms, so we are done. + return True, 0 + + # Update the progress to the last room ID we pulled from the DB, + # this ensures we always make progress. + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.FIXUP_MAX_DEPTH_CAP, + progress={"room_id": max_room_id}, + ) + + if not room_ids: + # There were no rooms in this batch that required the fix. + return False, 0 + + clause, list_args = make_in_list_sql_clause( + self.database_engine, "room_id", room_ids + ) + sql = f""" + UPDATE events SET topological_ordering = ? + WHERE topological_ordering > ? AND {clause} + """ + args = [MAX_DEPTH, MAX_DEPTH] + args.extend(list_args) + txn.execute(sql, args) + + return False, len(room_ids) + + done, num_rooms = await self.db_pool.runInteraction( + "redo_max_depth_bg_update", redo_max_depth_bg_update_txn + ) + + if done: + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.FIXUP_MAX_DEPTH_CAP + ) + + return num_rooms + def _resolve_stale_data_in_sliding_sync_tables( txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 403407068c..cc031d8996 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -17,7 +17,7 @@ # [This file includes modifications made by New Vector Limited] # # - +import json import logging import threading import weakref @@ -30,6 +30,7 @@ from typing import ( Dict, Iterable, List, + Literal, Mapping, MutableMapping, Optional, @@ -41,7 +42,6 @@ from typing import ( import attr from prometheus_client import Gauge -from typing_extensions import Literal from twisted.internet import defer @@ -68,6 +68,7 @@ from synapse.logging.opentracing import ( tag_args, trace, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( run_as_background_process, wrap_as_background_process, @@ -80,6 +81,7 @@ from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, LoggingTransaction, + make_tuple_in_list_sql_clause, ) from synapse.storage.types import Cursor from synapse.storage.util.id_generators import ( @@ -138,6 +140,7 @@ EVENT_QUEUE_TIMEOUT_S = 0.1 # Timeout when waiting for requests for events event_fetch_ongoing_gauge = Gauge( "synapse_event_fetch_ongoing", "The number of event fetchers that are running", + labelnames=[SERVER_NAME_LABEL], ) @@ -193,6 +196,14 @@ class _EventRow: outlier: bool +@attr.s(slots=True, frozen=True, auto_attribs=True) +class EventMetadata: + """Event metadata returned by `get_metadata_for_event(..)`""" + + sender: str + received_ts: int + + class EventRedactBehaviour(Enum): """ What to do when retrieving a redacted event from the database. @@ -227,6 +238,7 @@ class EventsWorkerStore(SQLBaseStore): db=database, notifier=hs.get_replication_notifier(), stream_name="events", + server_name=self.server_name, instance_name=hs.get_instance_name(), tables=[ ("events", "instance_name", "stream_ordering"), @@ -241,6 +253,7 @@ class EventsWorkerStore(SQLBaseStore): db=database, notifier=hs.get_replication_notifier(), stream_name="backfill", + server_name=self.server_name, instance_name=hs.get_instance_name(), tables=[ ("events", "instance_name", "stream_ordering"), @@ -261,8 +274,9 @@ class EventsWorkerStore(SQLBaseStore): limit=1000, ) self._curr_state_delta_stream_cache: StreamChangeCache = StreamChangeCache( - "_curr_state_delta_stream_cache", - min_curr_state_delta_id, + name="_curr_state_delta_stream_cache", + server_name=self.server_name, + current_stream_pos=min_curr_state_delta_id, prefilled_cache=curr_state_delta_prefill, ) @@ -275,6 +289,7 @@ class EventsWorkerStore(SQLBaseStore): self._get_event_cache: AsyncLruCache[Tuple[str], EventCacheEntry] = ( AsyncLruCache( + server_name=self.server_name, cache_name="*getEvent*", max_size=hs.config.caches.event_cache_size, # `extra_index_cb` Returns a tuple as that is the key type @@ -300,7 +315,9 @@ class EventsWorkerStore(SQLBaseStore): Tuple[Iterable[str], "defer.Deferred[Dict[str, _EventRow]]"] ] = [] self._event_fetch_ongoing = 0 - event_fetch_ongoing_gauge.set(self._event_fetch_ongoing) + event_fetch_ongoing_gauge.labels(**{SERVER_NAME_LABEL: self.server_name}).set( + self._event_fetch_ongoing + ) # We define this sequence here so that it can be referenced from both # the DataStore and PersistEventStore. @@ -324,6 +341,7 @@ class EventsWorkerStore(SQLBaseStore): db=database, notifier=hs.get_replication_notifier(), stream_name="un_partial_stated_event_stream", + server_name=self.server_name, instance_name=hs.get_instance_name(), tables=[("un_partial_stated_event_stream", "instance_name", "stream_id")], sequence_name="un_partial_stated_event_stream_sequence", @@ -331,6 +349,35 @@ class EventsWorkerStore(SQLBaseStore): writers=["master"], ) + # Added to accommodate some queries for the admin API in order to fetch/filter + # membership events by when it was received + self.db_pool.updates.register_background_index_update( + update_name="events_received_ts_index", + index_name="received_ts_idx", + table="events", + columns=("received_ts",), + where_clause="type = 'm.room.member'", + ) + + # Added to support efficient reverse lookups on the foreign key + # (user_id, device_id) when deleting devices. + # We already had a UNIQUE index on these 4 columns but out-of-order + # so replace that one. + self.db_pool.updates.register_background_index_update( + update_name="event_txn_id_device_id_txn_id2", + index_name="event_txn_id_device_id_txn_id2", + table="event_txn_id_device_id", + columns=("user_id", "device_id", "room_id", "txn_id"), + unique=True, + replaces_index="event_txn_id_device_id_txn_id", + ) + + self._has_finished_sliding_sync_background_jobs = False + """ + Flag to track when the sliding sync background jobs have + finished (so we don't have to keep querying it every time) + """ + def get_un_partial_stated_events_token(self, instance_name: str) -> int: return ( self._un_partial_stated_events_stream_id_gen.get_current_token_for_writer( @@ -806,9 +853,9 @@ class EventsWorkerStore(SQLBaseStore): if missing_events_ids: - async def get_missing_events_from_cache_or_db() -> ( - Dict[str, EventCacheEntry] - ): + async def get_missing_events_from_cache_or_db() -> Dict[ + str, EventCacheEntry + ]: """Fetches the events in `missing_event_ids` from the database. Also creates entries in `self._current_event_fetches` to allow @@ -943,6 +990,13 @@ class EventsWorkerStore(SQLBaseStore): self._event_ref.clear() self._current_event_fetches.clear() + def _invalidate_async_get_event_cache_room_id(self, room_id: str) -> None: + """ + Clears the async get_event cache for a room. Currently a no-op until + an async get_event cache is implemented - see https://github.com/matrix-org/synapse/pull/13242 + for preliminary work. + """ + async def _get_events_from_cache( self, events: Iterable[str], update_metrics: bool = True ) -> Dict[str, EventCacheEntry]: @@ -1091,14 +1145,18 @@ class EventsWorkerStore(SQLBaseStore): and self._event_fetch_ongoing < EVENT_QUEUE_THREADS ): self._event_fetch_ongoing += 1 - event_fetch_ongoing_gauge.set(self._event_fetch_ongoing) + event_fetch_ongoing_gauge.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(self._event_fetch_ongoing) # `_event_fetch_ongoing` is decremented in `_fetch_thread`. should_start = True else: should_start = False if should_start: - run_as_background_process("fetch_events", self._fetch_thread) + run_as_background_process( + "fetch_events", self.server_name, self._fetch_thread + ) async def _fetch_thread(self) -> None: """Services requests for events from `_event_fetch_list`.""" @@ -1113,7 +1171,9 @@ class EventsWorkerStore(SQLBaseStore): event_fetches_to_fail = [] with self._event_fetch_lock: self._event_fetch_ongoing -= 1 - event_fetch_ongoing_gauge.set(self._event_fetch_ongoing) + event_fetch_ongoing_gauge.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(self._event_fetch_ongoing) # There may still be work remaining in `_event_fetch_list` if we # failed, or it was added in between us deciding to exit and @@ -1215,7 +1275,9 @@ class EventsWorkerStore(SQLBaseStore): to event row. Note that it may well contain additional events that were not part of this request. """ - with Measure(self._clock, "_fetch_event_list"): + with Measure( + self._clock, name="_fetch_event_list", server_name=self.server_name + ): try: events_to_fetch = { event_id for events, _ in event_list for event_id in events @@ -1276,6 +1338,7 @@ class EventsWorkerStore(SQLBaseStore): fetched_event_ids: Set[str] = set() fetched_events: Dict[str, _EventRow] = {} + @trace async def _fetch_event_ids_and_get_outstanding_redactions( event_ids_to_fetch: Collection[str], ) -> Collection[str]: @@ -1283,6 +1346,10 @@ class EventsWorkerStore(SQLBaseStore): Fetch all of the given event_ids and return any associated redaction event_ids that we still need to fetch in the next iteration. """ + set_tag( + SynapseTags.FUNC_ARG_PREFIX + "event_ids_to_fetch.length", + str(len(event_ids_to_fetch)), + ) row_map = await self._enqueue_events(event_ids_to_fetch) # we need to recursively fetch any redactions of those events @@ -1540,6 +1607,51 @@ class EventsWorkerStore(SQLBaseStore): if d: d.redactions.append(redacter) + # check for MSC4932 redactions + to_check = [] + events: List[_EventRow] = [] + for e in evs: + event = event_dict.get(e) + if not event: + continue + events.append(event) + event_json = json.loads(event.json) + room_id = event_json.get("room_id") + user_id = event_json.get("sender") + to_check.append((room_id, user_id)) + + # likely that some of these events may be for the same room/user combo, in + # which case we don't need to do redundant queries + to_check_set = set(to_check) + room_redaction_sql = "SELECT room_id, user_id, redacting_event_id, redact_end_ordering FROM room_ban_redactions WHERE " + ( + in_list_clause, + room_redaction_args, + ) = make_tuple_in_list_sql_clause( + self.database_engine, ("room_id", "user_id"), to_check_set + ) + txn.execute(room_redaction_sql + in_list_clause, room_redaction_args) + for ( + returned_room_id, + returned_user_id, + redacting_event_id, + redact_end_ordering, + ) in txn: + for e_row in events: + e_json = json.loads(e_row.json) + room_id = e_json.get("room_id") + user_id = e_json.get("sender") + room_and_user = (returned_room_id, returned_user_id) + # check if we have a redaction match for this room, user combination + if room_and_user != (room_id, user_id): + continue + if redact_end_ordering: + # Avoid redacting any events arriving *after* the membership event which + # ends an active redaction - note that this will always redact + # backfilled events, as they have a negative stream ordering + if e_row.stream_ordering >= redact_end_ordering: + continue + e_row.redactions.append(redacting_event_id) return event_dict def _maybe_redact_event_row( @@ -2573,10 +2685,73 @@ class EventsWorkerStore(SQLBaseStore): async def have_finished_sliding_sync_background_jobs(self) -> bool: """Return if it's safe to use the sliding sync membership tables.""" - return await self.db_pool.updates.have_completed_background_updates( + if self._has_finished_sliding_sync_background_jobs: + # as an optimisation, once the job finishes, don't issue another + # database transaction to check it, since it won't 'un-finish' + return True + + self._has_finished_sliding_sync_background_jobs = await self.db_pool.updates.have_completed_background_updates( ( _BackgroundUpdates.SLIDING_SYNC_PREFILL_JOINED_ROOMS_TO_RECALCULATE_TABLE_BG_UPDATE, _BackgroundUpdates.SLIDING_SYNC_JOINED_ROOMS_BG_UPDATE, _BackgroundUpdates.SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_BG_UPDATE, ) ) + return self._has_finished_sliding_sync_background_jobs + + async def get_sent_invite_count_by_user(self, user_id: str, from_ts: int) -> int: + """ + Get the number of invites sent by the given user at or after the provided timestamp. + + Args: + user_id: user ID to search against + from_ts: a timestamp in milliseconds from the unix epoch. Filters against + `events.received_ts` + + """ + + def _get_sent_invite_count_by_user_txn( + txn: LoggingTransaction, user_id: str, from_ts: int + ) -> int: + sql = """ + SELECT COUNT(rm.event_id) + FROM room_memberships AS rm + INNER JOIN events AS e USING(event_id) + WHERE rm.sender = ? + AND rm.membership = 'invite' + AND e.type = 'm.room.member' + AND e.received_ts >= ? + """ + + txn.execute(sql, (user_id, from_ts)) + res = txn.fetchone() + + if res is None: + return 0 + return int(res[0]) + + return await self.db_pool.runInteraction( + "_get_sent_invite_count_by_user_txn", + _get_sent_invite_count_by_user_txn, + user_id, + from_ts, + ) + + @cached(tree=True) + async def get_metadata_for_event( + self, room_id: str, event_id: str + ) -> Optional[EventMetadata]: + row = await self.db_pool.simple_select_one( + table="events", + keyvalues={"room_id": room_id, "event_id": event_id}, + retcols=("sender", "received_ts"), + allow_none=True, + desc="get_metadata_for_event", + ) + if row is None: + return None + + return EventMetadata( + sender=row[0], + received_ts=row[1], + ) diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index 8277ad8c33..e733f65cb1 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -24,9 +24,13 @@ from types import TracebackType from typing import TYPE_CHECKING, Collection, Optional, Set, Tuple, Type from weakref import WeakValueDictionary +from twisted.internet import defer from twisted.internet.task import LoopingCall -from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.metrics.background_process_metrics import ( + run_as_background_process, + wrap_as_background_process, +) from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( DatabasePool, @@ -196,6 +200,7 @@ class LockStore(SQLBaseStore): return None lock = Lock( + self.server_name, self._reactor, self._clock, self, @@ -263,6 +268,7 @@ class LockStore(SQLBaseStore): ) lock = Lock( + self.server_name, self._reactor, self._clock, self, @@ -366,6 +372,7 @@ class Lock: def __init__( self, + server_name: str, reactor: ISynapseReactor, clock: Clock, store: LockStore, @@ -374,6 +381,11 @@ class Lock: lock_key: str, token: str, ) -> None: + """ + Args: + server_name: The homeserver name (used to label metrics) (this should be `hs.hostname`). + """ + self._server_name = server_name self._reactor = reactor self._clock = clock self._store = store @@ -396,6 +408,7 @@ class Lock: self._looping_call = self._clock.looping_call( self._renew, _RENEWAL_INTERVAL_MS, + self._server_name, self._store, self._clock, self._read_write, @@ -405,31 +418,55 @@ class Lock: ) @staticmethod - @wrap_as_background_process("Lock._renew") - async def _renew( + def _renew( + server_name: str, store: LockStore, clock: Clock, read_write: bool, lock_name: str, lock_key: str, token: str, - ) -> None: + ) -> "defer.Deferred[None]": """Renew the lock. Note: this is a static method, rather than using self.*, so that we don't end up with a reference to `self` in the reactor, which would stop this from being cleaned up if we dropped the context manager. + + Args: + server_name: The homeserver name (used to label metrics) (this should be `hs.hostname`). """ - table = "worker_read_write_locks" if read_write else "worker_locks" - await store.db_pool.simple_update( - table=table, - keyvalues={ - "lock_name": lock_name, - "lock_key": lock_key, - "token": token, - }, - updatevalues={"last_renewed_ts": clock.time_msec()}, - desc="renew_lock", + + async def _internal_renew( + store: LockStore, + clock: Clock, + read_write: bool, + lock_name: str, + lock_key: str, + token: str, + ) -> None: + table = "worker_read_write_locks" if read_write else "worker_locks" + await store.db_pool.simple_update( + table=table, + keyvalues={ + "lock_name": lock_name, + "lock_key": lock_key, + "token": token, + }, + updatevalues={"last_renewed_ts": clock.time_msec()}, + desc="renew_lock", + ) + + return run_as_background_process( + "Lock._renew", + server_name, + _internal_renew, + store, + clock, + read_write, + lock_name, + lock_key, + token, ) async def is_still_valid(self) -> bool: diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 7a96e25432..f726846e57 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -19,6 +19,7 @@ # [This file includes modifications made by New Vector Limited] # # +import logging from enum import Enum from typing import ( TYPE_CHECKING, @@ -51,6 +52,8 @@ BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD_2 = ( "media_repository_drop_index_wo_method_2" ) +logger = logging.getLogger(__name__) + @attr.s(slots=True, frozen=True, auto_attribs=True) class LocalMedia: @@ -65,6 +68,7 @@ class LocalMedia: safe_from_quarantine: bool user_id: Optional[str] authenticated: Optional[bool] + sha256: Optional[str] @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -79,6 +83,7 @@ class RemoteMedia: last_access_ts: int quarantined_by: Optional[str] authenticated: Optional[bool] + sha256: Optional[str] @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -154,6 +159,26 @@ class MediaRepositoryBackgroundUpdateStore(SQLBaseStore): unique=True, ) + self.db_pool.updates.register_background_index_update( + update_name="local_media_repository_sha256_idx", + index_name="local_media_repository_sha256", + table="local_media_repository", + where_clause="sha256 IS NOT NULL", + columns=[ + "sha256", + ], + ) + + self.db_pool.updates.register_background_index_update( + update_name="remote_media_cache_sha256_idx", + index_name="remote_media_cache_sha256", + table="remote_media_cache", + where_clause="sha256 IS NOT NULL", + columns=[ + "sha256", + ], + ) + self.db_pool.updates.register_background_update_handler( BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD_2, self._drop_media_index_without_method, @@ -221,6 +246,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): "safe_from_quarantine", "user_id", "authenticated", + "sha256", ), allow_none=True, desc="get_local_media", @@ -239,6 +265,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): safe_from_quarantine=row[7], user_id=row[8], authenticated=row[9], + sha256=row[10], ) async def get_local_media_by_user_paginate( @@ -295,7 +322,8 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): quarantined_by, safe_from_quarantine, user_id, - authenticated + authenticated, + sha256 FROM local_media_repository WHERE user_id = ? ORDER BY {order_by_column} {order}, media_id ASC @@ -320,6 +348,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): safe_from_quarantine=bool(row[8]), user_id=row[9], authenticated=row[10], + sha256=row[11], ) for row in txn ] @@ -449,6 +478,8 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): media_length: int, user_id: UserID, url_cache: Optional[str] = None, + sha256: Optional[str] = None, + quarantined_by: Optional[str] = None, ) -> None: if self.hs.config.media.enable_authenticated_media: authenticated = True @@ -466,6 +497,8 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): "user_id": user_id.to_string(), "url_cache": url_cache, "authenticated": authenticated, + "sha256": sha256, + "quarantined_by": quarantined_by, }, desc="store_local_media", ) @@ -477,20 +510,28 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): upload_name: Optional[str], media_length: int, user_id: UserID, + sha256: str, url_cache: Optional[str] = None, + quarantined_by: Optional[str] = None, ) -> None: + updatevalues = { + "media_type": media_type, + "upload_name": upload_name, + "media_length": media_length, + "url_cache": url_cache, + "sha256": sha256, + } + + # This should never be un-set by this function. + if quarantined_by is not None: + updatevalues["quarantined_by"] = quarantined_by + await self.db_pool.simple_update_one( "local_media_repository", keyvalues={ - "user_id": user_id.to_string(), "media_id": media_id, }, - updatevalues={ - "media_type": media_type, - "upload_name": upload_name, - "media_length": media_length, - "url_cache": url_cache, - }, + updatevalues=updatevalues, desc="update_local_media", ) @@ -657,6 +698,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): "last_access_ts", "quarantined_by", "authenticated", + "sha256", ), allow_none=True, desc="get_cached_remote_media", @@ -674,6 +716,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): last_access_ts=row[5], quarantined_by=row[6], authenticated=row[7], + sha256=row[8], ) async def store_cached_remote_media( @@ -685,6 +728,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): time_now_ms: int, upload_name: Optional[str], filesystem_id: str, + sha256: Optional[str], ) -> None: if self.hs.config.media.enable_authenticated_media: authenticated = True @@ -703,6 +747,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): "filesystem_id": filesystem_id, "last_access_ts": time_now_ms, "authenticated": authenticated, + "sha256": sha256, }, desc="store_cached_remote_media", ) @@ -946,3 +991,82 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): await self.db_pool.runInteraction( "delete_url_cache_media", _delete_url_cache_media_txn ) + + async def get_is_hash_quarantined(self, sha256: str) -> bool: + """Get whether a specific sha256 hash digest matches any quarantined media. + + Returns: + None if the media_id doesn't exist. + """ + + # If we don't have the index yet, performance tanks, so we return False. + # In the background updates, remote_media_cache_sha256_idx is created + # after local_media_repository_sha256_idx, which is why we only need to + # check for the completion of the former. + if not await self.db_pool.updates.has_completed_background_update( + "remote_media_cache_sha256_idx" + ): + return False + + def get_matching_media_txn( + txn: LoggingTransaction, table: str, sha256: str + ) -> bool: + # Return on first match + sql = """ + SELECT 1 + FROM local_media_repository + WHERE sha256 = ? AND quarantined_by IS NOT NULL + + UNION ALL + + SELECT 1 + FROM remote_media_cache + WHERE sha256 = ? AND quarantined_by IS NOT NULL + LIMIT 1 + """ + txn.execute(sql, (sha256, sha256)) + row = txn.fetchone() + return row is not None + + return await self.db_pool.runInteraction( + "get_matching_media_txn", + get_matching_media_txn, + "local_media_repository", + sha256, + ) + + async def get_media_uploaded_size_for_user( + self, user_id: str, time_period_ms: int + ) -> int: + """Get the total size of media uploaded by a user in the last + time_period_ms milliseconds. + + Args: + user_id: The user ID to check. + time_period_ms: The time period in milliseconds to consider. + + Returns: + The total size of media uploaded by the user in bytes. + """ + + sql = """ + SELECT COALESCE(SUM(media_length), 0) + FROM local_media_repository + WHERE user_id = ? AND created_ts > ? + """ + + def _get_media_uploaded_size_for_user_txn( + txn: LoggingTransaction, + ) -> int: + # Calculate the timestamp for the start of the time period + start_ts = self._clock.time_msec() - time_period_ms + txn.execute(sql, (user_id, start_ts)) + row = txn.fetchone() + if row is None: + return 0 + return row[0] + + return await self.db_pool.runInteraction( + "get_media_uploaded_size_for_user", + _get_media_uploaded_size_for_user_txn, + ) diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index 9ce1100b5c..a3467bff3d 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -23,7 +23,7 @@ import logging import time from typing import TYPE_CHECKING, Dict, List, Tuple, cast -from synapse.metrics import GaugeBucketCollector +from synapse.metrics import SERVER_NAME_LABEL, GaugeBucketCollector from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( @@ -42,9 +42,10 @@ logger = logging.getLogger(__name__) # Collect metrics on the number of forward extremities that exist. _extremities_collecter = GaugeBucketCollector( - "synapse_forward_extremities", - "Number of rooms on the server with the given number of forward extremities" + name="synapse_forward_extremities", + documentation="Number of rooms on the server with the given number of forward extremities" " or fewer", + labelnames=[SERVER_NAME_LABEL], buckets=[1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500], ) @@ -54,9 +55,10 @@ _extremities_collecter = GaugeBucketCollector( # we could remove from state resolution by reducing the graph to a single # forward extremity. _excess_state_events_collecter = GaugeBucketCollector( - "synapse_excess_extremity_events", - "Number of rooms on the server with the given number of excess extremity " + name="synapse_excess_extremity_events", + documentation="Number of rooms on the server with the given number of excess extremity " "events, or fewer", + labelnames=[SERVER_NAME_LABEL], buckets=[0] + [1 << n for n in range(12)], ) @@ -100,10 +102,12 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore): res = await self.db_pool.runInteraction("read_forward_extremities", fetch) - _extremities_collecter.update_data(x[0] for x in res) + _extremities_collecter.update_data( + values=(x[0] for x in res), labels=(self.server_name,) + ) _excess_state_events_collecter.update_data( - (x[0] - 1) * x[1] for x in res if x[1] + values=((x[0] - 1) * x[1] for x in res if x[1]), labels=(self.server_name,) ) async def count_daily_e2ee_messages(self) -> int: diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index 8e948c5e8d..f5a6b98be7 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -304,9 +304,9 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): txn: threepids: List of threepid dicts to reserve """ - assert ( - self._update_on_this_worker - ), "This worker is not designated to update MAUs" + assert self._update_on_this_worker, ( + "This worker is not designated to update MAUs" + ) # XXX what is this function trying to achieve? It upserts into # monthly_active_users for each *registered* reserved mau user, but why? @@ -331,7 +331,7 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): values={"timestamp": int(self._clock.time_msec())}, ) else: - logger.warning("mau limit reserved threepid %s not found in db" % tp) + logger.warning("mau limit reserved threepid %s not found in db", tp) async def upsert_monthly_active_user(self, user_id: str) -> None: """Updates or inserts the user into the monthly active user table, which @@ -340,9 +340,9 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): Args: user_id: user to add/update """ - assert ( - self._update_on_this_worker - ), "This worker is not designated to update MAUs" + assert self._update_on_this_worker, ( + "This worker is not designated to update MAUs" + ) # Support user never to be included in MAU stats. Note I can't easily call this # from upsert_monthly_active_user_txn because then I need a _txn form of @@ -379,9 +379,9 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): txn: user_id: user to add/update """ - assert ( - self._update_on_this_worker - ), "This worker is not designated to update MAUs" + assert self._update_on_this_worker, ( + "This worker is not designated to update MAUs" + ) # Am consciously deciding to lock the table on the basis that is ought # never be a big table and alternative approaches (batching multiple @@ -409,9 +409,9 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): Args: user_id: the user_id to query """ - assert ( - self._update_on_this_worker - ), "This worker is not designated to update MAUs" + assert self._update_on_this_worker, ( + "This worker is not designated to update MAUs" + ) if self._limit_usage_by_mau or self._mau_stats_only: # Trial users and guests should not be included as part of MAU group diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py index 065c885603..587f51df2c 100644 --- a/synapse/storage/databases/main/presence.py +++ b/synapse/storage/databases/main/presence.py @@ -91,6 +91,7 @@ class PresenceStore(PresenceBackgroundUpdateStore, CacheInvalidationWorkerStore) db=database, notifier=hs.get_replication_notifier(), stream_name="presence_stream", + server_name=self.server_name, instance_name=self._instance_name, tables=[("presence_stream", "instance_name", "stream_id")], sequence_name="presence_stream_sequence", @@ -108,8 +109,9 @@ class PresenceStore(PresenceBackgroundUpdateStore, CacheInvalidationWorkerStore) max_value=self._presence_id_gen.get_current_token(), ) self.presence_stream_cache = StreamChangeCache( - "PresenceStreamChangeCache", - min_presence_val, + name="PresenceStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_presence_val, prefilled_cache=presence_cache_prefill, ) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 41cf08211f..30d8a58d96 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -18,8 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import TYPE_CHECKING, Optional +import json +from typing import TYPE_CHECKING, Dict, Optional, Tuple, cast +from canonicaljson import encode_canonical_json + +from synapse.api.constants import ProfileFields +from synapse.api.errors import Codes, StoreError from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( DatabasePool, @@ -27,13 +32,17 @@ from synapse.storage.database import ( LoggingTransaction, ) from synapse.storage.databases.main.roommember import ProfileInfo -from synapse.storage.engines import PostgresEngine -from synapse.types import JsonDict, UserID +from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.types import JsonDict, JsonValue, UserID if TYPE_CHECKING: from synapse.server import HomeServer +# The number of bytes that the serialized profile can have. +MAX_PROFILE_SIZE = 65536 + + class ProfileWorkerStore(SQLBaseStore): def __init__( self, @@ -201,6 +210,89 @@ class ProfileWorkerStore(SQLBaseStore): desc="get_profile_avatar_url", ) + async def get_profile_field(self, user_id: UserID, field_name: str) -> JsonValue: + """ + Get a custom profile field for a user. + + Args: + user_id: The user's ID. + field_name: The custom profile field name. + + Returns: + The string value if the field exists, otherwise raises 404. + """ + + def get_profile_field(txn: LoggingTransaction) -> JsonValue: + # This will error if field_name has double quotes in it, but that's not + # possible due to the grammar. + field_path = f'$."{field_name}"' + + if isinstance(self.database_engine, PostgresEngine): + sql = """ + SELECT JSONB_PATH_EXISTS(fields, ?), JSONB_EXTRACT_PATH(fields, ?) + FROM profiles + WHERE user_id = ? + """ + txn.execute( + sql, + (field_path, field_name, user_id.localpart), + ) + + # Test exists first since value being None is used for both + # missing and a null JSON value. + exists, value = cast(Tuple[bool, JsonValue], txn.fetchone()) + if not exists: + raise StoreError(404, "No row found") + return value + + else: + sql = """ + SELECT JSON_TYPE(fields, ?), JSON_EXTRACT(fields, ?) + FROM profiles + WHERE user_id = ? + """ + txn.execute( + sql, + (field_path, field_path, user_id.localpart), + ) + + # If value_type is None, then the value did not exist. + value_type, value = cast( + Tuple[Optional[str], JsonValue], txn.fetchone() + ) + if not value_type: + raise StoreError(404, "No row found") + # If value_type is object or array, then need to deserialize the JSON. + # Scalar values are properly returned directly. + if value_type in ("object", "array"): + assert isinstance(value, str) + return json.loads(value) + return value + + return await self.db_pool.runInteraction("get_profile_field", get_profile_field) + + async def get_profile_fields(self, user_id: UserID) -> Dict[str, str]: + """ + Get all custom profile fields for a user. + + Args: + user_id: The user's ID. + + Returns: + A dictionary of custom profile fields. + """ + result = await self.db_pool.simple_select_one_onecol( + table="profiles", + keyvalues={"full_user_id": user_id.to_string()}, + retcol="fields", + desc="get_profile_fields", + ) + # The SQLite driver doesn't automatically convert JSON to + # Python objects + if isinstance(self.database_engine, Sqlite3Engine) and result: + result = json.loads(result) + return result or {} + async def create_profile(self, user_id: UserID) -> None: """ Create a blank profile for a user. @@ -215,6 +307,71 @@ class ProfileWorkerStore(SQLBaseStore): desc="create_profile", ) + def _check_profile_size( + self, + txn: LoggingTransaction, + user_id: UserID, + new_field_name: str, + new_value: JsonValue, + ) -> None: + # For each entry there are 4 quotes (2 each for key and value), 1 colon, + # and 1 comma. + PER_VALUE_EXTRA = 6 + + # Add the size of the current custom profile fields, ignoring the entry + # which will be overwritten. + if isinstance(txn.database_engine, PostgresEngine): + size_sql = """ + SELECT + OCTET_LENGTH((fields - ?)::text), OCTET_LENGTH(displayname), OCTET_LENGTH(avatar_url) + FROM profiles + WHERE + user_id = ? + """ + txn.execute( + size_sql, + (new_field_name, user_id.localpart), + ) + else: + size_sql = """ + SELECT + LENGTH(json_remove(fields, ?)), LENGTH(displayname), LENGTH(avatar_url) + FROM profiles + WHERE + user_id = ? + """ + txn.execute( + size_sql, + # This will error if field_name has double quotes in it, but that's not + # possible due to the grammar. + (f'$."{new_field_name}"', user_id.localpart), + ) + row = cast(Tuple[Optional[int], Optional[int], Optional[int]], txn.fetchone()) + + # The values return null if the column is null. + total_bytes = ( + # Discount the opening and closing braces to avoid double counting, + # but add one for a comma. + # -2 + 1 = -1 + (row[0] - 1 if row[0] else 0) + + ( + row[1] + len("displayname") + PER_VALUE_EXTRA + if new_field_name != ProfileFields.DISPLAYNAME and row[1] + else 0 + ) + + ( + row[2] + len("avatar_url") + PER_VALUE_EXTRA + if new_field_name != ProfileFields.AVATAR_URL and row[2] + else 0 + ) + ) + + # Add the length of the field being added + the braces. + total_bytes += len(encode_canonical_json({new_field_name: new_value})) + + if total_bytes > MAX_PROFILE_SIZE: + raise StoreError(400, "Profile too large", Codes.PROFILE_TOO_LARGE) + async def set_profile_displayname( self, user_id: UserID, new_displayname: Optional[str] ) -> None: @@ -227,14 +384,25 @@ class ProfileWorkerStore(SQLBaseStore): name is removed. """ user_localpart = user_id.localpart - await self.db_pool.simple_upsert( - table="profiles", - keyvalues={"user_id": user_localpart}, - values={ - "displayname": new_displayname, - "full_user_id": user_id.to_string(), - }, - desc="set_profile_displayname", + + def set_profile_displayname(txn: LoggingTransaction) -> None: + if new_displayname is not None: + self._check_profile_size( + txn, user_id, ProfileFields.DISPLAYNAME, new_displayname + ) + + self.db_pool.simple_upsert_txn( + txn, + table="profiles", + keyvalues={"user_id": user_localpart}, + values={ + "displayname": new_displayname, + "full_user_id": user_id.to_string(), + }, + ) + + await self.db_pool.runInteraction( + "set_profile_displayname", set_profile_displayname ) async def set_profile_avatar_url( @@ -249,13 +417,125 @@ class ProfileWorkerStore(SQLBaseStore): removed. """ user_localpart = user_id.localpart - await self.db_pool.simple_upsert( - table="profiles", - keyvalues={"user_id": user_localpart}, - values={"avatar_url": new_avatar_url, "full_user_id": user_id.to_string()}, - desc="set_profile_avatar_url", + + def set_profile_avatar_url(txn: LoggingTransaction) -> None: + if new_avatar_url is not None: + self._check_profile_size( + txn, user_id, ProfileFields.AVATAR_URL, new_avatar_url + ) + + self.db_pool.simple_upsert_txn( + txn, + table="profiles", + keyvalues={"user_id": user_localpart}, + values={ + "avatar_url": new_avatar_url, + "full_user_id": user_id.to_string(), + }, + ) + + await self.db_pool.runInteraction( + "set_profile_avatar_url", set_profile_avatar_url ) + async def set_profile_field( + self, user_id: UserID, field_name: str, new_value: JsonValue + ) -> None: + """ + Set a custom profile field for a user. + + Args: + user_id: The user's ID. + field_name: The name of the custom profile field. + new_value: The value of the custom profile field. + """ + + # Encode to canonical JSON. + canonical_value = encode_canonical_json(new_value) + + def set_profile_field(txn: LoggingTransaction) -> None: + self._check_profile_size(txn, user_id, field_name, new_value) + + if isinstance(self.database_engine, PostgresEngine): + from psycopg2.extras import Json + + # Note that the || jsonb operator is not recursive, any duplicate + # keys will be taken from the second value. + sql = """ + INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_BUILD_OBJECT(?, ?::jsonb)) + ON CONFLICT (user_id) + DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = COALESCE(profiles.fields, '{}'::jsonb) || EXCLUDED.fields + """ + + txn.execute( + sql, + ( + user_id.localpart, + user_id.to_string(), + field_name, + # Pass as a JSON object since we have passing bytes disabled + # at the database driver. + Json(json.loads(canonical_value)), + ), + ) + else: + # You may be tempted to use json_patch instead of providing the parameters + # twice, but that recursively merges objects instead of replacing. + sql = """ + INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_OBJECT(?, JSON(?))) + ON CONFLICT (user_id) + DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = JSON_SET(COALESCE(profiles.fields, '{}'), ?, JSON(?)) + """ + # This will error if field_name has double quotes in it, but that's not + # possible due to the grammar. + json_field_name = f'$."{field_name}"' + + txn.execute( + sql, + ( + user_id.localpart, + user_id.to_string(), + json_field_name, + canonical_value, + json_field_name, + canonical_value, + ), + ) + + await self.db_pool.runInteraction("set_profile_field", set_profile_field) + + async def delete_profile_field(self, user_id: UserID, field_name: str) -> None: + """ + Remove a custom profile field for a user. + + Args: + user_id: The user's ID. + field_name: The name of the custom profile field. + """ + + def delete_profile_field(txn: LoggingTransaction) -> None: + if isinstance(self.database_engine, PostgresEngine): + sql = """ + UPDATE profiles SET fields = fields - ? + WHERE user_id = ? + """ + txn.execute( + sql, + (field_name, user_id.localpart), + ) + else: + sql = """ + UPDATE profiles SET fields = json_remove(fields, ?) + WHERE user_id = ? + """ + txn.execute( + sql, + # This will error if field_name has double quotes in it. + (f'$."{field_name}"', user_id.localpart), + ) + + await self.db_pool.runInteraction("delete_profile_field", delete_profile_field) + class ProfileStore(ProfileWorkerStore): pass diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index 08244153a3..d4642a1309 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -20,7 +20,7 @@ # import logging -from typing import Any, List, Set, Tuple, cast +from typing import Any, Set, Tuple, cast from synapse.api.errors import SynapseError from synapse.storage.database import LoggingTransaction @@ -33,6 +33,73 @@ from synapse.types import RoomStreamToken logger = logging.getLogger(__name__) +purge_room_tables_with_event_id_index = ( + "event_auth", + "event_edges", + "event_json", + "event_push_actions_staging", + "event_relations", + "event_to_state_groups", + "event_auth_chains", + "event_auth_chain_to_calculate", + "redactions", + "rejections", + "state_events", +) +""" +Tables which lack an index on `room_id` but have one on `event_id` +""" + +purge_room_tables_with_room_id_column = ( + "current_state_events", + "destination_rooms", + "event_backward_extremities", + "event_forward_extremities", + "event_push_actions", + "event_search", + "event_failed_pull_attempts", + # Note: the partial state tables have foreign keys between each other, and to + # `events` and `rooms`. We need to delete from them in the right order. + "partial_state_events", + "partial_state_rooms_servers", + "partial_state_rooms", + # Note: the _membership(s) tables have foreign keys to the `events` table + # so must be deleted first. + "local_current_membership", + "room_memberships", + # Note: the sliding_sync_ tables have foreign keys to the `events` table + # so must be deleted first. + "sliding_sync_joined_rooms", + "sliding_sync_membership_snapshots", + "events", + "federation_inbound_events_staging", + "receipts_graph", + "receipts_linearized", + "room_aliases", + "room_depth", + "room_stats_state", + "room_stats_current", + "room_stats_earliest_token", + "stream_ordering_to_exterm", + "users_in_public_rooms", + "users_who_share_private_rooms", + # no useful index, but let's clear them anyway + "appservice_room_list", + "e2e_room_keys", + "event_push_summary", + "pusher_throttle", + "room_account_data", + "room_tags", + # "rooms" happens last, to keep the foreign keys in the other tables + # happy + "rooms", +) +""" +The tables with a `room_id` column regardless of whether they have a useful index on +`room_id`. +""" + + class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): async def purge_history( self, room_id: str, token: str, delete_local_events: bool @@ -199,8 +266,7 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): # Update backward extremeties txn.execute_batch( - "INSERT INTO event_backward_extremities (room_id, event_id)" - " VALUES (?, ?)", + "INSERT INTO event_backward_extremities (room_id, event_id) VALUES (?, ?)", [(room_id, event_id) for (event_id,) in new_backwards_extrems], ) @@ -332,7 +398,7 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): return referenced_state_groups - async def purge_room(self, room_id: str) -> List[int]: + async def purge_room(self, room_id: str) -> None: """Deletes all record of a room Args: @@ -348,7 +414,7 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): # purge any of those rows which were added during the first. logger.info("[purge] Starting initial main purge of [1/2]") - state_groups_to_delete = await self.db_pool.runInteraction( + await self.db_pool.runInteraction( "purge_room", self._purge_room_txn, room_id=room_id, @@ -356,18 +422,15 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): ) logger.info("[purge] Starting secondary main purge of [2/2]") - state_groups_to_delete.extend( - await self.db_pool.runInteraction( - "purge_room", - self._purge_room_txn, - room_id=room_id, - ), + await self.db_pool.runInteraction( + "purge_room", + self._purge_room_txn, + room_id=room_id, ) + logger.info("[purge] Done with main purge") - return state_groups_to_delete - - def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[int]: + def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> None: # This collides with event persistence so we cannot write new events and metadata into # a room while deleting it or this transaction will fail. if isinstance(self.database_engine, PostgresEngine): @@ -376,18 +439,10 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): (room_id,), ) - # First, fetch all the state groups that should be deleted, before - # we delete that information. - txn.execute( - """ - SELECT DISTINCT state_group FROM events - INNER JOIN event_to_state_groups USING(event_id) - WHERE events.room_id = ? - """, - (room_id,), - ) - - state_groups = [row[0] for row in txn] + if isinstance(self.database_engine, PostgresEngine): + # Disable statement timeouts for this transaction; purging rooms can + # take a while! + txn.execute("SET LOCAL statement_timeout = 0") # Get all the auth chains that are referenced by events that are to be # deleted. @@ -410,20 +465,8 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): referenced_chain_id_tuples, ) - # Now we delete tables which lack an index on room_id but have one on event_id - for table in ( - "event_auth", - "event_edges", - "event_json", - "event_push_actions_staging", - "event_relations", - "event_to_state_groups", - "event_auth_chains", - "event_auth_chain_to_calculate", - "redactions", - "rejections", - "state_events", - ): + # Now we delete tables which lack an index on `room_id` but have one on `event_id` + for table in purge_room_tables_with_event_id_index: logger.info("[purge] removing from %s", table) txn.execute( @@ -436,51 +479,9 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): (room_id,), ) - # next, the tables with an index on room_id (or no useful index) - for table in ( - "current_state_events", - "destination_rooms", - "event_backward_extremities", - "event_forward_extremities", - "event_push_actions", - "event_search", - "event_failed_pull_attempts", - # Note: the partial state tables have foreign keys between each other, and to - # `events` and `rooms`. We need to delete from them in the right order. - "partial_state_events", - "partial_state_rooms_servers", - "partial_state_rooms", - # Note: the _membership(s) tables have foreign keys to the `events` table - # so must be deleted first. - "local_current_membership", - "room_memberships", - # Note: the sliding_sync_ tables have foreign keys to the `events` table - # so must be deleted first. - "sliding_sync_joined_rooms", - "sliding_sync_membership_snapshots", - "events", - "federation_inbound_events_staging", - "receipts_graph", - "receipts_linearized", - "room_aliases", - "room_depth", - "room_stats_state", - "room_stats_current", - "room_stats_earliest_token", - "stream_ordering_to_exterm", - "users_in_public_rooms", - "users_who_share_private_rooms", - # no useful index, but let's clear them anyway - "appservice_room_list", - "e2e_room_keys", - "event_push_summary", - "pusher_throttle", - "room_account_data", - "room_tags", - # "rooms" happens last, to keep the foreign keys in the other tables - # happy - "rooms", - ): + # next, the tables with a `room_id` column regardless of whether they have a + # useful index on `room_id` + for table in purge_room_tables_with_room_id_column: logger.info("[purge] removing from %s", table) txn.execute("DELETE FROM %s WHERE room_id=?" % (table,), (room_id,)) @@ -508,5 +509,3 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): # periodically anyway (https://github.com/matrix-org/synapse/issues/5888) self._invalidate_caches_for_room_and_stream(txn, room_id) - - return state_groups diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index 86c87f78bf..d686140556 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -110,6 +110,7 @@ def _load_rules( msc3381_polls_enabled=experimental_config.msc3381_polls_enabled, msc4028_push_encrypted_events=experimental_config.msc4028_push_encrypted_events, msc4210_enabled=experimental_config.msc4210_enabled, + msc4306_enabled=experimental_config.msc4306_enabled, ) return filtered_rules @@ -146,6 +147,7 @@ class PushRulesWorkerStore( db=database, notifier=hs.get_replication_notifier(), stream_name="push_rules_stream", + server_name=self.server_name, instance_name=self._instance_name, tables=[ ("push_rules_stream", "instance_name", "stream_id"), @@ -163,8 +165,9 @@ class PushRulesWorkerStore( ) self.push_rules_stream_cache = StreamChangeCache( - "PushRulesStreamChangeCache", - push_rules_id, + name="PushRulesStreamChangeCache", + server_name=self.server_name, + current_stream_pos=push_rules_id, prefilled_cache=push_rules_prefill, ) diff --git a/synapse/storage/databases/main/pusher.py b/synapse/storage/databases/main/pusher.py index a8a37b6c85..9a0a12b5c1 100644 --- a/synapse/storage/databases/main/pusher.py +++ b/synapse/storage/databases/main/pusher.py @@ -88,6 +88,7 @@ class PusherWorkerStore(SQLBaseStore): db=database, notifier=hs.get_replication_notifier(), stream_name="pushers", + server_name=self.server_name, instance_name=self._instance_name, tables=[ ("pushers", "instance_name", "id"), diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 9964331510..d74bb0184a 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -36,7 +36,6 @@ from typing import ( ) import attr -from immutabledict import immutabledict from synapse.api.constants import EduTypes from synapse.replication.tcp.streams import ReceiptsStream @@ -125,6 +124,7 @@ class ReceiptsWorkerStore(SQLBaseStore): db_conn: LoggingDatabaseConnection, hs: "HomeServer", ): + super().__init__(database, db_conn, hs) self._instance_name = hs.get_instance_name() # In the worker store this is an ID tracker which we overwrite in the non-worker @@ -139,6 +139,7 @@ class ReceiptsWorkerStore(SQLBaseStore): db_conn=db_conn, db=database, notifier=hs.get_replication_notifier(), + server_name=self.server_name, stream_name="receipts", instance_name=self._instance_name, tables=[("receipts_linearized", "instance_name", "stream_id")], @@ -146,8 +147,6 @@ class ReceiptsWorkerStore(SQLBaseStore): writers=hs.config.worker.writers.receipts, ) - super().__init__(database, db_conn, hs) - max_receipts_stream_id = self.get_max_receipt_stream_id() receipts_stream_prefill, min_receipts_stream_id = self.db_pool.get_cache_dict( db_conn, @@ -158,33 +157,16 @@ class ReceiptsWorkerStore(SQLBaseStore): limit=10000, ) self._receipts_stream_cache = StreamChangeCache( - "ReceiptsRoomChangeCache", - min_receipts_stream_id, + name="ReceiptsRoomChangeCache", + server_name=self.server_name, + current_stream_pos=min_receipts_stream_id, prefilled_cache=receipts_stream_prefill, ) def get_max_receipt_stream_id(self) -> MultiWriterStreamToken: """Get the current max stream ID for receipts stream""" - min_pos = self._receipts_id_gen.get_current_token() - - positions = {} - if isinstance(self._receipts_id_gen, MultiWriterIdGenerator): - # The `min_pos` is the minimum position that we know all instances - # have finished persisting to, so we only care about instances whose - # positions are ahead of that. (Instance positions can be behind the - # min position as there are times we can work out that the minimum - # position is ahead of the naive minimum across all current - # positions. See MultiWriterIdGenerator for details) - positions = { - i: p - for i, p in self._receipts_id_gen.get_positions().items() - if p > min_pos - } - - return MultiWriterStreamToken( - stream=min_pos, instance_map=immutabledict(positions) - ) + return MultiWriterStreamToken.from_generator(self._receipts_id_gen) def get_receipt_stream_id_for_instance(self, instance_name: str) -> int: return self._receipts_id_gen.get_current_token_for_writer(instance_name) diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index d7cbe33411..117444e7b7 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -40,14 +40,16 @@ from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, LoggingTransaction, + make_in_list_sql_clause, ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.stats import StatsStore from synapse.storage.types import Cursor from synapse.storage.util.id_generators import IdGenerator from synapse.storage.util.sequence import build_sequence_generator -from synapse.types import JsonDict, UserID, UserInfo +from synapse.types import JsonDict, StrCollection, UserID, UserInfo from synapse.util.caches.descriptors import cached +from synapse.util.iterutils import batch_iter if TYPE_CHECKING: from synapse.server import HomeServer @@ -173,7 +175,7 @@ class ThreepidValidationSession: """timestamp of when this session was validated if so""" -class RegistrationWorkerStore(CacheInvalidationWorkerStore): +class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): def __init__( self, database: DatabasePool, @@ -215,12 +217,167 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): self._set_expiration_date_when_missing, ) + # If support for MSC3866 is enabled and configured to require approval for new + # account, we will create new users with an 'approved' flag set to false. + self._require_approval = ( + hs.config.experimental.msc3866.enabled + and hs.config.experimental.msc3866.require_approval_for_new_accounts + ) + # Create a background job for culling expired 3PID validity tokens if hs.config.worker.run_background_tasks: self._clock.looping_call( self.cull_expired_threepid_validation_tokens, THIRTY_MINUTES_IN_MS ) + async def register_user( + self, + user_id: str, + password_hash: Optional[str] = None, + was_guest: bool = False, + make_guest: bool = False, + appservice_id: Optional[str] = None, + create_profile_with_displayname: Optional[str] = None, + admin: bool = False, + user_type: Optional[str] = None, + shadow_banned: bool = False, + approved: bool = False, + ) -> None: + """Attempts to register an account. + + Args: + user_id: The desired user ID to register. + password_hash: Optional. The password hash for this user. + was_guest: Whether this is a guest account being upgraded to a + non-guest account. + make_guest: True if the the new user should be guest, false to add a + regular user account. + appservice_id: The ID of the appservice registering the user. + create_profile_with_displayname: Optionally create a profile for + the user, setting their displayname to the given value + admin: is an admin user? + user_type: type of user. One of the values from api.constants.UserTypes, + a custom value set in the configuration file, or None for a normal + user. + shadow_banned: Whether the user is shadow-banned, i.e. they may be + told their requests succeeded but we ignore them. + approved: Whether to consider the user has already been approved by an + administrator. + + Raises: + StoreError if the user_id could not be registered. + """ + await self.db_pool.runInteraction( + "register_user", + self._register_user, + user_id, + password_hash, + was_guest, + make_guest, + appservice_id, + create_profile_with_displayname, + admin, + user_type, + shadow_banned, + approved, + ) + + def _register_user( + self, + txn: LoggingTransaction, + user_id: str, + password_hash: Optional[str], + was_guest: bool, + make_guest: bool, + appservice_id: Optional[str], + create_profile_with_displayname: Optional[str], + admin: bool, + user_type: Optional[str], + shadow_banned: bool, + approved: bool, + ) -> None: + user_id_obj = UserID.from_string(user_id) + + now = int(self._clock.time()) + + user_approved = approved or not self._require_approval + + try: + if was_guest: + # Ensure that the guest user actually exists + # ``allow_none=False`` makes this raise an exception + # if the row isn't in the database. + self.db_pool.simple_select_one_txn( + txn, + "users", + keyvalues={"name": user_id, "is_guest": 1}, + retcols=("name",), + allow_none=False, + ) + + self.db_pool.simple_update_one_txn( + txn, + "users", + keyvalues={"name": user_id, "is_guest": 1}, + updatevalues={ + "password_hash": password_hash, + "upgrade_ts": now, + "is_guest": 1 if make_guest else 0, + "appservice_id": appservice_id, + "admin": 1 if admin else 0, + "user_type": user_type, + "shadow_banned": shadow_banned, + "approved": user_approved, + }, + ) + else: + self.db_pool.simple_insert_txn( + txn, + "users", + values={ + "name": user_id, + "password_hash": password_hash, + "creation_ts": now, + "is_guest": 1 if make_guest else 0, + "appservice_id": appservice_id, + "admin": 1 if admin else 0, + "user_type": user_type, + "shadow_banned": shadow_banned, + "approved": user_approved, + }, + ) + + except self.database_engine.module.IntegrityError: + raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE) + + if self._account_validity_enabled: + self.set_expiration_date_for_user_txn(txn, user_id) + + if create_profile_with_displayname: + # set a default displayname serverside to avoid ugly race + # between auto-joins and clients trying to set displaynames + # + # *obviously* the 'profiles' table uses localpart for user_id + # while everything else uses the full mxid. + txn.execute( + "INSERT INTO profiles(full_user_id, user_id, displayname) VALUES (?,?,?)", + (user_id, user_id_obj.localpart, create_profile_with_displayname), + ) + + if self.hs.config.stats.stats_enabled: + # we create a new completed user statistics row + + # we don't strictly need current_token since this user really can't + # have any state deltas before now (as it is a new user), but still, + # we include it for completeness. + current_token = self._get_max_stream_id_in_current_state_deltas_txn(txn) + + self._update_stats_delta_txn( + txn, now, "user", user_id, {}, complete_with_stream_id=current_token + ) + + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + @cached() async def get_user_by_id(self, user_id: str) -> Optional[UserInfo]: """Returns info about the user account, if it exists.""" @@ -516,7 +673,8 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): desc="delete_account_validity_for_user", ) - async def is_server_admin(self, user: UserID) -> bool: + @cached(max_entries=100000) + async def is_server_admin(self, user: str) -> bool: """Determines if a user is an admin of this homeserver. Args: @@ -527,7 +685,7 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): """ res = await self.db_pool.simple_select_one_onecol( table="users", - keyvalues={"name": user.to_string()}, + keyvalues={"name": user}, retcol="admin", allow_none=True, desc="is_server_admin", @@ -550,6 +708,9 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): self._invalidate_cache_and_stream( txn, self.get_user_by_id, (user.to_string(),) ) + self._invalidate_cache_and_stream( + txn, self.is_server_admin, (user.to_string(),) + ) await self.db_pool.runInteraction("set_server_admin", set_server_admin_txn) @@ -583,7 +744,9 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): await self.db_pool.runInteraction("set_shadow_banned", set_shadow_banned_txn) - async def set_user_type(self, user: UserID, user_type: Optional[UserTypes]) -> None: + async def set_user_type( + self, user: UserID, user_type: Optional[Union[UserTypes, str]] + ) -> None: """Sets the user type. Args: @@ -683,7 +846,7 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): retcol="user_type", allow_none=True, ) - return res is None + return res is None or res not in [UserTypes.BOT, UserTypes.SUPPORT] def is_support_user_txn(self, txn: LoggingTransaction, user_id: str) -> bool: res = self.db_pool.simple_select_one_onecol_txn( @@ -759,17 +922,37 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): external_id: id on that system user_id: complete mxid that it is mapped to """ + self._invalidate_cache_and_stream( + txn, self.get_user_by_external_id, (auth_provider, external_id) + ) - self.db_pool.simple_insert_txn( + # This INSERT ... ON CONFLICT DO NOTHING statement will cause a + # 'could not serialize access due to concurrent update' + # if the row is added concurrently by another transaction. + # This is exactly what we want, as it makes the transaction get retried + # in a new snapshot where we can check for a genuine conflict. + was_inserted = self.db_pool.simple_upsert_txn( txn, table="user_external_ids", - values={ - "auth_provider": auth_provider, - "external_id": external_id, - "user_id": user_id, - }, + keyvalues={"auth_provider": auth_provider, "external_id": external_id}, + values={}, + insertion_values={"user_id": user_id}, ) + if not was_inserted: + existing_id = self.db_pool.simple_select_one_onecol_txn( + txn, + table="user_external_ids", + keyvalues={"auth_provider": auth_provider, "user_id": user_id}, + retcol="external_id", + allow_none=True, + ) + + if existing_id != external_id: + raise ExternalIDReuseException( + f"{user_id!r} has external id {existing_id!r} for {auth_provider} but trying to add {external_id!r}" + ) + async def remove_user_external_id( self, auth_provider: str, external_id: str, user_id: str ) -> None: @@ -789,6 +972,9 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): }, desc="remove_user_external_id", ) + await self.invalidate_cache_and_stream( + "get_user_by_external_id", (auth_provider, external_id) + ) async def replace_user_external_id( self, @@ -809,29 +995,20 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): ExternalIDReuseException if the new external_id could not be mapped. """ - def _remove_user_external_ids_txn( + def _replace_user_external_id_txn( txn: LoggingTransaction, - user_id: str, ) -> None: - """Remove all mappings from external user ids to a mxid - If these mappings are not found, this method does nothing. - - Args: - user_id: complete mxid that it is mapped to - """ - self.db_pool.simple_delete_txn( txn, table="user_external_ids", keyvalues={"user_id": user_id}, ) - def _replace_user_external_id_txn( - txn: LoggingTransaction, - ) -> None: - _remove_user_external_ids_txn(txn, user_id) - for auth_provider, external_id in record_external_ids: + self._invalidate_cache_and_stream( + txn, self.get_user_by_external_id, (auth_provider, external_id) + ) + self._record_user_external_id_txn( txn, auth_provider, @@ -847,6 +1024,7 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): except self.database_engine.module.IntegrityError: raise ExternalIDReuseException() + @cached() async def get_user_by_external_id( self, auth_provider: str, external_id: str ) -> Optional[str]: @@ -944,10 +1122,12 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): return await self.db_pool.runInteraction("count_users", _count_users) async def count_real_users(self) -> int: - """Counts all users without a special user_type registered on the homeserver.""" + """Counts all users without the bot or support user_types registered on the homeserver.""" def _count_users(txn: LoggingTransaction) -> int: - txn.execute("SELECT COUNT(*) FROM users where user_type is null") + txn.execute( + f"SELECT COUNT(*) FROM users WHERE user_type IS NULL OR user_type NOT IN ('{UserTypes.BOT}', '{UserTypes.SUPPORT}')" + ) row = txn.fetchone() assert row is not None return row[0] @@ -1510,15 +1690,14 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): # Override type because the return type is only optional if # allow_none is True, and we don't want mypy throwing errors # about None not being indexable. - pending, completed = cast( - Tuple[int, int], - self.db_pool.simple_select_one_txn( - txn, - "registration_tokens", - keyvalues={"token": token}, - retcols=["pending", "completed"], - ), + row = self.db_pool.simple_select_one_txn( + txn, + "registration_tokens", + keyvalues={"token": token}, + retcols=("pending", "completed"), ) + pending = int(row[0]) + completed = int(row[1]) # Decrement pending and increment completed self.db_pool.simple_update_one_txn( @@ -1918,6 +2097,58 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): "replace_refresh_token", _replace_refresh_token_txn ) + async def set_device_for_refresh_token( + self, user_id: str, old_device_id: str, device_id: str + ) -> None: + """Moves refresh tokens from old device to current device + + Args: + user_id: The user of the devices. + old_device_id: The old device. + device_id: The new device ID. + Returns: + None + """ + + await self.db_pool.simple_update( + "refresh_tokens", + keyvalues={"user_id": user_id, "device_id": old_device_id}, + updatevalues={"device_id": device_id}, + desc="set_device_for_refresh_token", + ) + + def _set_device_for_access_token_txn( + self, txn: LoggingTransaction, token: str, device_id: str + ) -> str: + old_device_id = self.db_pool.simple_select_one_onecol_txn( + txn, "access_tokens", {"token": token}, "device_id" + ) + + self.db_pool.simple_update_txn( + txn, "access_tokens", {"token": token}, {"device_id": device_id} + ) + + self._invalidate_cache_and_stream(txn, self.get_user_by_access_token, (token,)) + + return old_device_id + + async def set_device_for_access_token(self, token: str, device_id: str) -> str: + """Sets the device ID associated with an access token. + + Args: + token: The access token to modify. + device_id: The new device ID. + Returns: + The old device ID associated with the access token. + """ + + return await self.db_pool.runInteraction( + "set_device_for_access_token", + self._set_device_for_access_token_txn, + token, + device_id, + ) + async def add_login_token_to_user( self, user_id: str, @@ -2091,6 +2322,314 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): func=is_user_approved_txn, ) + async def set_user_deactivated_status( + self, user_id: str, deactivated: bool + ) -> None: + """Set the `deactivated` property for the provided user to the provided value. + + Args: + user_id: The ID of the user to set the status for. + deactivated: The value to set for `deactivated`. + """ + + await self.db_pool.runInteraction( + "set_user_deactivated_status", + self.set_user_deactivated_status_txn, + user_id, + deactivated, + ) + + def set_user_deactivated_status_txn( + self, txn: LoggingTransaction, user_id: str, deactivated: bool + ) -> None: + self.db_pool.simple_update_one_txn( + txn=txn, + table="users", + keyvalues={"name": user_id}, + updatevalues={"deactivated": 1 if deactivated else 0}, + ) + self._invalidate_cache_and_stream( + txn, self.get_user_deactivated_status, (user_id,) + ) + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + self._invalidate_cache_and_stream(txn, self.is_guest, (user_id,)) + + async def set_user_suspended_status(self, user_id: str, suspended: bool) -> None: + """ + Set whether the user's account is suspended in the `users` table. + + Args: + user_id: The user ID of the user in question + suspended: True if the user is suspended, false if not + """ + await self.db_pool.runInteraction( + "set_user_suspended_status", + self.set_user_suspended_status_txn, + user_id, + suspended, + ) + + def set_user_suspended_status_txn( + self, txn: LoggingTransaction, user_id: str, suspended: bool + ) -> None: + self.db_pool.simple_update_one_txn( + txn=txn, + table="users", + keyvalues={"name": user_id}, + updatevalues={"suspended": suspended}, + ) + self._invalidate_cache_and_stream( + txn, self.get_user_suspended_status, (user_id,) + ) + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + + async def set_user_locked_status(self, user_id: str, locked: bool) -> None: + """Set the `locked` property for the provided user to the provided value. + + Args: + user_id: The ID of the user to set the status for. + locked: The value to set for `locked`. + """ + + await self.db_pool.runInteraction( + "set_user_locked_status", + self.set_user_locked_status_txn, + user_id, + locked, + ) + + def set_user_locked_status_txn( + self, txn: LoggingTransaction, user_id: str, locked: bool + ) -> None: + self.db_pool.simple_update_one_txn( + txn=txn, + table="users", + keyvalues={"name": user_id}, + updatevalues={"locked": locked}, + ) + self._invalidate_cache_and_stream(txn, self.get_user_locked_status, (user_id,)) + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + + async def update_user_approval_status( + self, user_id: UserID, approved: bool + ) -> None: + """Set the user's 'approved' flag to the given value. + + The boolean will be turned into an int (in update_user_approval_status_txn) + because the column is a smallint. + + Args: + user_id: the user to update the flag for. + approved: the value to set the flag to. + """ + await self.db_pool.runInteraction( + "update_user_approval_status", + self.update_user_approval_status_txn, + user_id.to_string(), + approved, + ) + + def update_user_approval_status_txn( + self, txn: LoggingTransaction, user_id: str, approved: bool + ) -> None: + """Set the user's 'approved' flag to the given value. + + The boolean is turned into an int because the column is a smallint. + + Args: + txn: the current database transaction. + user_id: the user to update the flag for. + approved: the value to set the flag to. + """ + self.db_pool.simple_update_one_txn( + txn=txn, + table="users", + keyvalues={"name": user_id}, + updatevalues={"approved": approved}, + ) + + # Invalidate the caches of methods that read the value of the 'approved' flag. + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + self._invalidate_cache_and_stream(txn, self.is_user_approved, (user_id,)) + + async def user_delete_access_tokens( + self, + user_id: str, + except_token_id: Optional[int] = None, + device_id: Optional[str] = None, + ) -> List[Tuple[str, int, Optional[str]]]: + """ + Invalidate access and refresh tokens belonging to a user + + Args: + user_id: ID of user the tokens belong to + except_token_id: access_tokens ID which should *not* be deleted + device_id: ID of device the tokens are associated with. + If None, tokens associated with any device (or no device) will + be deleted + Returns: + A tuple of (token, token id, device id) for each of the deleted tokens + """ + + def f(txn: LoggingTransaction) -> List[Tuple[str, int, Optional[str]]]: + keyvalues = {"user_id": user_id} + if device_id is not None: + keyvalues["device_id"] = device_id + + items = keyvalues.items() + where_clause = " AND ".join(k + " = ?" for k, _ in items) + values: List[Union[str, int]] = [v for _, v in items] + # Conveniently, refresh_tokens and access_tokens both use the user_id and device_id fields. Only caveat + # is the `except_token_id` param that is tricky to get right, so for now we're just using the same where + # clause and values before we handle that. This seems to be only used in the "set password" handler. + refresh_where_clause = where_clause + refresh_values = values.copy() + if except_token_id: + # TODO: support that for refresh tokens + where_clause += " AND id != ?" + values.append(except_token_id) + + txn.execute( + "SELECT token, id, device_id FROM access_tokens WHERE %s" + % where_clause, + values, + ) + tokens_and_devices = [(r[0], r[1], r[2]) for r in txn] + + self._invalidate_cache_and_stream_bulk( + txn, + self.get_user_by_access_token, + [(token,) for token, _, _ in tokens_and_devices], + ) + + txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values) + + txn.execute( + "DELETE FROM refresh_tokens WHERE %s" % refresh_where_clause, + refresh_values, + ) + + return tokens_and_devices + + return await self.db_pool.runInteraction("user_delete_access_tokens", f) + + async def user_delete_access_tokens_for_devices( + self, + user_id: str, + device_ids: StrCollection, + ) -> List[Tuple[str, int, Optional[str]]]: + """ + Invalidate access and refresh tokens belonging to a user + + Args: + user_id: ID of user the tokens belong to + device_ids: The devices to delete tokens for. + Returns: + A tuple of (token, token id, device id) for each of the deleted tokens + """ + + def user_delete_access_tokens_for_devices_txn( + txn: LoggingTransaction, batch_device_ids: StrCollection + ) -> List[Tuple[str, int, Optional[str]]]: + self.db_pool.simple_delete_many_txn( + txn, + table="refresh_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + clause, args = make_in_list_sql_clause( + txn.database_engine, "device_id", batch_device_ids + ) + args.append(user_id) + + if self.database_engine.supports_returning: + sql = f""" + DELETE FROM access_tokens + WHERE {clause} AND user_id = ? + RETURNING token, id, device_id + """ + txn.execute(sql, args) + tokens_and_devices = txn.fetchall() + else: + tokens_and_devices = self.db_pool.simple_select_many_txn( + txn, + table="access_tokens", + column="device_id", + iterable=batch_device_ids, + keyvalues={"user_id": user_id}, + retcols=("token", "id", "device_id"), + ) + + self.db_pool.simple_delete_many_txn( + txn, + table="access_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + self._invalidate_cache_and_stream_bulk( + txn, + self.get_user_by_access_token, + [(t[0],) for t in tokens_and_devices], + ) + return tokens_and_devices + + results = [] + for batch_device_ids in batch_iter(device_ids, 1000): + tokens_and_devices = await self.db_pool.runInteraction( + "user_delete_access_tokens_for_devices", + user_delete_access_tokens_for_devices_txn, + batch_device_ids, + ) + results.extend(tokens_and_devices) + + return results + + async def delete_access_token(self, access_token: str) -> None: + def f(txn: LoggingTransaction) -> None: + self.db_pool.simple_delete_one_txn( + txn, table="access_tokens", keyvalues={"token": access_token} + ) + + self._invalidate_cache_and_stream( + txn, self.get_user_by_access_token, (access_token,) + ) + + await self.db_pool.runInteraction("delete_access_token", f) + + async def user_set_password_hash( + self, user_id: str, password_hash: Optional[str] + ) -> None: + """ + NB. This does *not* evict any cache because the one use for this + removes most of the entries subsequently anyway so it would be + pointless. Use flush_user separately. + """ + + def user_set_password_hash_txn(txn: LoggingTransaction) -> None: + self.db_pool.simple_update_one_txn( + txn, "users", {"name": user_id}, {"password_hash": password_hash} + ) + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + + await self.db_pool.runInteraction( + "user_set_password_hash", user_set_password_hash_txn + ) + + async def add_user_pending_deactivation(self, user_id: str) -> None: + """ + Adds a user to the table of users who need to be parted from all the rooms they're + in + """ + await self.db_pool.simple_insert( + "users_pending_deactivation", + values={"user_id": user_id}, + desc="add_user_pending_deactivation", + ) + class RegistrationBackgroundUpdateStore(RegistrationWorkerStore): def __init__( @@ -2203,119 +2742,8 @@ class RegistrationBackgroundUpdateStore(RegistrationWorkerStore): return nb_processed - async def set_user_deactivated_status( - self, user_id: str, deactivated: bool - ) -> None: - """Set the `deactivated` property for the provided user to the provided value. - Args: - user_id: The ID of the user to set the status for. - deactivated: The value to set for `deactivated`. - """ - - await self.db_pool.runInteraction( - "set_user_deactivated_status", - self.set_user_deactivated_status_txn, - user_id, - deactivated, - ) - - def set_user_deactivated_status_txn( - self, txn: LoggingTransaction, user_id: str, deactivated: bool - ) -> None: - self.db_pool.simple_update_one_txn( - txn=txn, - table="users", - keyvalues={"name": user_id}, - updatevalues={"deactivated": 1 if deactivated else 0}, - ) - self._invalidate_cache_and_stream( - txn, self.get_user_deactivated_status, (user_id,) - ) - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - txn.call_after(self.is_guest.invalidate, (user_id,)) - - async def set_user_suspended_status(self, user_id: str, suspended: bool) -> None: - """ - Set whether the user's account is suspended in the `users` table. - - Args: - user_id: The user ID of the user in question - suspended: True if the user is suspended, false if not - """ - await self.db_pool.runInteraction( - "set_user_suspended_status", - self.set_user_suspended_status_txn, - user_id, - suspended, - ) - - def set_user_suspended_status_txn( - self, txn: LoggingTransaction, user_id: str, suspended: bool - ) -> None: - self.db_pool.simple_update_one_txn( - txn=txn, - table="users", - keyvalues={"name": user_id}, - updatevalues={"suspended": suspended}, - ) - self._invalidate_cache_and_stream( - txn, self.get_user_suspended_status, (user_id,) - ) - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - - async def set_user_locked_status(self, user_id: str, locked: bool) -> None: - """Set the `locked` property for the provided user to the provided value. - - Args: - user_id: The ID of the user to set the status for. - locked: The value to set for `locked`. - """ - - await self.db_pool.runInteraction( - "set_user_locked_status", - self.set_user_locked_status_txn, - user_id, - locked, - ) - - def set_user_locked_status_txn( - self, txn: LoggingTransaction, user_id: str, locked: bool - ) -> None: - self.db_pool.simple_update_one_txn( - txn=txn, - table="users", - keyvalues={"name": user_id}, - updatevalues={"locked": locked}, - ) - self._invalidate_cache_and_stream(txn, self.get_user_locked_status, (user_id,)) - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - - def update_user_approval_status_txn( - self, txn: LoggingTransaction, user_id: str, approved: bool - ) -> None: - """Set the user's 'approved' flag to the given value. - - The boolean is turned into an int because the column is a smallint. - - Args: - txn: the current database transaction. - user_id: the user to update the flag for. - approved: the value to set the flag to. - """ - self.db_pool.simple_update_one_txn( - txn=txn, - table="users", - keyvalues={"name": user_id}, - updatevalues={"approved": approved}, - ) - - # Invalidate the caches of methods that read the value of the 'approved' flag. - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - self._invalidate_cache_and_stream(txn, self.is_user_approved, (user_id,)) - - -class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): +class RegistrationStore(RegistrationBackgroundUpdateStore): def __init__( self, database: DatabasePool, @@ -2331,13 +2759,6 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id") self._refresh_tokens_id_gen = IdGenerator(db_conn, "refresh_tokens", "id") - # If support for MSC3866 is enabled and configured to require approval for new - # account, we will create new users with an 'approved' flag set to false. - self._require_approval = ( - hs.config.experimental.msc3866.enabled - and hs.config.experimental.msc3866.require_approval_for_new_accounts - ) - # Create a background job for removing expired login tokens if hs.config.worker.run_background_tasks: self._clock.looping_call( @@ -2433,223 +2854,6 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): return next_id - async def set_device_for_refresh_token( - self, user_id: str, old_device_id: str, device_id: str - ) -> None: - """Moves refresh tokens from old device to current device - - Args: - user_id: The user of the devices. - old_device_id: The old device. - device_id: The new device ID. - Returns: - None - """ - - await self.db_pool.simple_update( - "refresh_tokens", - keyvalues={"user_id": user_id, "device_id": old_device_id}, - updatevalues={"device_id": device_id}, - desc="set_device_for_refresh_token", - ) - - def _set_device_for_access_token_txn( - self, txn: LoggingTransaction, token: str, device_id: str - ) -> str: - old_device_id = self.db_pool.simple_select_one_onecol_txn( - txn, "access_tokens", {"token": token}, "device_id" - ) - - self.db_pool.simple_update_txn( - txn, "access_tokens", {"token": token}, {"device_id": device_id} - ) - - self._invalidate_cache_and_stream(txn, self.get_user_by_access_token, (token,)) - - return old_device_id - - async def set_device_for_access_token(self, token: str, device_id: str) -> str: - """Sets the device ID associated with an access token. - - Args: - token: The access token to modify. - device_id: The new device ID. - Returns: - The old device ID associated with the access token. - """ - - return await self.db_pool.runInteraction( - "set_device_for_access_token", - self._set_device_for_access_token_txn, - token, - device_id, - ) - - async def register_user( - self, - user_id: str, - password_hash: Optional[str] = None, - was_guest: bool = False, - make_guest: bool = False, - appservice_id: Optional[str] = None, - create_profile_with_displayname: Optional[str] = None, - admin: bool = False, - user_type: Optional[str] = None, - shadow_banned: bool = False, - approved: bool = False, - ) -> None: - """Attempts to register an account. - - Args: - user_id: The desired user ID to register. - password_hash: Optional. The password hash for this user. - was_guest: Whether this is a guest account being upgraded to a - non-guest account. - make_guest: True if the the new user should be guest, false to add a - regular user account. - appservice_id: The ID of the appservice registering the user. - create_profile_with_displayname: Optionally create a profile for - the user, setting their displayname to the given value - admin: is an admin user? - user_type: type of user. One of the values from api.constants.UserTypes, - or None for a normal user. - shadow_banned: Whether the user is shadow-banned, i.e. they may be - told their requests succeeded but we ignore them. - approved: Whether to consider the user has already been approved by an - administrator. - - Raises: - StoreError if the user_id could not be registered. - """ - await self.db_pool.runInteraction( - "register_user", - self._register_user, - user_id, - password_hash, - was_guest, - make_guest, - appservice_id, - create_profile_with_displayname, - admin, - user_type, - shadow_banned, - approved, - ) - - def _register_user( - self, - txn: LoggingTransaction, - user_id: str, - password_hash: Optional[str], - was_guest: bool, - make_guest: bool, - appservice_id: Optional[str], - create_profile_with_displayname: Optional[str], - admin: bool, - user_type: Optional[str], - shadow_banned: bool, - approved: bool, - ) -> None: - user_id_obj = UserID.from_string(user_id) - - now = int(self._clock.time()) - - user_approved = approved or not self._require_approval - - try: - if was_guest: - # Ensure that the guest user actually exists - # ``allow_none=False`` makes this raise an exception - # if the row isn't in the database. - self.db_pool.simple_select_one_txn( - txn, - "users", - keyvalues={"name": user_id, "is_guest": 1}, - retcols=("name",), - allow_none=False, - ) - - self.db_pool.simple_update_one_txn( - txn, - "users", - keyvalues={"name": user_id, "is_guest": 1}, - updatevalues={ - "password_hash": password_hash, - "upgrade_ts": now, - "is_guest": 1 if make_guest else 0, - "appservice_id": appservice_id, - "admin": 1 if admin else 0, - "user_type": user_type, - "shadow_banned": shadow_banned, - "approved": user_approved, - }, - ) - else: - self.db_pool.simple_insert_txn( - txn, - "users", - values={ - "name": user_id, - "password_hash": password_hash, - "creation_ts": now, - "is_guest": 1 if make_guest else 0, - "appservice_id": appservice_id, - "admin": 1 if admin else 0, - "user_type": user_type, - "shadow_banned": shadow_banned, - "approved": user_approved, - }, - ) - - except self.database_engine.module.IntegrityError: - raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE) - - if self._account_validity_enabled: - self.set_expiration_date_for_user_txn(txn, user_id) - - if create_profile_with_displayname: - # set a default displayname serverside to avoid ugly race - # between auto-joins and clients trying to set displaynames - # - # *obviously* the 'profiles' table uses localpart for user_id - # while everything else uses the full mxid. - txn.execute( - "INSERT INTO profiles(full_user_id, user_id, displayname) VALUES (?,?,?)", - (user_id, user_id_obj.localpart, create_profile_with_displayname), - ) - - if self.hs.config.stats.stats_enabled: - # we create a new completed user statistics row - - # we don't strictly need current_token since this user really can't - # have any state deltas before now (as it is a new user), but still, - # we include it for completeness. - current_token = self._get_max_stream_id_in_current_state_deltas_txn(txn) - self._update_stats_delta_txn( - txn, now, "user", user_id, {}, complete_with_stream_id=current_token - ) - - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - - async def user_set_password_hash( - self, user_id: str, password_hash: Optional[str] - ) -> None: - """ - NB. This does *not* evict any cache because the one use for this - removes most of the entries subsequently anyway so it would be - pointless. Use flush_user separately. - """ - - def user_set_password_hash_txn(txn: LoggingTransaction) -> None: - self.db_pool.simple_update_one_txn( - txn, "users", {"name": user_id}, {"password_hash": password_hash} - ) - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - - await self.db_pool.runInteraction( - "user_set_password_hash", user_set_password_hash_txn - ) - async def user_set_consent_version( self, user_id: str, consent_version: str ) -> None: @@ -2702,98 +2906,6 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): await self.db_pool.runInteraction("user_set_consent_server_notice_sent", f) - async def user_delete_access_tokens( - self, - user_id: str, - except_token_id: Optional[int] = None, - device_id: Optional[str] = None, - ) -> List[Tuple[str, int, Optional[str]]]: - """ - Invalidate access and refresh tokens belonging to a user - - Args: - user_id: ID of user the tokens belong to - except_token_id: access_tokens ID which should *not* be deleted - device_id: ID of device the tokens are associated with. - If None, tokens associated with any device (or no device) will - be deleted - Returns: - A tuple of (token, token id, device id) for each of the deleted tokens - """ - - def f(txn: LoggingTransaction) -> List[Tuple[str, int, Optional[str]]]: - keyvalues = {"user_id": user_id} - if device_id is not None: - keyvalues["device_id"] = device_id - - items = keyvalues.items() - where_clause = " AND ".join(k + " = ?" for k, _ in items) - values: List[Union[str, int]] = [v for _, v in items] - # Conveniently, refresh_tokens and access_tokens both use the user_id and device_id fields. Only caveat - # is the `except_token_id` param that is tricky to get right, so for now we're just using the same where - # clause and values before we handle that. This seems to be only used in the "set password" handler. - refresh_where_clause = where_clause - refresh_values = values.copy() - if except_token_id: - # TODO: support that for refresh tokens - where_clause += " AND id != ?" - values.append(except_token_id) - - txn.execute( - "SELECT token, id, device_id FROM access_tokens WHERE %s" - % where_clause, - values, - ) - tokens_and_devices = [(r[0], r[1], r[2]) for r in txn] - - self._invalidate_cache_and_stream_bulk( - txn, - self.get_user_by_access_token, - [(token,) for token, _, _ in tokens_and_devices], - ) - - txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values) - - txn.execute( - "DELETE FROM refresh_tokens WHERE %s" % refresh_where_clause, - refresh_values, - ) - - return tokens_and_devices - - return await self.db_pool.runInteraction("user_delete_access_tokens", f) - - async def delete_access_token(self, access_token: str) -> None: - def f(txn: LoggingTransaction) -> None: - self.db_pool.simple_delete_one_txn( - txn, table="access_tokens", keyvalues={"token": access_token} - ) - - self._invalidate_cache_and_stream( - txn, self.get_user_by_access_token, (access_token,) - ) - - await self.db_pool.runInteraction("delete_access_token", f) - - async def delete_refresh_token(self, refresh_token: str) -> None: - def f(txn: LoggingTransaction) -> None: - self.db_pool.simple_delete_one_txn( - txn, table="refresh_tokens", keyvalues={"token": refresh_token} - ) - - await self.db_pool.runInteraction("delete_refresh_token", f) - - async def add_user_pending_deactivation(self, user_id: str) -> None: - """ - Adds a user to the table of users who need to be parted from all the rooms they're - in - """ - await self.db_pool.simple_insert( - "users_pending_deactivation", - values={"user_id": user_id}, - desc="add_user_pending_deactivation", - ) - async def validate_threepid_session( self, session_id: str, client_secret: str, token: str, current_ts: int ) -> Optional[str]: @@ -2942,25 +3054,6 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): start_or_continue_validation_session_txn, ) - async def update_user_approval_status( - self, user_id: UserID, approved: bool - ) -> None: - """Set the user's 'approved' flag to the given value. - - The boolean will be turned into an int (in update_user_approval_status_txn) - because the column is a smallint. - - Args: - user_id: the user to update the flag for. - approved: the value to set the flag to. - """ - await self.db_pool.runInteraction( - "update_user_approval_status", - self.update_user_approval_status_txn, - user_id.to_string(), - approved, - ) - @wrap_as_background_process("delete_expired_login_tokens") async def _delete_expired_login_tokens(self) -> None: """Remove login tokens with expiry dates that have passed.""" diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 29a001ff92..ea746e0511 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -53,7 +53,7 @@ from synapse.storage.databases.main.stream import ( generate_pagination_where_clause, ) from synapse.storage.engines import PostgresEngine -from synapse.types import JsonDict, MultiWriterStreamToken, StreamKeyType, StreamToken +from synapse.types import JsonDict, StreamKeyType, StreamToken from synapse.util.caches.descriptors import cached, cachedList if TYPE_CHECKING: @@ -316,17 +316,8 @@ class RelationsWorkerStore(SQLBaseStore): StreamKeyType.ROOM, next_key ) else: - next_token = StreamToken( - room_key=next_key, - presence_key=0, - typing_key=0, - receipt_key=MultiWriterStreamToken(stream=0), - account_data_key=0, - push_rules_key=0, - to_device_key=0, - device_list_key=0, - groups_key=0, - un_partial_stated_rooms_key=0, + next_token = StreamToken.START.copy_and_replace( + StreamKeyType.ROOM, next_key ) return events[:limit], next_token diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index cc3ce0951e..6ffc3aed34 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -51,11 +51,15 @@ from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.config.homeserver import HomeServerConfig from synapse.events import EventBase from synapse.replication.tcp.streams.partial_state import UnPartialStatedRoomStream -from synapse.storage._base import db_to_json, make_in_list_sql_clause +from synapse.storage._base import ( + db_to_json, + make_in_list_sql_clause, +) from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, LoggingTransaction, + make_tuple_in_list_sql_clause, ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.types import Cursor @@ -73,6 +77,8 @@ logger = logging.getLogger(__name__) @attr.s(slots=True, frozen=True, auto_attribs=True) class RatelimitOverride: + # n.b. elsewhere in Synapse messages_per_second is represented as a float, but it is + # an integer in the database messages_per_second: int burst_count: int @@ -154,6 +160,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): db=database, notifier=hs.get_replication_notifier(), stream_name="un_partial_stated_room_stream", + server_name=self.server_name, instance_name=self._instance_name, tables=[("un_partial_stated_room_stream", "instance_name", "stream_id")], sequence_name="un_partial_stated_room_stream_sequence", @@ -1127,6 +1134,109 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): return local_media_ids + def _quarantine_local_media_txn( + self, + txn: LoggingTransaction, + hashes: Set[str], + media_ids: Set[str], + quarantined_by: Optional[str], + ) -> int: + """Quarantine and unquarantine local media items. + + Args: + txn (cursor) + hashes: A set of sha256 hashes for any media that should be quarantined + media_ids: A set of media IDs for any media that should be quarantined + quarantined_by: The ID of the user who initiated the quarantine request + If it is `None` media will be removed from quarantine + Returns: + The total number of media items quarantined + """ + total_media_quarantined = 0 + + # Effectively a legacy path, update any media that was explicitly named. + if media_ids: + sql_many_clause_sql, sql_many_clause_args = make_in_list_sql_clause( + txn.database_engine, "media_id", media_ids + ) + sql = f""" + UPDATE local_media_repository + SET quarantined_by = ? + WHERE {sql_many_clause_sql}""" + + if quarantined_by is not None: + sql += " AND safe_from_quarantine = FALSE" + + txn.execute(sql, [quarantined_by] + sql_many_clause_args) + # Note that a rowcount of -1 can be used to indicate no rows were affected. + total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + + # Update any media that was identified via hash. + if hashes: + sql_many_clause_sql, sql_many_clause_args = make_in_list_sql_clause( + txn.database_engine, "sha256", hashes + ) + sql = f""" + UPDATE local_media_repository + SET quarantined_by = ? + WHERE {sql_many_clause_sql}""" + + if quarantined_by is not None: + sql += " AND safe_from_quarantine = FALSE" + + txn.execute(sql, [quarantined_by] + sql_many_clause_args) + total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + + return total_media_quarantined + + def _quarantine_remote_media_txn( + self, + txn: LoggingTransaction, + hashes: Set[str], + media: Set[Tuple[str, str]], + quarantined_by: Optional[str], + ) -> int: + """Quarantine and unquarantine remote items + + Args: + txn (cursor) + hashes: A set of sha256 hashes for any media that should be quarantined + media_ids: A set of tuples (media_origin, media_id) for any media that should be quarantined + quarantined_by: The ID of the user who initiated the quarantine request + If it is `None` media will be removed from quarantine + Returns: + The total number of media items quarantined + """ + total_media_quarantined = 0 + + if media: + sql_in_list_clause, sql_args = make_tuple_in_list_sql_clause( + txn.database_engine, + ("media_origin", "media_id"), + media, + ) + sql = f""" + UPDATE remote_media_cache + SET quarantined_by = ? + WHERE {sql_in_list_clause}""" + + txn.execute(sql, [quarantined_by] + sql_args) + total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + + total_media_quarantined = 0 + if hashes: + sql_many_clause_sql, sql_many_clause_args = make_in_list_sql_clause( + txn.database_engine, "sha256", hashes + ) + sql = f""" + UPDATE remote_media_cache + SET quarantined_by = ? + WHERE {sql_many_clause_sql}""" + txn.execute(sql, [quarantined_by] + sql_many_clause_args) + total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + + return total_media_quarantined + def _quarantine_media_txn( self, txn: LoggingTransaction, @@ -1146,40 +1256,93 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): Returns: The total number of media items quarantined """ + hashes = set() + media_ids = set() + remote_media = set() - # Update all the tables to set the quarantined_by flag - sql = """ - UPDATE local_media_repository - SET quarantined_by = ? - WHERE media_id = ? - """ - - # set quarantine - if quarantined_by is not None: - sql += "AND safe_from_quarantine = FALSE" - txn.executemany( - sql, [(quarantined_by, media_id) for media_id in local_mxcs] + # First, determine the hashes of the media we want to delete. + # We also want the media_ids for any media that lacks a hash. + if local_mxcs: + hash_sql_many_clause_sql, hash_sql_many_clause_args = ( + make_in_list_sql_clause(txn.database_engine, "media_id", local_mxcs) ) - # remove from quarantine - else: - txn.executemany( - sql, [(quarantined_by, media_id) for media_id in local_mxcs] + hash_sql = f"SELECT sha256, media_id FROM local_media_repository WHERE {hash_sql_many_clause_sql}" + if quarantined_by is not None: + hash_sql += " AND safe_from_quarantine = FALSE" + + txn.execute(hash_sql, hash_sql_many_clause_args) + for sha256, media_id in txn: + if sha256: + hashes.add(sha256) + else: + media_ids.add(media_id) + + # Do the same for remote media + if remote_mxcs: + hash_sql_in_list_clause, hash_sql_args = make_tuple_in_list_sql_clause( + txn.database_engine, + ("media_origin", "media_id"), + remote_mxcs, ) - # Note that a rowcount of -1 can be used to indicate no rows were affected. - total_media_quarantined = txn.rowcount if txn.rowcount > 0 else 0 + hash_sql = f"SELECT sha256, media_origin, media_id FROM remote_media_cache WHERE {hash_sql_in_list_clause}" + txn.execute(hash_sql, hash_sql_args) + for sha256, media_origin, media_id in txn: + if sha256: + hashes.add(sha256) + else: + remote_media.add((media_origin, media_id)) - txn.executemany( - """ - UPDATE remote_media_cache - SET quarantined_by = ? - WHERE media_origin = ? AND media_id = ? - """, - [(quarantined_by, origin, media_id) for origin, media_id in remote_mxcs], + count = self._quarantine_local_media_txn(txn, hashes, media_ids, quarantined_by) + count += self._quarantine_remote_media_txn( + txn, hashes, remote_media, quarantined_by ) - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 - return total_media_quarantined + return count + + async def block_room(self, room_id: str, user_id: str) -> None: + """Marks the room as blocked. + + Can be called multiple times (though we'll only track the last user to + block this room). + + Can be called on a room unknown to this homeserver. + + Args: + room_id: Room to block + user_id: Who blocked it + """ + await self.db_pool.simple_upsert( + table="blocked_rooms", + keyvalues={"room_id": room_id}, + values={}, + insertion_values={"user_id": user_id}, + desc="block_room", + ) + await self.db_pool.runInteraction( + "block_room_invalidation", + self._invalidate_cache_and_stream, + self.is_room_blocked, + (room_id,), + ) + + async def unblock_room(self, room_id: str) -> None: + """Remove the room from blocking list. + + Args: + room_id: Room to unblock + """ + await self.db_pool.simple_delete( + table="blocked_rooms", + keyvalues={"room_id": room_id}, + desc="unblock_room", + ) + await self.db_pool.runInteraction( + "block_room_invalidation", + self._invalidate_cache_and_stream, + self.is_room_blocked, + (room_id,), + ) async def get_rooms_for_retention_period_in_range( self, min_ms: Optional[int], max_ms: Optional[int], include_null: bool = False @@ -1412,6 +1575,11 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): """Get the event ID of the initial join that started the partial join, and the device list stream ID at the point we started the partial join. + + This only returns the minimum device list stream ID at the time of + joining, not the full device list stream token. The only impact of this + is that we may be sending again device list updates that we've already + sent to some destinations, which is harmless. """ return cast( @@ -1586,6 +1754,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): direction: Direction = Direction.BACKWARDS, user_id: Optional[str] = None, room_id: Optional[str] = None, + event_sender_user_id: Optional[str] = None, ) -> Tuple[List[Dict[str, Any]], int]: """Retrieve a paginated list of event reports @@ -1596,6 +1765,8 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): oldest first (forwards) user_id: search for user_id. Ignored if user_id is None room_id: search for room_id. Ignored if room_id is None + event_sender_user_id: search for the sender of the reported event. Ignored if + event_sender_user_id is None Returns: Tuple of: json list of event reports @@ -1615,6 +1786,10 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): filters.append("er.room_id LIKE ?") args.extend(["%" + room_id + "%"]) + if event_sender_user_id: + filters.append("events.sender = ?") + args.extend([event_sender_user_id]) + if direction == Direction.BACKWARDS: order = "DESC" else: @@ -1630,6 +1805,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): sql = """ SELECT COUNT(*) as total_event_reports FROM event_reports AS er + LEFT JOIN events USING(event_id) JOIN room_stats_state ON room_stats_state.room_id = er.room_id {} """.format(where_clause) @@ -1648,8 +1824,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): room_stats_state.canonical_alias, room_stats_state.name FROM event_reports AS er - LEFT JOIN events - ON events.event_id = er.event_id + LEFT JOIN events USING(event_id) JOIN room_stats_state ON room_stats_state.room_id = er.room_id {where_clause} @@ -1766,6 +1941,65 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): desc="set_room_is_public_appservice_false", ) + async def has_auth_chain_index(self, room_id: str) -> bool: + """Check if the room has (or can have) a chain cover index. + + Defaults to True if we don't have an entry in `rooms` table nor any + events for the room. + """ + + has_auth_chain_index = await self.db_pool.simple_select_one_onecol( + table="rooms", + keyvalues={"room_id": room_id}, + retcol="has_auth_chain_index", + desc="has_auth_chain_index", + allow_none=True, + ) + + if has_auth_chain_index: + return True + + # It's possible that we already have events for the room in our DB + # without a corresponding room entry. If we do then we don't want to + # mark the room as having an auth chain cover index. + max_ordering = await self.db_pool.simple_select_one_onecol( + table="events", + keyvalues={"room_id": room_id}, + retcol="MAX(stream_ordering)", + allow_none=True, + desc="has_auth_chain_index_fallback", + ) + + return max_ordering is None + + async def maybe_store_room_on_outlier_membership( + self, room_id: str, room_version: RoomVersion + ) -> None: + """ + When we receive an invite or any other event over federation that may relate to a room + we are not in, store the version of the room if we don't already know the room version. + """ + # It's possible that we already have events for the room in our DB + # without a corresponding room entry. If we do then we don't want to + # mark the room as having an auth chain cover index. + has_auth_chain_index = await self.has_auth_chain_index(room_id) + + await self.db_pool.simple_upsert( + desc="maybe_store_room_on_outlier_membership", + table="rooms", + keyvalues={"room_id": room_id}, + values={}, + insertion_values={ + "room_version": room_version.identifier, + "is_public": False, + # We don't worry about setting the `creator` here because + # we don't process any messages in a room while a user is + # invited (only after the join). + "creator": "", + "has_auth_chain_index": has_auth_chain_index, + }, + ) + class _BackgroundUpdates: REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory" @@ -2017,37 +2251,6 @@ class RoomBackgroundUpdateStore(RoomWorkerStore): return len(rooms) - async def has_auth_chain_index(self, room_id: str) -> bool: - """Check if the room has (or can have) a chain cover index. - - Defaults to True if we don't have an entry in `rooms` table nor any - events for the room. - """ - - has_auth_chain_index = await self.db_pool.simple_select_one_onecol( - table="rooms", - keyvalues={"room_id": room_id}, - retcol="has_auth_chain_index", - desc="has_auth_chain_index", - allow_none=True, - ) - - if has_auth_chain_index: - return True - - # It's possible that we already have events for the room in our DB - # without a corresponding room entry. If we do then we don't want to - # mark the room as having an auth chain cover index. - max_ordering = await self.db_pool.simple_select_one_onecol( - table="events", - keyvalues={"room_id": room_id}, - retcol="MAX(stream_ordering)", - allow_none=True, - desc="has_auth_chain_index_fallback", - ) - - return max_ordering is None - async def _background_populate_room_depth_min_depth2( self, progress: JsonDict, batch_size: int ) -> int: @@ -2252,6 +2455,7 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore): self._event_reports_id_gen = IdGenerator(db_conn, "event_reports", "id") self._room_reports_id_gen = IdGenerator(db_conn, "room_reports", "id") + self._user_reports_id_gen = IdGenerator(db_conn, "user_reports", "id") self._instance_name = hs.get_instance_name() @@ -2397,34 +2601,6 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore): updatevalues={"join_event_id": join_event_id}, ) - async def maybe_store_room_on_outlier_membership( - self, room_id: str, room_version: RoomVersion - ) -> None: - """ - When we receive an invite or any other event over federation that may relate to a room - we are not in, store the version of the room if we don't already know the room version. - """ - # It's possible that we already have events for the room in our DB - # without a corresponding room entry. If we do then we don't want to - # mark the room as having an auth chain cover index. - has_auth_chain_index = await self.has_auth_chain_index(room_id) - - await self.db_pool.simple_upsert( - desc="maybe_store_room_on_outlier_membership", - table="rooms", - keyvalues={"room_id": room_id}, - values={}, - insertion_values={ - "room_version": room_version.identifier, - "is_public": False, - # We don't worry about setting the `creator` here because - # we don't process any messages in a room while a user is - # invited (only after the join). - "creator": "", - "has_auth_chain_index": has_auth_chain_index, - }, - ) - async def add_event_report( self, room_id: str, @@ -2493,49 +2669,36 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore): ) return next_id - async def block_room(self, room_id: str, user_id: str) -> None: - """Marks the room as blocked. - - Can be called multiple times (though we'll only track the last user to - block this room). - - Can be called on a room unknown to this homeserver. + async def add_user_report( + self, + target_user_id: str, + user_id: str, + reason: str, + received_ts: int, + ) -> int: + """Add a user report Args: - room_id: Room to block - user_id: Who blocked it + target_user_id: The user ID being reported. + user_id: User who reported the user. + reason: Description that the user specifies. + received_ts: Time when the user submitted the report (milliseconds). + Returns: + ID of the room report. """ - await self.db_pool.simple_upsert( - table="blocked_rooms", - keyvalues={"room_id": room_id}, - values={}, - insertion_values={"user_id": user_id}, - desc="block_room", - ) - await self.db_pool.runInteraction( - "block_room_invalidation", - self._invalidate_cache_and_stream, - self.is_room_blocked, - (room_id,), - ) - - async def unblock_room(self, room_id: str) -> None: - """Remove the room from blocking list. - - Args: - room_id: Room to unblock - """ - await self.db_pool.simple_delete( - table="blocked_rooms", - keyvalues={"room_id": room_id}, - desc="unblock_room", - ) - await self.db_pool.runInteraction( - "block_room_invalidation", - self._invalidate_cache_and_stream, - self.is_room_blocked, - (room_id,), + next_id = self._user_reports_id_gen.get_next() + await self.db_pool.simple_insert( + table="user_reports", + values={ + "id": next_id, + "received_ts": received_ts, + "target_user_id": target_user_id, + "user_id": user_id, + "reason": reason, + }, + desc="add_user_report", ) + return next_id async def clear_partial_state_room(self, room_id: str) -> Optional[int]: """Clears the partial state flag for a room. diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 4249cf77e5..9db2e14a06 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -43,7 +43,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.logging.opentracing import trace -from synapse.metrics import LaterGauge +from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( @@ -53,6 +53,7 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events_worker import EventsWorkerStore +from synapse.storage.databases.main.stream import _filter_results_by_stream from synapse.storage.engines import Sqlite3Engine from synapse.storage.roommember import ( MemberSummary, @@ -65,6 +66,7 @@ from synapse.types import ( PersistedEventPosition, StateMap, StrCollection, + StreamToken, get_domain_from_id, ) from synapse.util.caches.descriptors import _CacheContext, cached, cachedList @@ -79,6 +81,14 @@ logger = logging.getLogger(__name__) _MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update" _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME = "current_state_events_membership" +_POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE = 1000 + + +federation_known_servers_gauge = LaterGauge( + name="synapse_federation_known_servers", + desc="", + labelnames=[SERVER_NAME_LABEL], +) @attr.s(frozen=True, slots=True, auto_attribs=True) @@ -113,11 +123,9 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): 1, self._count_known_servers, ) - LaterGauge( - "synapse_federation_known_servers", - "", - [], - lambda: self._known_servers_count, + federation_known_servers_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: {(self.server_name,): self._known_servers_count}, ) @wrap_as_background_process("_count_known_servers") @@ -193,6 +201,19 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): retcol="state_key", ) + async def get_invited_users_in_room(self, room_id: str) -> StrCollection: + """Returns a list of users invited to the room.""" + return await self.db_pool.simple_select_onecol( + table="current_state_events", + keyvalues={ + "type": EventTypes.Member, + "room_id": room_id, + "membership": Membership.INVITE, + }, + retcol="state_key", + desc="get_invited_users_in_room", + ) + @cached() def get_user_in_room_with_profile(self, room_id: str, user_id: str) -> ProfileInfo: raise NotImplementedError() @@ -868,6 +889,73 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): return {u for u, share_room in user_dict.items() if share_room} + @cached(max_entries=10000) + async def does_pair_of_users_share_a_room_joined_or_invited( + self, user_id: str, other_user_id: str + ) -> bool: + raise NotImplementedError() + + @cachedList( + cached_method_name="does_pair_of_users_share_a_room_joined_or_invited", + list_name="other_user_ids", + ) + async def _do_users_share_a_room_joined_or_invited( + self, user_id: str, other_user_ids: Collection[str] + ) -> Mapping[str, Optional[bool]]: + """Return mapping from user ID to whether they share a room with the + given user via being either joined or invited. + + Note: `None` and `False` are equivalent and mean they don't share a + room. + """ + + def do_users_share_a_room_joined_or_invited_txn( + txn: LoggingTransaction, user_ids: Collection[str] + ) -> Dict[str, bool]: + clause, args = make_in_list_sql_clause( + self.database_engine, "state_key", user_ids + ) + + # This query works by fetching both the list of rooms for the target + # user and the set of other users, and then checking if there is any + # overlap. + sql = f""" + SELECT DISTINCT b.state_key + FROM ( + SELECT room_id FROM current_state_events + WHERE type = 'm.room.member' AND (membership = 'join' OR membership = 'invite') AND state_key = ? + ) AS a + INNER JOIN ( + SELECT room_id, state_key FROM current_state_events + WHERE type = 'm.room.member' AND (membership = 'join' OR membership = 'invite') AND {clause} + ) AS b using (room_id) + """ + + txn.execute(sql, (user_id, *args)) + return {u: True for (u,) in txn} + + to_return = {} + for batch_user_ids in batch_iter(other_user_ids, 1000): + res = await self.db_pool.runInteraction( + "do_users_share_a_room_joined_or_invited", + do_users_share_a_room_joined_or_invited_txn, + batch_user_ids, + ) + to_return.update(res) + + return to_return + + async def do_users_share_a_room_joined_or_invited( + self, user_id: str, other_user_ids: Collection[str] + ) -> Set[str]: + """Return the set of users who share a room with the first users via being either joined or invited""" + + user_dict = await self._do_users_share_a_room_joined_or_invited( + user_id, other_user_ids + ) + + return {u for u, share_room in user_dict.items() if share_room} + async def get_users_who_share_room_with_user(self, user_id: str) -> Set[str]: """Returns the set of users who share a room with `user_id`""" room_ids = await self.get_rooms_for_user(user_id) @@ -913,7 +1001,11 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): `_get_user_ids_from_membership_event_ids` for any uncached events. """ - with Measure(self._clock, "get_joined_user_ids_from_state"): + with Measure( + self._clock, + name="get_joined_user_ids_from_state", + server_name=self.server_name, + ): users_in_room = set() member_event_ids = [ e_id for key, e_id in state.items() if key[0] == EventTypes.Member @@ -1388,7 +1480,9 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): txn, self.get_forgotten_rooms_for_user, (user_id,) ) self._invalidate_cache_and_stream( - txn, self.get_sliding_sync_rooms_for_user, (user_id,) + txn, + self.get_sliding_sync_rooms_for_user_from_membership_snapshots, + (user_id,), ) await self.db_pool.runInteraction("forget_membership", f) @@ -1420,25 +1514,30 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): ) @cached(iterable=True, max_entries=10000) - async def get_sliding_sync_rooms_for_user( - self, - user_id: str, + async def get_sliding_sync_rooms_for_user_from_membership_snapshots( + self, user_id: str ) -> Mapping[str, RoomsForUserSlidingSync]: - """Get all the rooms for a user to handle a sliding sync request. + """ + Get all the rooms for a user to handle a sliding sync request from the + `sliding_sync_membership_snapshots` table. These will be current memberships and + need to be rewound to the token range. Ignores forgotten rooms and rooms that the user has left themselves. + Args: + user_id: The user ID to get the rooms for. + Returns: Map from room ID to membership info """ - def get_sliding_sync_rooms_for_user_txn( + def _txn( txn: LoggingTransaction, ) -> Dict[str, RoomsForUserSlidingSync]: # XXX: If you use any new columns that can change (like from # `sliding_sync_joined_rooms` or `forgotten`), make sure to bust the - # `get_sliding_sync_rooms_for_user` cache in the appropriate places (and add - # tests). + # `get_sliding_sync_rooms_for_user_from_membership_snapshots` cache in the + # appropriate places (and add tests). sql = """ SELECT m.room_id, m.sender, m.membership, m.membership_event_id, r.room_version, @@ -1454,6 +1553,7 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): AND (m.membership != 'leave' OR m.user_id != m.sender) """ txn.execute(sql, (user_id,)) + return { row[0]: RoomsForUserSlidingSync( room_id=row[0], @@ -1474,8 +1574,113 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): } return await self.db_pool.runInteraction( - "get_sliding_sync_rooms_for_user", - get_sliding_sync_rooms_for_user_txn, + "get_sliding_sync_rooms_for_user_from_membership_snapshots", + _txn, + ) + + async def get_sliding_sync_self_leave_rooms_after_to_token( + self, + user_id: str, + to_token: StreamToken, + ) -> Dict[str, RoomsForUserSlidingSync]: + """ + Get all the self-leave rooms for a user after the `to_token` (outside the token + range) that are potentially relevant[1] and needed to handle a sliding sync + request. The results are from the `sliding_sync_membership_snapshots` table and + will be current memberships and need to be rewound to the token range. + + [1] If a leave happens after the token range, we may have still been joined (or + any non-self-leave which is relevant to sync) to the room before so we need to + include it in the list of potentially relevant rooms and apply + our rewind logic (outside of this function) to see if it's actually relevant. + + This is basically a sister-function to + `get_sliding_sync_rooms_for_user_from_membership_snapshots`. We could + alternatively incorporate this logic into + `get_sliding_sync_rooms_for_user_from_membership_snapshots` but those results + are cached and the `to_token` isn't very cache friendly (people are constantly + requesting with new tokens) so we separate it out here. + + Args: + user_id: The user ID to get the rooms for. + to_token: Any self-leave memberships after this position will be returned. + + Returns: + Map from room ID to membership info + """ + # TODO: Potential to check + # `self._membership_stream_cache.has_entity_changed(...)` as an early-return + # shortcut. + + def _txn( + txn: LoggingTransaction, + ) -> Dict[str, RoomsForUserSlidingSync]: + sql = """ + SELECT m.room_id, m.sender, m.membership, m.membership_event_id, + r.room_version, + m.event_instance_name, m.event_stream_ordering, + m.has_known_state, + m.room_type, + m.is_encrypted + FROM sliding_sync_membership_snapshots AS m + INNER JOIN rooms AS r USING (room_id) + WHERE user_id = ? + AND m.forgotten = 0 + AND m.membership = 'leave' + AND m.user_id = m.sender + AND (m.event_stream_ordering > ?) + """ + # If a leave happens after the token range, we may have still been joined + # (or any non-self-leave which is relevant to sync) to the room before so we + # need to include it in the list of potentially relevant rooms and apply our + # rewind logic (outside of this function). + # + # To handle tokens with a non-empty instance_map we fetch more + # results than necessary and then filter down + min_to_token_position = to_token.room_key.stream + txn.execute(sql, (user_id, min_to_token_position)) + + # Map from room_id to membership info + room_membership_for_user_map: Dict[str, RoomsForUserSlidingSync] = {} + for row in txn: + room_for_user = RoomsForUserSlidingSync( + room_id=row[0], + sender=row[1], + membership=row[2], + event_id=row[3], + room_version_id=row[4], + event_pos=PersistedEventPosition(row[5], row[6]), + has_known_state=bool(row[7]), + room_type=row[8], + is_encrypted=bool(row[9]), + ) + + # We filter out unknown room versions proactively. They shouldn't go + # down sync and their metadata may be in a broken state (causing + # errors). + if row[4] not in KNOWN_ROOM_VERSIONS: + continue + + # We only want to include the self-leave membership if it happened after + # the token range. + # + # Since the database pulls out more than necessary, we need to filter it + # down here. + if _filter_results_by_stream( + lower_token=None, + upper_token=to_token.room_key, + instance_name=room_for_user.event_pos.instance_name, + stream_ordering=room_for_user.event_pos.stream, + ): + continue + + room_membership_for_user_map[room_for_user.room_id] = room_for_user + + return room_membership_for_user_map + + return await self.db_pool.runInteraction( + "get_sliding_sync_self_leave_rooms_after_to_token", + _txn, ) async def get_sliding_sync_room_for_user( @@ -1572,6 +1777,109 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): get_sliding_sync_room_for_user_batch_txn, ) + async def get_rooms_for_user_by_date( + self, user_id: str, from_ts: int + ) -> FrozenSet[str]: + """ + Fetch a list of rooms that the user has joined at or after the given timestamp, including + those they subsequently have left/been banned from. + + Args: + user_id: user ID of the user to search for + from_ts: a timestamp in ms from the unix epoch at which to begin the search at + """ + + def _get_rooms_for_user_by_join_date_txn( + txn: LoggingTransaction, user_id: str, timestamp: int + ) -> frozenset: + sql = """ + SELECT rm.room_id + FROM room_memberships AS rm + INNER JOIN events AS e USING (event_id) + WHERE rm.user_id = ? + AND rm.membership = 'join' + AND e.type = 'm.room.member' + AND e.received_ts >= ? + """ + txn.execute(sql, (user_id, timestamp)) + return frozenset([r[0] for r in txn]) + + return await self.db_pool.runInteraction( + "_get_rooms_for_user_by_join_date_txn", + _get_rooms_for_user_by_join_date_txn, + user_id, + from_ts, + ) + + async def set_room_participation(self, user_id: str, room_id: str) -> None: + """ + Record the provided user as participating in the given room + + Args: + user_id: the user ID of the user + room_id: ID of the room to set the participant in + """ + + def _set_room_participation_txn( + txn: LoggingTransaction, user_id: str, room_id: str + ) -> None: + sql = """ + UPDATE room_memberships + SET participant = true + WHERE event_id IN ( + SELECT event_id FROM local_current_membership + WHERE user_id = ? AND room_id = ? + ) + AND NOT participant + """ + txn.execute(sql, (user_id, room_id)) + + await self.db_pool.runInteraction( + "_set_room_participation_txn", _set_room_participation_txn, user_id, room_id + ) + + async def get_room_participation(self, user_id: str, room_id: str) -> bool: + """ + Check whether a user is listed as a participant in a room + + Args: + user_id: user ID of the user + room_id: ID of the room to check in + """ + + def _get_room_participation_txn( + txn: LoggingTransaction, user_id: str, room_id: str + ) -> bool: + sql = """ + SELECT participant + FROM local_current_membership AS l + INNER JOIN room_memberships AS r USING (event_id) + WHERE l.user_id = ? + AND l.room_id = ? + """ + txn.execute(sql, (user_id, room_id)) + res = txn.fetchone() + if res: + return res[0] + return False + + return await self.db_pool.runInteraction( + "_get_room_participation_txn", _get_room_participation_txn, user_id, room_id + ) + + async def get_ban_event_ids_in_room(self, room_id: str) -> StrCollection: + """Get all event IDs for ban events in the given room.""" + return await self.db_pool.simple_select_onecol( + table="current_state_events", + keyvalues={ + "room_id": room_id, + "type": EventTypes.Member, + "membership": Membership.BAN, + }, + retcol="event_id", + desc="get_ban_event_ids_in_room", + ) + class RoomMemberBackgroundUpdateStore(SQLBaseStore): def __init__( diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index 1d5c5e72ff..47dfdf64e5 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -49,6 +49,7 @@ from synapse.storage.database import ( from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.types import JsonDict +from synapse.util.events import get_plain_text_topic_from_event_content if TYPE_CHECKING: from synapse.server import HomeServer @@ -212,7 +213,9 @@ class SearchBackgroundUpdateStore(SearchWorkerStore): value = content["body"] elif etype == "m.room.topic": key = "content.topic" - value = content["topic"] + value = ( + get_plain_text_topic_from_event_content(content) or "", + ) elif etype == "m.room.name": key = "content.name" value = content["name"] diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 874dfdcb77..72ec8e6b90 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -1,7 +1,7 @@ # # This file is licensed under the Affero General Public License (AGPL) version 3. # -# Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2023, 2025 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -61,6 +61,21 @@ class SlidingSyncStore(SQLBaseStore): columns=("required_state_id",), ) + self.db_pool.updates.register_background_index_update( + update_name="sliding_sync_membership_snapshots_membership_event_id_idx", + index_name="sliding_sync_membership_snapshots_membership_event_id_idx", + table="sliding_sync_membership_snapshots", + columns=("membership_event_id",), + ) + + self.db_pool.updates.register_background_index_update( + update_name="sliding_sync_membership_snapshots_user_id_stream_ordering", + index_name="sliding_sync_membership_snapshots_user_id_stream_ordering", + table="sliding_sync_membership_snapshots", + columns=("user_id", "event_stream_ordering"), + replaces_index="sliding_sync_membership_snapshots_user_id", + ) + async def get_latest_bump_stamp_for_room( self, room_id: str, @@ -477,7 +492,7 @@ class PerConnectionStateDB: """An equivalent to `PerConnectionState` that holds data in a format stored in the DB. - The principle difference is that the tokens for the different streams are + The principal difference is that the tokens for the different streams are serialized to strings. When persisting this *only* contains updates to the state. diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 788f7d1e32..cfcc731f86 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -990,11 +990,12 @@ class StateMapWrapper(Dict[StateKey, str]): raise Exception("State map was filtered and doesn't include: %s", key) return super().__getitem__(key) + @overload # type: ignore[override] + def get(self, key: StateKey, default: None = None, /) -> Optional[str]: ... @overload - def get(self, key: Tuple[str, str]) -> Optional[str]: ... - + def get(self, key: StateKey, default: str, /) -> str: ... @overload - def get(self, key: Tuple[str, str], default: Union[str, _T]) -> Union[str, _T]: ... + def get(self, key: StateKey, default: _T, /) -> Union[str, _T]: ... def get( self, key: StateKey, default: Union[str, _T, None] = None diff --git a/synapse/storage/databases/main/state_deltas.py b/synapse/storage/databases/main/state_deltas.py index 117ee89d0a..00f87cc3a1 100644 --- a/synapse/storage/databases/main/state_deltas.py +++ b/synapse/storage/databases/main/state_deltas.py @@ -98,9 +98,9 @@ class StateDeltasStore(SQLBaseStore): prev_stream_id = int(prev_stream_id) # check we're not going backwards - assert ( - prev_stream_id <= max_stream_id - ), f"New stream id {max_stream_id} is smaller than prev stream id {prev_stream_id}" + assert prev_stream_id <= max_stream_id, ( + f"New stream id {max_stream_id} is smaller than prev stream id {prev_stream_id}" + ) if not self._curr_state_delta_stream_cache.has_any_entity_changed( prev_stream_id @@ -243,6 +243,13 @@ class StateDeltasStore(SQLBaseStore): (> `from_token` and <= `to_token`) """ + # We can bail early if the `from_token` is after the `to_token` + if ( + to_token is not None + and from_token is not None + and to_token.is_before_or_eq(from_token) + ): + return [] if ( from_token is not None diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py index 79c49e7fd9..74830b7129 100644 --- a/synapse/storage/databases/main/stats.py +++ b/synapse/storage/databases/main/stats.py @@ -48,6 +48,7 @@ from synapse.storage.databases.main.events_worker import InvalidEventError from synapse.storage.databases.main.state_deltas import StateDeltasStore from synapse.types import JsonDict from synapse.util.caches.descriptors import cached +from synapse.util.events import get_plain_text_topic_from_event_content if TYPE_CHECKING: from synapse.server import HomeServer @@ -611,7 +612,9 @@ class StatsStore(StateDeltasStore): elif event.type == EventTypes.Name: room_state["name"] = event.content.get("name") elif event.type == EventTypes.Topic: - room_state["topic"] = event.content.get("topic") + room_state["topic"] = get_plain_text_topic_from_event_content( + event.content + ) elif event.type == EventTypes.RoomAvatar: room_state["avatar"] = event.content.get("url") elif event.type == EventTypes.CanonicalAlias: diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index b4258a4436..66280f2f9a 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -50,6 +50,7 @@ from typing import ( Dict, Iterable, List, + Literal, Mapping, Optional, Protocol, @@ -60,8 +61,7 @@ from typing import ( ) import attr -from immutabledict import immutabledict -from typing_extensions import Literal, assert_never +from typing_extensions import assert_never from twisted.internet import defer @@ -79,6 +79,7 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine +from synapse.storage.roommember import RoomsForUserStateReset from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import PersistedEventPosition, RoomStreamToken, StrCollection from synapse.util.caches.descriptors import cached, cachedList @@ -452,6 +453,8 @@ def _filter_results_by_stream( stream_ordering falls between the two tokens (taking a None token to mean unbounded). + The token range is defined by > `lower_token` and <= `upper_token`. + Used to filter results from fetching events in the DB against the given tokens. This is necessary to handle the case where the tokens include position maps, which we handle by fetching more than necessary from the DB @@ -613,12 +616,15 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): max_value=events_max, ) self._events_stream_cache = StreamChangeCache( - "EventsRoomStreamChangeCache", - min_event_val, + name="EventsRoomStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_event_val, prefilled_cache=event_cache_prefill, ) self._membership_stream_cache = StreamChangeCache( - "MembershipStreamChangeCache", events_max + name="MembershipStreamChangeCache", + server_name=self.server_name, + current_stream_pos=events_max, ) self._stream_order_on_start = self.get_room_max_stream_ordering() @@ -650,23 +656,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): component. """ - min_pos = self._stream_id_gen.get_current_token() - - positions = {} - if isinstance(self._stream_id_gen, MultiWriterIdGenerator): - # The `min_pos` is the minimum position that we know all instances - # have finished persisting to, so we only care about instances whose - # positions are ahead of that. (Instance positions can be behind the - # min position as there are times we can work out that the minimum - # position is ahead of the naive minimum across all current - # positions. See MultiWriterIdGenerator for details) - positions = { - i: p - for i, p in self._stream_id_gen.get_positions().items() - if p > min_pos - } - - return RoomStreamToken(stream=min_pos, instance_map=immutabledict(positions)) + return RoomStreamToken.from_generator(self._stream_id_gen) def get_events_stream_id_generator(self) -> MultiWriterIdGenerator: return self._stream_id_gen @@ -990,6 +980,10 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): available in the `current_state_delta_stream` table. To actually check for a state reset, you need to check if a membership still exists in the room. """ + + assert from_key.topological is None + assert to_key.topological is None + # Start by ruling out cases where a DB query is not necessary. if from_key == to_key: return [] @@ -1135,6 +1129,203 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): if membership_change.room_id not in room_ids_to_exclude ] + @trace + async def get_sliding_sync_membership_changes( + self, + user_id: str, + from_key: RoomStreamToken, + to_key: RoomStreamToken, + excluded_room_ids: Optional[AbstractSet[str]] = None, + ) -> Dict[str, RoomsForUserStateReset]: + """ + Fetch membership events that result in a meaningful membership change for a + given user. + + A meaningful membership changes is one where the `membership` value actually + changes. This means memberships changes from `join` to `join` (like a display + name change) will be filtered out since they result in no meaningful change. + + Note: This function only works with "live" tokens with `stream_ordering` only. + + We're looking for membership changes in the token range (> `from_key` and <= + `to_key`). + + Args: + user_id: The user ID to fetch membership events for. + from_key: The point in the stream to sync from (fetching events > this point). + to_key: The token to fetch rooms up to (fetching events <= this point). + excluded_room_ids: Optional list of room IDs to exclude from the results. + + Returns: + All meaningful membership changes to the current state in the token range. + Events are sorted by `stream_ordering` ascending. + + `event_id`/`sender` can be `None` when the server leaves a room (meaning + everyone locally left) or a state reset which removed the person from the + room. We can't tell the difference between the two cases with what's + available in the `current_state_delta_stream` table. To actually check for a + state reset, you need to check if a membership still exists in the room. + """ + + assert from_key.topological is None + assert to_key.topological is None + + # Start by ruling out cases where a DB query is not necessary. + if from_key == to_key: + return {} + + if from_key: + has_changed = self._membership_stream_cache.has_entity_changed( + user_id, int(from_key.stream) + ) + if not has_changed: + return {} + + room_ids_to_exclude: AbstractSet[str] = set() + if excluded_room_ids is not None: + room_ids_to_exclude = excluded_room_ids + + def f(txn: LoggingTransaction) -> Dict[str, RoomsForUserStateReset]: + # To handle tokens with a non-empty instance_map we fetch more + # results than necessary and then filter down + min_from_id = from_key.stream + max_to_id = to_key.get_max_stream_pos() + + # This query looks at membership changes in + # `sliding_sync_membership_snapshots` which will not include users + # that were state reset out of rooms; so we need to look for that + # case in `current_state_delta_stream`. + sql = """ + SELECT + room_id, + membership_event_id, + event_instance_name, + event_stream_ordering, + membership, + sender, + prev_membership, + room_version + FROM + ( + SELECT + s.room_id, + s.membership_event_id, + s.event_instance_name, + s.event_stream_ordering, + s.membership, + s.sender, + m_prev.membership AS prev_membership + FROM sliding_sync_membership_snapshots as s + LEFT JOIN event_edges AS e ON e.event_id = s.membership_event_id + LEFT JOIN room_memberships AS m_prev ON m_prev.event_id = e.prev_event_id + WHERE s.user_id = ? + + UNION ALL + + SELECT + s.room_id, + e.event_id, + s.instance_name, + s.stream_id, + m.membership, + e.sender, + m_prev.membership AS prev_membership + FROM current_state_delta_stream AS s + LEFT JOIN events AS e ON e.event_id = s.event_id + LEFT JOIN room_memberships AS m ON m.event_id = s.event_id + LEFT JOIN room_memberships AS m_prev ON m_prev.event_id = s.prev_event_id + WHERE + s.type = ? + AND s.state_key = ? + ) AS c + INNER JOIN rooms USING (room_id) + WHERE event_stream_ordering > ? AND event_stream_ordering <= ? + ORDER BY event_stream_ordering ASC + """ + + txn.execute( + sql, + (user_id, EventTypes.Member, user_id, min_from_id, max_to_id), + ) + + membership_changes: Dict[str, RoomsForUserStateReset] = {} + for ( + room_id, + membership_event_id, + event_instance_name, + event_stream_ordering, + membership, + sender, + prev_membership, + room_version_id, + ) in txn: + assert room_id is not None + assert event_stream_ordering is not None + + if room_id in room_ids_to_exclude: + continue + + if _filter_results_by_stream( + from_key, + to_key, + event_instance_name, + event_stream_ordering, + ): + # When the server leaves a room, it will insert new rows into the + # `current_state_delta_stream` table with `event_id = null` for all + # current state. This means we might already have a row for the + # leave event and then another for the same leave where the + # `event_id=null` but the `prev_event_id` is pointing back at the + # earlier leave event. We don't want to report the leave, if we + # already have a leave event. + if ( + membership_event_id is None + and prev_membership == Membership.LEAVE + ): + continue + + if membership_event_id is None and room_id in membership_changes: + # SUSPICIOUS: if we join a room and get state reset out of it + # in the same queried window, + # won't this ignore the 'state reset out of it' part? + continue + + # When `s.event_id = null`, we won't be able to get respective + # `room_membership` but can assume the user has left the room + # because this only happens when the server leaves a room + # (meaning everyone locally left) or a state reset which removed + # the person from the room. + membership = ( + membership if membership is not None else Membership.LEAVE + ) + + if membership == prev_membership: + # If `membership` and `prev_membership` are the same then this + # is not a meaningful change so we can skip it. + # An example of this happening is when the user changes their display name. + continue + + membership_change = RoomsForUserStateReset( + room_id=room_id, + sender=sender, + membership=membership, + event_id=membership_event_id, + event_pos=PersistedEventPosition( + event_instance_name, event_stream_ordering + ), + room_version_id=room_version_id, + ) + + membership_changes[room_id] = membership_change + + return membership_changes + + membership_changes = await self.db_pool.runInteraction( + "get_sliding_sync_membership_changes", f + ) + + return membership_changes + @cancellable async def get_membership_changes_for_user( self, @@ -1837,15 +2028,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): dict """ - stream_ordering, topological_ordering = cast( - Tuple[int, int], - self.db_pool.simple_select_one_txn( - txn, - "events", - keyvalues={"event_id": event_id, "room_id": room_id}, - retcols=["stream_ordering", "topological_ordering"], - ), + row = self.db_pool.simple_select_one_txn( + txn, + "events", + keyvalues={"event_id": event_id, "room_id": room_id}, + retcols=("stream_ordering", "topological_ordering"), ) + stream_ordering = int(row[0]) + topological_ordering = int(row[1]) # Paginating backwards includes the event at the token, but paginating # forward doesn't. diff --git a/synapse/storage/databases/main/tags.py b/synapse/storage/databases/main/tags.py index 44f395f315..97b190bccc 100644 --- a/synapse/storage/databases/main/tags.py +++ b/synapse/storage/databases/main/tags.py @@ -274,10 +274,7 @@ class TagsWorkerStore(AccountDataWorkerStore): assert isinstance(self._account_data_id_gen, AbstractStreamIdGenerator) def remove_tag_txn(txn: LoggingTransaction, next_id: int) -> None: - sql = ( - "DELETE FROM room_tags " - " WHERE user_id = ? AND room_id = ? AND tag = ?" - ) + sql = "DELETE FROM room_tags WHERE user_id = ? AND room_id = ? AND tag = ?" txn.execute(sql, (user_id, room_id, tag)) self._update_revision_txn(txn, user_id, room_id, next_id) diff --git a/synapse/storage/databases/main/thread_subscriptions.py b/synapse/storage/databases/main/thread_subscriptions.py new file mode 100644 index 0000000000..50084887a4 --- /dev/null +++ b/synapse/storage/databases/main/thread_subscriptions.py @@ -0,0 +1,594 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +import logging +from typing import ( + TYPE_CHECKING, + Any, + FrozenSet, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) + +import attr + +from synapse.replication.tcp.streams._base import ThreadSubscriptionsStream +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) +from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore +from synapse.storage.util.id_generators import MultiWriterIdGenerator +from synapse.types import EventOrderings +from synapse.util.caches.descriptors import cached + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ThreadSubscription: + automatic: bool + """ + whether the subscription was made automatically (as opposed to by manual + action from the user) + """ + + +class AutomaticSubscriptionConflicted: + """ + Marker return value to signal that an automatic subscription was skipped, + because it conflicted with an unsubscription that we consider to have + been made later than the event causing the automatic subscription. + """ + + +class ThreadSubscriptionsWorkerStore(CacheInvalidationWorkerStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self._can_write_to_thread_subscriptions = ( + self._instance_name in hs.config.worker.writers.thread_subscriptions + ) + + self._thread_subscriptions_id_gen: MultiWriterIdGenerator = ( + MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name="thread_subscriptions", + server_name=self.server_name, + instance_name=self._instance_name, + tables=[ + ("thread_subscriptions", "instance_name", "stream_id"), + ], + sequence_name="thread_subscriptions_sequence", + writers=hs.config.worker.writers.thread_subscriptions, + ) + ) + + def process_replication_rows( + self, + stream_name: str, + instance_name: str, + token: int, + rows: Iterable[Any], + ) -> None: + if stream_name == ThreadSubscriptionsStream.NAME: + for row in rows: + self.get_subscription_for_thread.invalidate( + (row.user_id, row.room_id, row.event_id) + ) + self.get_subscribers_to_thread.invalidate((row.room_id, row.event_id)) + + super().process_replication_rows(stream_name, instance_name, token, rows) + + def process_replication_position( + self, stream_name: str, instance_name: str, token: int + ) -> None: + if stream_name == ThreadSubscriptionsStream.NAME: + self._thread_subscriptions_id_gen.advance(instance_name, token) + super().process_replication_position(stream_name, instance_name, token) + + @staticmethod + def _should_skip_autosubscription_after_unsubscription( + *, + autosub: EventOrderings, + unsubscribed_at: EventOrderings, + ) -> bool: + """ + Returns whether an automatic subscription occurring *after* an unsubscription + should be skipped, because the unsubscription already 'acknowledges' the event + causing the automatic subscription (the cause event). + + To determine *after*, we use `stream_ordering` unless the event is backfilled + (negative `stream_ordering`) and fallback to topological ordering. + + Args: + autosub: the stream_ordering and topological_ordering of the cause event + unsubscribed_at: + the maximum stream ordering and the maximum topological ordering at the time of unsubscription + + Returns: + True if the automatic subscription should be skipped + """ + # For normal rooms, these two orderings should be positive, because + # they don't refer to a specific event but rather the maximum at the + # time of unsubscription. + # + # However, for rooms that have never been joined and that are being peeked at, + # we might not have a single non-backfilled event and therefore the stream + # ordering might be negative, so we don't assert this case. + assert unsubscribed_at.topological > 0 + + unsubscribed_at_backfilled = unsubscribed_at.stream < 0 + if ( + not unsubscribed_at_backfilled + and unsubscribed_at.stream >= autosub.stream > 0 + ): + # non-backfilled events: the unsubscription is later according to + # the stream + return True + + if autosub.stream < 0: + # the auto-subscription cause event was backfilled, so fall back to + # topological ordering + if unsubscribed_at.topological >= autosub.topological: + return True + + return False + + async def subscribe_user_to_thread( + self, + user_id: str, + room_id: str, + thread_root_event_id: str, + *, + automatic_event_orderings: Optional[EventOrderings], + ) -> Optional[Union[int, AutomaticSubscriptionConflicted]]: + """Updates a user's subscription settings for a specific thread root. + + If no change would be made to the subscription, does not produce any database change. + + Case-by-case: + - if we already have an automatic subscription: + - new automatic subscriptions will be no-ops (no database write), + - new manual subscriptions will overwrite the automatic subscription + - if we already have a manual subscription: + we don't update (no database write) in either case, because: + - the existing manual subscription wins over a new automatic subscription request + - there would be no need to write a manual subscription because we already have one + + Args: + user_id: The ID of the user whose settings are being updated. + room_id: The ID of the room the thread root belongs to. + thread_root_event_id: The event ID of the thread root. + automatic_event_orderings: + Value depends on whether the subscription was performed automatically by the user's client. + For manual subscriptions: None. + For automatic subscriptions: the orderings of the event. + + Returns: + If a subscription is made: (int) the stream ID for this update. + If a subscription already exists and did not need to be updated: None + If an automatic subscription conflicted with an unsubscription: AutomaticSubscriptionConflicted + """ + assert self._can_write_to_thread_subscriptions + + def _invalidate_subscription_caches(txn: LoggingTransaction) -> None: + txn.call_after( + self.get_subscription_for_thread.invalidate, + (user_id, room_id, thread_root_event_id), + ) + txn.call_after( + self.get_subscribers_to_thread.invalidate, + (room_id, thread_root_event_id), + ) + + def _subscribe_user_to_thread_txn( + txn: LoggingTransaction, + ) -> Optional[Union[int, AutomaticSubscriptionConflicted]]: + requested_automatic = automatic_event_orderings is not None + + row = self.db_pool.simple_select_one_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + }, + retcols=( + "subscribed", + "automatic", + "unsubscribed_at_stream_ordering", + "unsubscribed_at_topological_ordering", + ), + allow_none=True, + ) + + if row is None: + # We have never subscribed before, simply insert the row and finish + stream_id = self._thread_subscriptions_id_gen.get_next_txn(txn) + self.db_pool.simple_insert_txn( + txn, + table="thread_subscriptions", + values={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + "subscribed": True, + "stream_id": stream_id, + "instance_name": self._instance_name, + "automatic": requested_automatic, + "unsubscribed_at_stream_ordering": None, + "unsubscribed_at_topological_ordering": None, + }, + ) + _invalidate_subscription_caches(txn) + return stream_id + + # we already have either a subscription or a prior unsubscription here + ( + subscribed, + already_automatic, + unsubscribed_at_stream_ordering, + unsubscribed_at_topological_ordering, + ) = row + + if subscribed and (not already_automatic or requested_automatic): + # we are already subscribed and the current subscription state + # is good enough (either we already have a manual subscription, + # or we requested an automatic subscription) + # In that case, nothing to change here. + # (See docstring for case-by-case explanation) + return None + + if not subscribed and requested_automatic: + assert automatic_event_orderings is not None + # we previously unsubscribed and we are now automatically subscribing + # Check whether the new autosubscription should be skipped + if ThreadSubscriptionsWorkerStore._should_skip_autosubscription_after_unsubscription( + autosub=automatic_event_orderings, + unsubscribed_at=EventOrderings( + unsubscribed_at_stream_ordering, + unsubscribed_at_topological_ordering, + ), + ): + # skip the subscription + return AutomaticSubscriptionConflicted() + + # At this point: we have now finished checking that we need to make + # a subscription, updating the current row. + + stream_id = self._thread_subscriptions_id_gen.get_next_txn(txn) + self.db_pool.simple_update_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + }, + updatevalues={ + "subscribed": True, + "stream_id": stream_id, + "instance_name": self._instance_name, + "automatic": requested_automatic, + "unsubscribed_at_stream_ordering": None, + "unsubscribed_at_topological_ordering": None, + }, + ) + _invalidate_subscription_caches(txn) + + return stream_id + + return await self.db_pool.runInteraction( + "subscribe_user_to_thread", _subscribe_user_to_thread_txn + ) + + async def unsubscribe_user_from_thread( + self, user_id: str, room_id: str, thread_root_event_id: str + ) -> Optional[int]: + """Unsubscribes a user from a thread. + + If no change would be made to the subscription, does not produce any database change. + + Args: + user_id: The ID of the user whose settings are being updated. + room_id: The ID of the room the thread root belongs to. + thread_root_event_id: The event ID of the thread root. + + Returns: + The stream ID for this update, if the update isn't no-opped. + """ + + assert self._can_write_to_thread_subscriptions + + def _unsubscribe_user_from_thread_txn(txn: LoggingTransaction) -> Optional[int]: + already_subscribed = self.db_pool.simple_select_one_onecol_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + }, + retcol="subscribed", + allow_none=True, + ) + + if already_subscribed is None or already_subscribed is False: + # there is nothing we need to do here + return None + + stream_id = self._thread_subscriptions_id_gen.get_next_txn(txn) + + # Find the maximum stream ordering and topological ordering of the room, + # which we then store against this unsubscription so we can skip future + # automatic subscriptions that are caused by an event logically earlier + # than this unsubscription. + txn.execute( + """ + SELECT MAX(stream_ordering) AS mso, MAX(topological_ordering) AS mto FROM events + WHERE room_id = ? + """, + (room_id,), + ) + ord_row = txn.fetchone() + assert ord_row is not None + max_stream_ordering, max_topological_ordering = ord_row + + self.db_pool.simple_update_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + "subscribed": True, + }, + updatevalues={ + "subscribed": False, + "stream_id": stream_id, + "instance_name": self._instance_name, + "unsubscribed_at_stream_ordering": max_stream_ordering, + "unsubscribed_at_topological_ordering": max_topological_ordering, + }, + ) + + txn.call_after( + self.get_subscription_for_thread.invalidate, + (user_id, room_id, thread_root_event_id), + ) + txn.call_after( + self.get_subscribers_to_thread.invalidate, + (room_id, thread_root_event_id), + ) + + return stream_id + + return await self.db_pool.runInteraction( + "unsubscribe_user_from_thread", _unsubscribe_user_from_thread_txn + ) + + async def purge_thread_subscription_settings_for_user(self, user_id: str) -> None: + """ + Purge all subscriptions for the user. + The fact that subscriptions have been purged will not be streamed; + all stream rows for the user will in fact be removed. + + This must only be used for user deactivation, + because it does not invalidate the `subscribers_to_thread` cache. + """ + + def _purge_thread_subscription_settings_for_user_txn( + txn: LoggingTransaction, + ) -> None: + self.db_pool.simple_delete_txn( + txn, + table="thread_subscriptions", + keyvalues={"user_id": user_id}, + ) + self._invalidate_cache_and_stream( + txn, self.get_subscription_for_thread, (user_id,) + ) + + await self.db_pool.runInteraction( + desc="purge_thread_subscription_settings_for_user", + func=_purge_thread_subscription_settings_for_user_txn, + ) + + @cached(tree=True) + async def get_subscription_for_thread( + self, user_id: str, room_id: str, thread_root_event_id: str + ) -> Optional[ThreadSubscription]: + """Get the thread subscription for a specific thread and user. + + Args: + user_id: The ID of the user + room_id: The ID of the room + thread_root_event_id: The event ID of the thread root + + Returns: + A `ThreadSubscription` dataclass if there is a subscription, + or `None` if there is no subscription. + + If there is a row in the table but `subscribed` is `False`, + behaves the same as if there was no row at all and returns `None`. + """ + row = await self.db_pool.simple_select_one( + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "room_id": room_id, + "event_id": thread_root_event_id, + "subscribed": True, + }, + retcols=("automatic",), + allow_none=True, + desc="get_subscription_for_thread", + ) + + if row is None: + return None + + (automatic_rawbool,) = row + + # convert SQLite integer booleans into real booleans + automatic = bool(automatic_rawbool) + + return ThreadSubscription(automatic=automatic) + + # max_entries=100 rationale: + # this returns a potentially large datastructure + # (since each entry contains a set which contains a potentially large number of user IDs), + # whereas the default of 10'000 entries for @cached feels more + # suitable for very small cache entries. + # + # Overall, when bearing in mind the usual profile of a small community-server or company-server + # (where cache tuning hasn't been done, so we're in out-of-box configuration), it is very + # unlikely we would benefit from keeping hot the subscribers for as many as 100 threads, + # since it's unlikely that so many threads will be active in a short span of time on a small homeserver. + # It feels that medium servers will probably also not exhaust this limit. + # Larger homeservers are more likely to be carefully tuned, either with a larger global cache factor + # or carefully following the usage patterns & cache metrics. + # Finally, the query is not so intensive that computing it every time is a huge deal, but given people + # often send messages back-to-back in the same thread it seems like it would offer a mild benefit. + @cached(max_entries=100) + async def get_subscribers_to_thread( + self, room_id: str, thread_root_event_id: str + ) -> FrozenSet[str]: + """ + Returns: + the set of user_ids for local users who are subscribed to the given thread. + """ + return frozenset( + await self.db_pool.simple_select_onecol( + table="thread_subscriptions", + keyvalues={ + "room_id": room_id, + "event_id": thread_root_event_id, + "subscribed": True, + }, + retcol="user_id", + desc="get_subscribers_to_thread", + ) + ) + + def get_max_thread_subscriptions_stream_id(self) -> int: + """Get the current maximum stream_id for thread subscriptions. + + Returns: + The maximum stream_id + """ + return self._thread_subscriptions_id_gen.get_current_token() + + def get_thread_subscriptions_stream_id_generator(self) -> MultiWriterIdGenerator: + return self._thread_subscriptions_id_gen + + async def get_updated_thread_subscriptions( + self, *, from_id: int, to_id: int, limit: int + ) -> List[Tuple[int, str, str, str]]: + """Get updates to thread subscriptions between two stream IDs. + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + list of (stream_id, user_id, room_id, thread_root_id) tuples + """ + + def get_updated_thread_subscriptions_txn( + txn: LoggingTransaction, + ) -> List[Tuple[int, str, str, str]]: + sql = """ + SELECT stream_id, user_id, room_id, event_id + FROM thread_subscriptions + WHERE ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + """ + + txn.execute(sql, (from_id, to_id, limit)) + return cast(List[Tuple[int, str, str, str]], txn.fetchall()) + + return await self.db_pool.runInteraction( + "get_updated_thread_subscriptions", + get_updated_thread_subscriptions_txn, + ) + + async def get_latest_updated_thread_subscriptions_for_user( + self, user_id: str, *, from_id: int, to_id: int, limit: int + ) -> List[Tuple[int, str, str, bool, Optional[bool]]]: + """Get the latest updates to thread subscriptions for a specific user. + + Args: + user_id: The ID of the user + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + If there are too many rows to return, rows from the start (closer to `from_id`) + will be omitted. + + Returns: + A list of (stream_id, room_id, thread_root_event_id, subscribed, automatic) tuples. + The row with lowest `stream_id` is the first row. + """ + + def get_updated_thread_subscriptions_for_user_txn( + txn: LoggingTransaction, + ) -> List[Tuple[int, str, str, bool, Optional[bool]]]: + sql = """ + WITH the_updates AS ( + SELECT stream_id, room_id, event_id, subscribed, automatic + FROM thread_subscriptions + WHERE user_id = ? AND ? < stream_id AND stream_id <= ? + ORDER BY stream_id DESC + LIMIT ? + ) + SELECT stream_id, room_id, event_id, subscribed, automatic + FROM the_updates + ORDER BY stream_id ASC + """ + + txn.execute(sql, (user_id, from_id, to_id, limit)) + return [ + ( + stream_id, + room_id, + event_id, + # SQLite integer to boolean conversions + bool(subscribed), + bool(automatic) if subscribed else None, + ) + for (stream_id, room_id, event_id, subscribed, automatic) in txn + ] + + return await self.db_pool.runInteraction( + "get_updated_thread_subscriptions_for_user", + get_updated_thread_subscriptions_for_user_txn, + ) diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index 770802483c..bfc324b80d 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -86,10 +86,10 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): @wrap_as_background_process("cleanup_transactions") async def _cleanup_transactions(self) -> None: now = self._clock.time_msec() - month_ago = now - 30 * 24 * 60 * 60 * 1000 + day_ago = now - 24 * 60 * 60 * 1000 def _cleanup_transactions_txn(txn: LoggingTransaction) -> None: - txn.execute("DELETE FROM received_transactions WHERE ts < ?", (month_ago,)) + txn.execute("DELETE FROM received_transactions WHERE ts < ?", (day_ago,)) await self.db_pool.runInteraction( "_cleanup_transactions", _cleanup_transactions_txn diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 51cffb0986..9deb9ab73c 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -31,22 +31,14 @@ from typing import ( Sequence, Set, Tuple, + TypedDict, cast, ) import attr -try: - # Figure out if ICU support is available for searching users. - import icu - - USE_ICU = True -except ModuleNotFoundError: - USE_ICU = False - -from typing_extensions import TypedDict - from synapse.api.errors import StoreError +from synapse.synapse_rust import segmenter as icu from synapse.util.stringutils import non_null_str_or_none if TYPE_CHECKING: @@ -253,8 +245,9 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore): return 1 logger.debug( - "Processing the next %d rooms of %d remaining" - % (len(rooms_to_work_on), progress["remaining"]) + "Processing the next %d rooms of %d remaining", + len(rooms_to_work_on), + progress["remaining"], ) processed_event_count = 0 @@ -583,9 +576,9 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore): retry_counter: number of failures in refreshing the profile so far. Used for exponential backoff calculations. """ - assert not self.hs.is_mine_id( - user_id - ), "Can't mark a local user as a stale remote user." + assert not self.hs.is_mine_id(user_id), ( + "Can't mark a local user as a stale remote user." + ) server_name = UserID.from_string(user_id).domain @@ -1038,11 +1031,11 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): } """ + join_args: Tuple[str, ...] = (user_id,) + if self.hs.config.userdirectory.user_directory_search_all_users: - join_args = (user_id,) where_clause = "user_id != ?" else: - join_args = (user_id,) where_clause = """ ( EXISTS (select 1 from users_in_public_rooms WHERE user_id = t.user_id) @@ -1056,6 +1049,14 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): if not show_locked_users: where_clause += " AND (u.locked IS NULL OR u.locked = FALSE)" + # Adjust the JOIN type based on the exclude_remote_users flag (the users + # table only contains local users so an inner join is a good way to + # to exclude remote users) + if self.hs.config.userdirectory.user_directory_exclude_remote_users: + join_type = "JOIN" + else: + join_type = "LEFT JOIN" + # We allow manipulating the ranking algorithm by injecting statements # based on config options. additional_ordering_statements = [] @@ -1087,7 +1088,7 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): SELECT d.user_id AS user_id, display_name, avatar_url FROM matching_users as t INNER JOIN user_directory AS d USING (user_id) - LEFT JOIN users AS u ON t.user_id = u.name + %(join_type)s users AS u ON t.user_id = u.name WHERE %(where_clause)s ORDER BY @@ -1116,6 +1117,7 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): """ % { "where_clause": where_clause, "order_case_statements": " ".join(additional_ordering_statements), + "join_type": join_type, } args = ( (full_query,) @@ -1143,7 +1145,7 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): SELECT d.user_id AS user_id, display_name, avatar_url FROM user_directory_search as t INNER JOIN user_directory AS d USING (user_id) - LEFT JOIN users AS u ON t.user_id = u.name + %(join_type)s users AS u ON t.user_id = u.name WHERE %(where_clause)s AND value MATCH ? @@ -1156,6 +1158,7 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): """ % { "where_clause": where_clause, "order_statements": " ".join(additional_ordering_statements), + "join_type": join_type, } args = join_args + (search_query,) + ordering_arguments + (limit + 1,) else: @@ -1215,7 +1218,7 @@ def _filter_text_for_index(text: str) -> str: def _parse_query_sqlite(search_term: str) -> str: """Takes a plain unicode string from the user and converts it into a form - that can be passed to database. + that can be passed to the database. We use this so that we can add prefix matching, which isn't something that is supported by default. @@ -1231,14 +1234,20 @@ def _parse_query_sqlite(search_term: str) -> str: def _parse_query_postgres(search_term: str) -> Tuple[str, str, str]: """Takes a plain unicode string from the user and converts it into a form - that can be passed to database. + that can be passed to the database. We use this so that we can add prefix matching, which isn't something that is supported by default. """ search_term = _filter_text_for_index(search_term) escaped_words = [] - for word in _parse_words(search_term): + for index, word in enumerate(_parse_words(search_term)): + if index >= 10: + # We limit how many terms we include, as otherwise it can use + # excessive database time if people accidentally search for large + # strings. + break + # Postgres tsvector and tsquery quoting rules: # words potentially containing punctuation should be quoted # and then existing quotes and backslashes should be doubled @@ -1255,12 +1264,7 @@ def _parse_query_postgres(search_term: str) -> Tuple[str, str, str]: def _parse_words(search_term: str) -> List[str]: - """Split the provided search string into a list of its words. - - If support for ICU (International Components for Unicode) is available, use it. - Otherwise, fall back to using a regex to detect word boundaries. This latter - solution works well enough for most latin-based languages, but doesn't work as well - with other languages. + """Split the provided search string into a list of its words using ICU. Args: search_term: The search string. @@ -1268,18 +1272,7 @@ def _parse_words(search_term: str) -> List[str]: Returns: A list of the words in the search string. """ - if USE_ICU: - return _parse_words_with_icu(search_term) - - return _parse_words_with_regex(search_term) - - -def _parse_words_with_regex(search_term: str) -> List[str]: - """ - Break down search term into words, when we don't have ICU available. - See: `_parse_words` - """ - return re.findall(r"([\w-]+)", search_term, re.UNICODE) + return _parse_words_with_icu(search_term) def _parse_words_with_icu(search_term: str) -> List[str]: @@ -1293,22 +1286,13 @@ def _parse_words_with_icu(search_term: str) -> List[str]: A list of the words in the search string. """ results = [] - breaker = icu.BreakIterator.createWordInstance(icu.Locale.getDefault()) - breaker.setText(search_term) - i = 0 - while True: - j = breaker.nextBoundary() - if j < 0: - break - + for part in icu.parse_words(search_term): # We want to make sure that we split on `@` and `:` specifically, as # they occur in user IDs. - for result in re.split(r"[@:]+", search_term[i:j]): + for result in re.split(r"[@:]+", part): results.append(result.strip()) - i = j - - # libicu will break up words that have punctuation in them, but to handle + # icu will break up words that have punctuation in them, but to handle # cases where user IDs have '-', '.' and '_' in them we want to *not* break # those into words and instead allow the DB to tokenise them how it wants. # diff --git a/synapse/storage/databases/main/user_erasure_store.py b/synapse/storage/databases/main/user_erasure_store.py index bbde8491fd..cceed484c3 100644 --- a/synapse/storage/databases/main/user_erasure_store.py +++ b/synapse/storage/databases/main/user_erasure_store.py @@ -70,8 +70,6 @@ class UserErasureWorkerStore(CacheInvalidationWorkerStore): return {u: u in erased_users for u in user_ids} - -class UserErasureStore(UserErasureWorkerStore): async def mark_user_erased(self, user_id: str) -> None: """Indicate that user_id wishes their message history to be erased. @@ -113,3 +111,7 @@ class UserErasureStore(UserErasureWorkerStore): self._invalidate_cache_and_stream(txn, self.is_user_erased, (user_id,)) await self.db_pool.runInteraction("mark_user_not_erased", f) + + +class UserErasureStore(UserErasureWorkerStore): + pass diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index f7824cba0f..ac38b2ab19 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -20,7 +20,15 @@ # import logging -from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Dict, + List, + Mapping, + Optional, + Tuple, + Union, +) from synapse.logging.opentracing import tag_args, trace from synapse.storage._base import SQLBaseStore @@ -282,16 +290,6 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): STATE_GROUPS_ROOM_INDEX_UPDATE_NAME = "state_groups_room_id_idx" STATE_GROUP_EDGES_UNIQUE_INDEX_UPDATE_NAME = "state_group_edges_unique_idx" - CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( - "current_state_events_stream_ordering_idx" - ) - ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( - "room_memberships_stream_ordering_idx" - ) - LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME = ( - "local_current_membership_stream_ordering_idx" - ) - def __init__( self, database: DatabasePool, @@ -328,27 +326,6 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): replaces_index="state_group_edges_idx", ) - # These indices are needed to validate the foreign key constraint - # when events are deleted. - self.db_pool.updates.register_background_index_update( - self.CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME, - index_name="current_state_events_stream_ordering_idx", - table="current_state_events", - columns=["event_stream_ordering"], - ) - self.db_pool.updates.register_background_index_update( - self.ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME, - index_name="room_memberships_stream_ordering_idx", - table="room_memberships", - columns=["event_stream_ordering"], - ) - self.db_pool.updates.register_background_index_update( - self.LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME, - index_name="local_current_membership_stream_ordering_idx", - table="local_current_membership", - columns=["event_stream_ordering"], - ) - async def _background_deduplicate_state( self, progress: dict, batch_size: int ) -> int: @@ -388,8 +365,7 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): return True, count txn.execute( - "SELECT state_group FROM state_group_edges" - " WHERE state_group = ?", + "SELECT state_group FROM state_group_edges WHERE state_group = ?", (state_group,), ) diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py new file mode 100644 index 0000000000..9b62c1d814 --- /dev/null +++ b/synapse/storage/databases/state/deletion.py @@ -0,0 +1,560 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +import contextlib +from typing import ( + TYPE_CHECKING, + AbstractSet, + AsyncIterator, + Collection, + Mapping, + Optional, + Set, + Tuple, +) + +from synapse.events.snapshot import EventPersistencePair +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, + make_in_list_sql_clause, +) +from synapse.storage.engines import PostgresEngine +from synapse.util.stringutils import shortstr + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +class StateDeletionDataStore: + """Manages deletion of state groups in a safe manner. + + Deleting state groups is challenging as before we actually delete them we + need to ensure that there are no in-flight events that refer to the state + groups that we want to delete. + + To handle this, we take two approaches. First, before we persist any event + we ensure that the state group still exists and mark in the + `state_groups_persisting` table that the state group is about to be used. + (Note that we have to have the extra table here as state groups and events + can be in different databases, and thus we can't check for the existence of + state groups in the persist event transaction). Once the event has been + persisted, we can remove the row from `state_groups_persisting`. So long as + we check that table before deleting state groups, we can ensure that we + never persist events that reference deleted state groups, maintaining + database integrity. + + However, we want to avoid throwing exceptions so deep in the process of + persisting events. So instead of deleting state groups immediately, we mark + them as pending/proposed for deletion and wait for a certain amount of time + before performing the deletion. When we come to handle new events that + reference state groups, we check if they are pending deletion and bump the + time for when they'll be deleted (to give a chance for the event to be + persisted, or not). + + When deleting, we need to check that state groups remain unreferenced. There + is a race here where we a) fetch state groups that are ready for deletion, + b) check they're unreferenced, c) the state group becomes referenced but + then gets marked as pending deletion again, d) during the deletion + transaction we recheck `state_groups_pending_deletion` table again and see + that it exists and so continue with the deletion. To prevent this from + happening we add a `sequence_number` column to + `state_groups_pending_deletion`, and during deletion we ensure that for a + state group we're about to delete that the sequence number doesn't change + between steps (a) and (d). So long as we always bump the sequence number + whenever an event may become used the race can never happen. + """ + + # How long to wait before we delete state groups. This should be long enough + # for any in-flight events to be persisted. If events take longer to persist + # and any of the state groups they reference have been deleted, then the + # event will fail to persist (as well as any event in the same batch). + DELAY_BEFORE_DELETION_MS = 10 * 60 * 1000 + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + self._clock = hs.get_clock() + self.db_pool = database + self._instance_name = hs.get_instance_name() + + with db_conn.cursor(txn_name="_clear_existing_persising") as txn: + self._clear_existing_persising(txn) + + def _clear_existing_persising(self, txn: LoggingTransaction) -> None: + """On startup we clear any entries in `state_groups_persisting` that + match our instance name, in case of a previous unclean shutdown""" + + self.db_pool.simple_delete_txn( + txn, + table="state_groups_persisting", + keyvalues={"instance_name": self._instance_name}, + ) + + async def check_state_groups_and_bump_deletion( + self, state_groups: AbstractSet[int] + ) -> Collection[int]: + """Checks to make sure that the state groups haven't been deleted, and + if they're pending deletion we delay it (allowing time for any event + that will use them to finish persisting). + + Returns: + The state groups that are missing, if any. + """ + + return await self.db_pool.runInteraction( + "check_state_groups_and_bump_deletion", + self._check_state_groups_and_bump_deletion_txn, + state_groups, + # We don't need to lock if we're just doing a quick check, as the + # lock doesn't prevent any races here. + lock=False, + ) + + def _check_state_groups_and_bump_deletion_txn( + self, txn: LoggingTransaction, state_groups: AbstractSet[int], lock: bool = True + ) -> Collection[int]: + """Checks to make sure that the state groups haven't been deleted, and + if they're pending deletion we delay it (allowing time for any event + that will use them to finish persisting). + + The `lock` flag sets if we should lock the `state_group` rows we're + checking, which we should do when storing new groups. + + Returns: + The state groups that are missing, if any. + """ + + existing_state_groups = self._get_existing_groups_with_lock( + txn, state_groups, lock=lock + ) + + self._bump_deletion_txn(txn, existing_state_groups) + + missing_state_groups = state_groups - existing_state_groups + if missing_state_groups: + return missing_state_groups + + return () + + def _bump_deletion_txn( + self, txn: LoggingTransaction, state_groups: Collection[int] + ) -> None: + """Update any pending deletions of the state group that they may now be + referenced.""" + + if not state_groups: + return + + now = self._clock.time_msec() + if isinstance(self.db_pool.engine, PostgresEngine): + clause, args = make_in_list_sql_clause( + self.db_pool.engine, "state_group", state_groups + ) + sql = f""" + UPDATE state_groups_pending_deletion + SET sequence_number = DEFAULT, insertion_ts = ? + WHERE {clause} + """ + args.insert(0, now) + txn.execute(sql, args) + else: + rows = self.db_pool.simple_select_many_txn( + txn, + table="state_groups_pending_deletion", + column="state_group", + iterable=state_groups, + keyvalues={}, + retcols=("state_group",), + ) + if not rows: + return + + state_groups_to_update = [state_group for (state_group,) in rows] + + self.db_pool.simple_delete_many_txn( + txn, + table="state_groups_pending_deletion", + column="state_group", + values=state_groups_to_update, + keyvalues={}, + ) + self.db_pool.simple_insert_many_txn( + txn, + table="state_groups_pending_deletion", + keys=("state_group", "insertion_ts"), + values=[(state_group, now) for state_group in state_groups_to_update], + ) + + def _get_existing_groups_with_lock( + self, txn: LoggingTransaction, state_groups: Collection[int], lock: bool = True + ) -> AbstractSet[int]: + """Return which of the given state groups are in the database, and locks + those rows with `KEY SHARE` to ensure they don't get concurrently + deleted (if `lock` is true).""" + clause, args = make_in_list_sql_clause(self.db_pool.engine, "id", state_groups) + + sql = f""" + SELECT id FROM state_groups + WHERE {clause} + """ + if lock and isinstance(self.db_pool.engine, PostgresEngine): + # On postgres we add a row level lock to the rows to ensure that we + # conflict with any concurrent DELETEs. `FOR KEY SHARE` lock will + # not conflict with other read + sql += """ + FOR KEY SHARE + """ + + txn.execute(sql, args) + return {state_group for (state_group,) in txn} + + @contextlib.asynccontextmanager + async def persisting_state_group_references( + self, event_and_contexts: Collection[EventPersistencePair] + ) -> AsyncIterator[None]: + """Wraps the persistence of the given events and contexts, ensuring that + any state groups referenced still exist and that they don't get deleted + during this.""" + + referenced_state_groups: Set[int] = set() + for event, ctx in event_and_contexts: + if ctx.rejected or event.internal_metadata.is_outlier(): + continue + + assert ctx.state_group is not None + + referenced_state_groups.add(ctx.state_group) + + if ctx.state_group_before_event: + referenced_state_groups.add(ctx.state_group_before_event) + + if not referenced_state_groups: + # We don't reference any state groups, so nothing to do + yield + return + + await self.db_pool.runInteraction( + "mark_state_groups_as_persisting", + self._mark_state_groups_as_persisting_txn, + referenced_state_groups, + ) + + error = True + try: + yield None + error = False + finally: + await self.db_pool.runInteraction( + "finish_persisting", + self._finish_persisting_txn, + referenced_state_groups, + error=error, + ) + + def _mark_state_groups_as_persisting_txn( + self, txn: LoggingTransaction, state_groups: Set[int] + ) -> None: + """Marks the given state groups as being persisted.""" + + existing_state_groups = self._get_existing_groups_with_lock(txn, state_groups) + missing_state_groups = state_groups - existing_state_groups + if missing_state_groups: + raise Exception( + f"state groups have been deleted: {shortstr(missing_state_groups)}" + ) + + self.db_pool.simple_insert_many_txn( + txn, + table="state_groups_persisting", + keys=("state_group", "instance_name"), + values=[(state_group, self._instance_name) for state_group in state_groups], + ) + + def _finish_persisting_txn( + self, txn: LoggingTransaction, state_groups: Collection[int], error: bool + ) -> None: + """Mark the state groups as having finished persistence. + + If `error` is true then we assume the state groups were not persisted, + and so we do not clear them from the pending deletion table. + """ + self.db_pool.simple_delete_many_txn( + txn, + table="state_groups_persisting", + column="state_group", + values=state_groups, + keyvalues={"instance_name": self._instance_name}, + ) + + if error: + # The state groups may or may not have been persisted, so we need to + # bump the deletion to ensure we recheck if they have become + # referenced. + self._bump_deletion_txn(txn, state_groups) + return + + self.db_pool.simple_delete_many_batch_txn( + txn, + table="state_groups_pending_deletion", + keys=("state_group",), + values=[(state_group,) for state_group in state_groups], + ) + + async def mark_state_groups_as_pending_deletion( + self, state_groups: Collection[int] + ) -> None: + """Mark the given state groups as pending deletion. + + If any of the state groups are already pending deletion, then those records are + left as is. + """ + + await self.db_pool.runInteraction( + "mark_state_groups_as_pending_deletion", + self._mark_state_groups_as_pending_deletion_txn, + state_groups, + ) + + def _mark_state_groups_as_pending_deletion_txn( + self, + txn: LoggingTransaction, + state_groups: Collection[int], + ) -> None: + sql = """ + INSERT INTO state_groups_pending_deletion (state_group, insertion_ts) + VALUES %s + ON CONFLICT (state_group) + DO NOTHING + """ + + now = self._clock.time_msec() + rows = [ + ( + state_group, + now, + ) + for state_group in state_groups + ] + if isinstance(txn.database_engine, PostgresEngine): + txn.execute_values(sql % ("?",), rows, fetch=False) + else: + txn.execute_batch(sql % ("(?, ?)",), rows) + + async def mark_state_groups_as_used(self, state_groups: Collection[int]) -> None: + """Mark the given state groups as now being referenced""" + + await self.db_pool.simple_delete_many( + table="state_groups_pending_deletion", + column="state_group", + iterable=state_groups, + keyvalues={}, + desc="mark_state_groups_as_used", + ) + + async def get_pending_deletions( + self, state_groups: Collection[int] + ) -> Mapping[int, int]: + """Get which state groups are pending deletion. + + Returns: + a mapping from state groups that are pending deletion to their + sequence number + """ + + rows = await self.db_pool.simple_select_many_batch( + table="state_groups_pending_deletion", + column="state_group", + iterable=state_groups, + retcols=("state_group", "sequence_number"), + keyvalues={}, + desc="get_pending_deletions", + ) + + return dict(rows) + + def get_state_groups_ready_for_potential_deletion_txn( + self, + txn: LoggingTransaction, + state_groups_to_sequence_numbers: Mapping[int, int], + ) -> Collection[int]: + """Given a set of state groups, return which state groups can + potentially be deleted. + + The state groups must have been checked to see if they remain + unreferenced before calling this function. + + Note: This must be called within the same transaction that the state + groups are deleted. + + Args: + state_groups_to_sequence_numbers: The state groups, and the sequence + numbers from before the state groups were checked to see if they + were unreferenced. + + Returns: + The subset of state groups that can safely be deleted + + """ + + if not state_groups_to_sequence_numbers: + return state_groups_to_sequence_numbers + + if isinstance(self.db_pool.engine, PostgresEngine): + # On postgres we want to lock the rows FOR UPDATE as early as + # possible to help conflicts. + clause, args = make_in_list_sql_clause( + self.db_pool.engine, "id", state_groups_to_sequence_numbers + ) + sql = f""" + SELECT id FROM state_groups + WHERE {clause} + FOR UPDATE + """ + txn.execute(sql, args) + + # Check the deletion status in the DB of the given state groups + clause, args = make_in_list_sql_clause( + self.db_pool.engine, + column="state_group", + iterable=state_groups_to_sequence_numbers, + ) + + sql = f""" + SELECT state_group, insertion_ts, sequence_number FROM ( + SELECT state_group, insertion_ts, sequence_number FROM state_groups_pending_deletion + UNION + SELECT state_group, null, null FROM state_groups_persisting + ) AS s + WHERE {clause} + """ + + txn.execute(sql, args) + + # The above query will return potentially two rows per state group (one + # for each table), so we track which state groups have enough time + # elapsed and which are not ready to be persisted. + ready_to_be_deleted = set() + not_ready_to_be_deleted = set() + + now = self._clock.time_msec() + for state_group, insertion_ts, sequence_number in txn: + if insertion_ts is None: + # A null insertion_ts means that we are currently persisting + # events that reference the state group, so we don't delete + # them. + not_ready_to_be_deleted.add(state_group) + continue + + # We know this can't be None if insertion_ts is not None + assert sequence_number is not None + + # Check if the sequence number has changed, if it has then it + # indicates that the state group may have become referenced since we + # checked. + if state_groups_to_sequence_numbers[state_group] != sequence_number: + not_ready_to_be_deleted.add(state_group) + continue + + if now - insertion_ts < self.DELAY_BEFORE_DELETION_MS: + # Not enough time has elapsed to allow us to delete. + not_ready_to_be_deleted.add(state_group) + continue + + ready_to_be_deleted.add(state_group) + + can_be_deleted = ready_to_be_deleted - not_ready_to_be_deleted + if not_ready_to_be_deleted: + # If there are any state groups that aren't ready to be deleted, + # then we also need to remove any state groups that are referenced + # by them. + clause, args = make_in_list_sql_clause( + self.db_pool.engine, + column="state_group", + iterable=state_groups_to_sequence_numbers, + ) + sql = f""" + WITH RECURSIVE ancestors(state_group) AS ( + SELECT DISTINCT prev_state_group + FROM state_group_edges WHERE {clause} + UNION + SELECT prev_state_group + FROM state_group_edges + INNER JOIN ancestors USING (state_group) + ) + SELECT state_group FROM ancestors + """ + txn.execute(sql, args) + + can_be_deleted.difference_update(state_group for (state_group,) in txn) + + return can_be_deleted + + async def get_next_state_group_collection_to_delete( + self, + ) -> Optional[Tuple[str, Mapping[int, int]]]: + """Get the next set of state groups to try and delete + + Returns: + 2-tuple of room_id and mapping of state groups to sequence number. + """ + return await self.db_pool.runInteraction( + "get_next_state_group_collection_to_delete", + self._get_next_state_group_collection_to_delete_txn, + ) + + def _get_next_state_group_collection_to_delete_txn( + self, + txn: LoggingTransaction, + ) -> Optional[Tuple[str, Mapping[int, int]]]: + """Implementation of `get_next_state_group_collection_to_delete`""" + + # We want to return chunks of state groups that were marked for deletion + # at the same time (this isn't necessary, just more efficient). We do + # this by looking for the oldest insertion_ts, and then pulling out all + # rows that have the same insertion_ts (and room ID). + now = self._clock.time_msec() + + sql = """ + SELECT room_id, insertion_ts + FROM state_groups_pending_deletion AS sd + INNER JOIN state_groups AS sg ON (id = sd.state_group) + LEFT JOIN state_groups_persisting AS sp USING (state_group) + WHERE insertion_ts < ? AND sp.state_group IS NULL + ORDER BY insertion_ts + LIMIT 1 + """ + txn.execute(sql, (now - self.DELAY_BEFORE_DELETION_MS,)) + row = txn.fetchone() + if not row: + return None + + (room_id, insertion_ts) = row + + sql = """ + SELECT state_group, sequence_number + FROM state_groups_pending_deletion AS sd + INNER JOIN state_groups AS sg ON (id = sd.state_group) + LEFT JOIN state_groups_persisting AS sp USING (state_group) + WHERE room_id = ? AND insertion_ts = ? AND sp.state_group IS NULL + ORDER BY insertion_ts + """ + txn.execute(sql, (room_id, insertion_ts)) + + return room_id, dict(txn) diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index f7a59c8992..9b3b7e086f 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -22,10 +22,10 @@ import logging from typing import ( TYPE_CHECKING, - Collection, Dict, Iterable, List, + Mapping, Optional, Set, Tuple, @@ -36,7 +36,10 @@ import attr from synapse.api.constants import EventTypes from synapse.events import EventBase -from synapse.events.snapshot import UnpersistedEventContext, UnpersistedEventContextBase +from synapse.events.snapshot import ( + UnpersistedEventContext, + UnpersistedEventContextBase, +) from synapse.logging.opentracing import tag_args, trace from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( @@ -45,6 +48,7 @@ from synapse.storage.database import ( LoggingTransaction, ) from synapse.storage.databases.state.bg_updates import StateBackgroundUpdateStore +from synapse.storage.engines import PostgresEngine from synapse.storage.types import Cursor from synapse.storage.util.sequence import build_sequence_generator from synapse.types import MutableStateMap, StateKey, StateMap @@ -55,6 +59,7 @@ from synapse.util.cancellation import cancellable if TYPE_CHECKING: from synapse.server import HomeServer + from synapse.storage.databases.state.deletion import StateDeletionDataStore logger = logging.getLogger(__name__) @@ -83,8 +88,11 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): database: DatabasePool, db_conn: LoggingDatabaseConnection, hs: "HomeServer", + state_deletion_store: "StateDeletionDataStore", ): super().__init__(database, db_conn, hs) + self._state_deletion_store = state_deletion_store + self.server_name = hs.hostname # Originally the state store used a single DictionaryCache to cache the # event IDs for the state types in a given state group to avoid hammering @@ -116,14 +124,16 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): # vast majority of state in Matrix (today) is member events. self._state_group_cache: DictionaryCache[int, StateKey, str] = DictionaryCache( - "*stateGroupCache*", + name="*stateGroupCache*", + server_name=self.server_name, # TODO: this hasn't been tuned yet - 50000, + max_entries=50000, ) self._state_group_members_cache: DictionaryCache[int, StateKey, str] = ( DictionaryCache( - "*stateGroupMembersCache*", - 500000, + name="*stateGroupMembersCache*", + server_name=self.server_name, + max_entries=500000, ) ) @@ -467,14 +477,15 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): Returns: A list of state groups """ - is_in_db = self.db_pool.simple_select_one_onecol_txn( - txn, - table="state_groups", - keyvalues={"id": prev_group}, - retcol="id", - allow_none=True, + + # We need to check that the prev group isn't about to be deleted + is_missing = ( + self._state_deletion_store._check_state_groups_and_bump_deletion_txn( + txn, + {prev_group}, + ) ) - if not is_in_db: + if is_missing: raise Exception( "Trying to persist state with unpersisted prev_group: %r" % (prev_group,) @@ -546,6 +557,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): for key, state_id in context.state_delta_due_to_event.items() ], ) + return events_and_context return await self.db_pool.runInteraction( @@ -601,14 +613,15 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): The state group if successfully created, or None if the state needs to be persisted as a full state. """ - is_in_db = self.db_pool.simple_select_one_onecol_txn( - txn, - table="state_groups", - keyvalues={"id": prev_group}, - retcol="id", - allow_none=True, + + # We need to check that the prev group isn't about to be deleted + is_missing = ( + self._state_deletion_store._check_state_groups_and_bump_deletion_txn( + txn, + {prev_group}, + ) ) - if not is_in_db: + if is_missing: raise Exception( "Trying to persist state with unpersisted prev_group: %r" % (prev_group,) @@ -726,8 +739,10 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): ) async def purge_unreferenced_state_groups( - self, room_id: str, state_groups_to_delete: Collection[int] - ) -> None: + self, + room_id: str, + state_groups_to_sequence_numbers: Mapping[int, int], + ) -> bool: """Deletes no longer referenced state groups and de-deltas any state groups that reference them. @@ -735,21 +750,31 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): room_id: The room the state groups belong to (must all be in the same room). state_groups_to_delete: Set of all state groups to delete. + + Returns: + Whether any state groups were actually deleted. """ - await self.db_pool.runInteraction( + return await self.db_pool.runInteraction( "purge_unreferenced_state_groups", self._purge_unreferenced_state_groups, room_id, - state_groups_to_delete, + state_groups_to_sequence_numbers, ) def _purge_unreferenced_state_groups( self, txn: LoggingTransaction, room_id: str, - state_groups_to_delete: Collection[int], - ) -> None: + state_groups_to_sequence_numbers: Mapping[int, int], + ) -> bool: + state_groups_to_delete = self._state_deletion_store.get_state_groups_ready_for_potential_deletion_txn( + txn, state_groups_to_sequence_numbers + ) + + if not state_groups_to_delete: + return False + logger.info( "[purge] found %i state groups to delete", len(state_groups_to_delete) ) @@ -807,10 +832,20 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): "DELETE FROM state_groups_state WHERE state_group = ?", [(sg,) for sg in state_groups_to_delete], ) + txn.execute_batch( + "DELETE FROM state_group_edges WHERE state_group = ?", + [(sg,) for sg in state_groups_to_delete], + ) txn.execute_batch( "DELETE FROM state_groups WHERE id = ?", [(sg,) for sg in state_groups_to_delete], ) + txn.execute_batch( + "DELETE FROM state_groups_pending_deletion WHERE state_group = ?", + [(sg,) for sg in state_groups_to_delete], + ) + + return True @trace @tag_args @@ -830,7 +865,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): List[Tuple[int, int]], await self.db_pool.simple_select_many_batch( table="state_group_edges", - column="prev_state_group", + column="state_group", iterable=state_groups, keyvalues={}, retcols=("state_group", "prev_state_group"), @@ -840,60 +875,77 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): return dict(rows) - async def purge_room_state( - self, room_id: str, state_groups_to_delete: Collection[int] - ) -> None: - """Deletes all record of a room from state tables + @trace + @tag_args + async def get_next_state_groups( + self, state_groups: Iterable[int] + ) -> Dict[int, int]: + """Fetch the groups that have the given state groups as their previous + state groups. Args: - room_id: - state_groups_to_delete: State groups to delete + state_groups + + Returns: + A mapping from state group to previous state group. """ - logger.info("[purge] Starting state purge") - await self.db_pool.runInteraction( + rows = cast( + List[Tuple[int, int]], + await self.db_pool.simple_select_many_batch( + table="state_group_edges", + column="prev_state_group", + iterable=state_groups, + keyvalues={}, + retcols=("state_group", "prev_state_group"), + desc="get_next_state_groups", + ), + ) + + return dict(rows) + + async def purge_room_state(self, room_id: str) -> None: + return await self.db_pool.runInteraction( "purge_room_state", self._purge_room_state_txn, room_id, - state_groups_to_delete, ) - logger.info("[purge] Done with state purge") def _purge_room_state_txn( self, txn: LoggingTransaction, room_id: str, - state_groups_to_delete: Collection[int], ) -> None: - # first we have to delete the state groups states - logger.info("[purge] removing %s from state_groups_state", room_id) - - self.db_pool.simple_delete_many_txn( - txn, - table="state_groups_state", - column="state_group", - values=state_groups_to_delete, - keyvalues={}, - ) - - # ... and the state group edges + # Delete all edges that reference a state group linked to room_id logger.info("[purge] removing %s from state_group_edges", room_id) - self.db_pool.simple_delete_many_txn( - txn, - table="state_group_edges", - column="state_group", - values=state_groups_to_delete, - keyvalues={}, + if isinstance(self.database_engine, PostgresEngine): + # Disable statement timeouts for this transaction; purging rooms can + # take a while! + txn.execute("SET LOCAL statement_timeout = 0") + + txn.execute( + """ + DELETE FROM state_group_edges AS sge WHERE sge.state_group IN ( + SELECT id FROM state_groups AS sg WHERE sg.room_id = ? + )""", + (room_id,), ) - # ... and the state groups - logger.info("[purge] removing %s from state_groups", room_id) + # state_groups_state table has a room_id column but no index on it, unlike state_groups, + # so we delete them by matching the room_id through the state_groups table. + logger.info("[purge] removing %s from state_groups_state", room_id) + txn.execute( + """ + DELETE FROM state_groups_state AS sgs WHERE sgs.state_group IN ( + SELECT id FROM state_groups AS sg WHERE sg.room_id = ? + )""", + (room_id,), + ) - self.db_pool.simple_delete_many_txn( + logger.info("[purge] removing %s from state_groups", room_id) + self.db_pool.simple_delete_txn( txn, table="state_groups", - column="id", - values=state_groups_to_delete, - keyvalues={}, + keyvalues={"room_id": room_id}, ) diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py index 9d82c59384..9fec42c2e0 100644 --- a/synapse/storage/engines/_base.py +++ b/synapse/storage/engines/_base.py @@ -34,9 +34,9 @@ AUTO_INCREMENT_PRIMARY_KEYPLACEHOLDER = "$%AUTO_INCREMENT_PRIMARY_KEY%$" class IsolationLevel(IntEnum): - READ_COMMITTED: int = 1 - REPEATABLE_READ: int = 2 - SERIALIZABLE: int = 3 + READ_COMMITTED = 1 + REPEATABLE_READ = 2 + SERIALIZABLE = 3 class IncorrectDatabaseSetup(RuntimeError): diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 8c8c6d0414..e4cd359201 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -99,8 +99,8 @@ class PostgresEngine( allow_unsafe_locale = self.config.get("allow_unsafe_locale", False) # Are we on a supported PostgreSQL version? - if not allow_outdated_version and self._version < 110000: - raise RuntimeError("Synapse requires PostgreSQL 11 or above.") + if not allow_outdated_version and self._version < 130000: + raise RuntimeError("Synapse requires PostgreSQL 13 or above.") with db_conn.cursor() as txn: txn.execute("SHOW SERVER_ENCODING") diff --git a/synapse/storage/invite_rule.py b/synapse/storage/invite_rule.py new file mode 100644 index 0000000000..f63390871e --- /dev/null +++ b/synapse/storage/invite_rule.py @@ -0,0 +1,112 @@ +import logging +from enum import Enum +from typing import Optional, Pattern + +from matrix_common.regex import glob_to_regex + +from synapse.types import JsonMapping, UserID + +logger = logging.getLogger(__name__) + + +class InviteRule(Enum): + """Enum to define the action taken when an invite matches a rule.""" + + ALLOW = "allow" + BLOCK = "block" + IGNORE = "ignore" + + +class InviteRulesConfig: + """Class to determine if a given user permits an invite from another user, and the action to take.""" + + def __init__(self, account_data: Optional[JsonMapping]): + self.allowed_users: list[Pattern[str]] = [] + self.ignored_users: list[Pattern[str]] = [] + self.blocked_users: list[Pattern[str]] = [] + + self.allowed_servers: list[Pattern[str]] = [] + self.ignored_servers: list[Pattern[str]] = [] + self.blocked_servers: list[Pattern[str]] = [] + + def process_field( + values: Optional[list[str]], + ruleset: list[Pattern[str]], + rule: InviteRule, + ) -> None: + if isinstance(values, list): + for value in values: + if isinstance(value, str) and len(value) > 0: + # User IDs cannot exceed 255 bytes. Don't process large, potentially + # expensive glob patterns. + if len(value) > 255: + logger.debug( + "Ignoring invite config glob pattern that is >255 bytes: {value}" + ) + continue + + try: + ruleset.append(glob_to_regex(value)) + except Exception as e: + # If for whatever reason we can't process this, just ignore it. + logger.debug( + "Could not process '%s' field of invite rule config, ignoring: %s", + value, + e, + ) + + if account_data: + process_field( + account_data.get("allowed_users"), self.allowed_users, InviteRule.ALLOW + ) + process_field( + account_data.get("ignored_users"), self.ignored_users, InviteRule.IGNORE + ) + process_field( + account_data.get("blocked_users"), self.blocked_users, InviteRule.BLOCK + ) + process_field( + account_data.get("allowed_servers"), + self.allowed_servers, + InviteRule.ALLOW, + ) + process_field( + account_data.get("ignored_servers"), + self.ignored_servers, + InviteRule.IGNORE, + ) + process_field( + account_data.get("blocked_servers"), + self.blocked_servers, + InviteRule.BLOCK, + ) + + def get_invite_rule(self, user_id: str) -> InviteRule: + """Get the invite rule that matches this user. Will return InviteRule.ALLOW if no rules match + + Args: + user_id: The user ID of the inviting user. + + """ + user = UserID.from_string(user_id) + # The order here is important. We always process user rules before server rules + # and we always process in the order of Allow, Ignore, Block. + for patterns, rule in [ + (self.allowed_users, InviteRule.ALLOW), + (self.ignored_users, InviteRule.IGNORE), + (self.blocked_users, InviteRule.BLOCK), + ]: + for regex in patterns: + if regex.match(user_id): + return rule + + for patterns, rule in [ + (self.allowed_servers, InviteRule.ALLOW), + (self.ignored_servers, InviteRule.IGNORE), + (self.blocked_servers, InviteRule.BLOCK), + ]: + for regex in patterns: + if regex.match(user.domain): + return rule + + return InviteRule.ALLOW diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 934e1ccced..3c3b13437e 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 88 # remember to update the list below when updating +SCHEMA_VERSION = 92 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -155,6 +155,19 @@ Changes in SCHEMA_VERSION = 88 be posted in response to a resettable timeout or an on-demand action. - Add background update to fix data integrity issue in the `sliding_sync_membership_snapshots` -> `forgotten` column + +Changes in SCHEMA_VERSION = 89 + - Add `state_groups_pending_deletion` and `state_groups_persisting` tables. + +Changes in SCHEMA_VERSION = 90 + - Add a column `participant` to `room_memberships` table + - Add background update to delete unreferenced state groups. + +Changes in SCHEMA_VERSION = 91 + - Add a `sha256` column to the `local_media_repository` and `remote_media_cache` tables. + +Changes in SCHEMA_VERSION = 92 + - Cleaned up a trigger that was added in #18260 and then reverted. """ diff --git a/synapse/storage/schema/main/delta/25/fts.py b/synapse/storage/schema/main/delta/25/fts.py index b050cc16a7..c01c1325cb 100644 --- a/synapse/storage/schema/main/delta/25/fts.py +++ b/synapse/storage/schema/main/delta/25/fts.py @@ -75,8 +75,7 @@ def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> progress_json = json.dumps(progress) sql = ( - "INSERT into background_updates (update_name, progress_json)" - " VALUES (?, ?)" + "INSERT into background_updates (update_name, progress_json) VALUES (?, ?)" ) cur.execute(sql, ("event_search", progress_json)) diff --git a/synapse/storage/schema/main/delta/27/ts.py b/synapse/storage/schema/main/delta/27/ts.py index d7f360b6e6..e6e73e1b77 100644 --- a/synapse/storage/schema/main/delta/27/ts.py +++ b/synapse/storage/schema/main/delta/27/ts.py @@ -55,8 +55,7 @@ def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> progress_json = json.dumps(progress) sql = ( - "INSERT into background_updates (update_name, progress_json)" - " VALUES (?, ?)" + "INSERT into background_updates (update_name, progress_json) VALUES (?, ?)" ) cur.execute(sql, ("event_origin_server_ts", progress_json)) diff --git a/synapse/storage/schema/main/delta/30/as_users.py b/synapse/storage/schema/main/delta/30/as_users.py index 2a3023cd07..060217575b 100644 --- a/synapse/storage/schema/main/delta/30/as_users.py +++ b/synapse/storage/schema/main/delta/30/as_users.py @@ -63,8 +63,11 @@ def run_upgrade( if user_id in owned.keys(): logger.error( "user_id %s was owned by more than one application" - " service (IDs %s and %s); assigning arbitrarily to %s" - % (user_id, owned[user_id], appservice.id, owned[user_id]) + " service (IDs %s and %s); assigning arbitrarily to %s", + user_id, + owned[user_id], + appservice.id, + owned[user_id], ) owned.setdefault(appservice.id, []).append(user_id) diff --git a/synapse/storage/schema/main/delta/31/search_update.py b/synapse/storage/schema/main/delta/31/search_update.py index 0e65c9a841..46355122bb 100644 --- a/synapse/storage/schema/main/delta/31/search_update.py +++ b/synapse/storage/schema/main/delta/31/search_update.py @@ -59,8 +59,7 @@ def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> progress_json = json.dumps(progress) sql = ( - "INSERT into background_updates (update_name, progress_json)" - " VALUES (?, ?)" + "INSERT into background_updates (update_name, progress_json) VALUES (?, ?)" ) cur.execute(sql, ("event_search_order", progress_json)) diff --git a/synapse/storage/schema/main/delta/33/event_fields.py b/synapse/storage/schema/main/delta/33/event_fields.py index 9c02aeda88..53d215337e 100644 --- a/synapse/storage/schema/main/delta/33/event_fields.py +++ b/synapse/storage/schema/main/delta/33/event_fields.py @@ -55,8 +55,7 @@ def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> progress_json = json.dumps(progress) sql = ( - "INSERT into background_updates (update_name, progress_json)" - " VALUES (?, ?)" + "INSERT into background_updates (update_name, progress_json) VALUES (?, ?)" ) cur.execute(sql, ("event_fields_sender_url", progress_json)) diff --git a/synapse/storage/schema/main/delta/88/01_custom_profile_fields.sql b/synapse/storage/schema/main/delta/88/01_custom_profile_fields.sql new file mode 100644 index 0000000000..63cbd7ffa9 --- /dev/null +++ b/synapse/storage/schema/main/delta/88/01_custom_profile_fields.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2024 Patrick Cloke +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Custom profile fields. +ALTER TABLE profiles ADD COLUMN fields JSONB; diff --git a/synapse/storage/schema/main/delta/88/06_events_received_ts_index.sql b/synapse/storage/schema/main/delta/88/06_events_received_ts_index.sql new file mode 100644 index 0000000000..d70a4a8dbc --- /dev/null +++ b/synapse/storage/schema/main/delta/88/06_events_received_ts_index.sql @@ -0,0 +1,17 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2024 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add an index on `events.received_ts` for `m.room.member` events to allow for +-- efficient lookup of events by timestamp in some Admin API's +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (8806, 'events_received_ts_index', '{}'); diff --git a/synapse/storage/schema/main/delta/89/01_sliding_sync_membership_snapshot_index.sql b/synapse/storage/schema/main/delta/89/01_sliding_sync_membership_snapshot_index.sql new file mode 100644 index 0000000000..7799cffdce --- /dev/null +++ b/synapse/storage/schema/main/delta/89/01_sliding_sync_membership_snapshot_index.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (8901, 'sliding_sync_membership_snapshots_membership_event_id_idx', '{}'); diff --git a/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql b/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql new file mode 100644 index 0000000000..dafd046499 --- /dev/null +++ b/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add a column `participant` to `room_memberships` table to track whether a room member has sent +-- a `m.room.message` or `m.room.encrypted` event into a room they are a member of +ALTER TABLE room_memberships ADD COLUMN participant BOOLEAN DEFAULT FALSE; \ No newline at end of file diff --git a/synapse/storage/schema/main/delta/91/01_media_hash.sql b/synapse/storage/schema/main/delta/91/01_media_hash.sql new file mode 100644 index 0000000000..34a372f1ed --- /dev/null +++ b/synapse/storage/schema/main/delta/91/01_media_hash.sql @@ -0,0 +1,28 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Store the SHA256 content hash of media files. +ALTER TABLE local_media_repository ADD COLUMN sha256 TEXT; +ALTER TABLE remote_media_cache ADD COLUMN sha256 TEXT; + +-- Add a background updates to handle creating the new index. +-- +-- Note that the ordering of the update is not following the usual scheme. This +-- is because when upgrading from Synapse 1.127, this index is fairly important +-- to have up quickly, so that it doesn't tank performance, which is why it is +-- scheduled before other background updates in the 1.127 -> 1.128 upgrade +INSERT INTO + background_updates (ordering, update_name, progress_json) +VALUES + (8890, 'local_media_repository_sha256_idx', '{}'), + (8891, 'remote_media_cache_sha256_idx', '{}'); diff --git a/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.postgres b/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.postgres new file mode 100644 index 0000000000..e9f160cdcc --- /dev/null +++ b/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.postgres @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Removes the trigger that was added in #18260 and then reverted +DROP TRIGGER IF EXISTS event_stats_increment_counts_trigger ON events; +DROP FUNCTION IF EXISTS event_stats_increment_counts(); diff --git a/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.sqlite b/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.sqlite new file mode 100644 index 0000000000..b5f084dde8 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.sqlite @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Removes the trigger that was added in #18260 and then reverted +DROP TRIGGER IF EXISTS event_stats_events_insert_trigger; +DROP TRIGGER IF EXISTS event_stats_events_delete_trigger; diff --git a/synapse/storage/schema/main/delta/92/02_remove_populate_participant_bg_update.sql b/synapse/storage/schema/main/delta/92/02_remove_populate_participant_bg_update.sql new file mode 100644 index 0000000000..e1f377c37d --- /dev/null +++ b/synapse/storage/schema/main/delta/92/02_remove_populate_participant_bg_update.sql @@ -0,0 +1,17 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Remove the background update if it was scheduled, as it is not rollback-safe +-- See https://github.com/element-hq/synapse/issues/18356 for context +DELETE FROM background_updates +WHERE update_name = 'populate_participant_bg_update'; \ No newline at end of file diff --git a/synapse/storage/schema/main/delta/92/04_ss_membership_snapshot_idx.sql b/synapse/storage/schema/main/delta/92/04_ss_membership_snapshot_idx.sql new file mode 100644 index 0000000000..6f5b7cb06e --- /dev/null +++ b/synapse/storage/schema/main/delta/92/04_ss_membership_snapshot_idx.sql @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- So we can fetch all rooms for a given user sorted by stream order +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9204, 'sliding_sync_membership_snapshots_user_id_stream_ordering', '{}'); diff --git a/synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql b/synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql new file mode 100644 index 0000000000..d19dd7a46d --- /dev/null +++ b/synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql @@ -0,0 +1,59 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Introduce a table for tracking users' subscriptions to threads. +CREATE TABLE thread_subscriptions ( + stream_id INTEGER NOT NULL PRIMARY KEY, + instance_name TEXT NOT NULL, + + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + + subscribed BOOLEAN NOT NULL, + automatic BOOLEAN NOT NULL, + + CONSTRAINT thread_subscriptions_fk_users + FOREIGN KEY (user_id) + REFERENCES users(name), + + CONSTRAINT thread_subscriptions_fk_rooms + FOREIGN KEY (room_id) + -- When we delete a room, we should already have deleted all the events in that room + -- and so there shouldn't be any subscriptions left in that room. + -- So the `ON DELETE CASCADE` should be optional, but included anyway for good measure. + REFERENCES rooms(room_id) ON DELETE CASCADE, + + CONSTRAINT thread_subscriptions_fk_events + FOREIGN KEY (event_id) + REFERENCES events(event_id) ON DELETE CASCADE, + + -- This order provides a useful index for: + -- 1. foreign key constraint on (room_id) + -- 2. foreign key constraint on (room_id, event_id) + -- 3. finding the user's settings for a specific thread (as well as enforcing uniqueness) + UNIQUE (room_id, event_id, user_id) +); + +-- this provides a useful index for finding a user's own rules, +-- potentially scoped to a single room +CREATE INDEX thread_subscriptions_user_room ON thread_subscriptions (user_id, room_id); + +-- this provides a useful way for clients to efficiently find new changes to +-- their subscriptions. +-- (This is necessary to sync subscriptions between multiple devices.) +CREATE INDEX thread_subscriptions_by_user ON thread_subscriptions (user_id, stream_id); + +-- this provides a useful index for deleting the subscriptions when the underlying +-- events are removed. This also covers the foreign key constraint on `events`. +CREATE INDEX thread_subscriptions_by_event ON thread_subscriptions (event_id); diff --git a/synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres b/synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres new file mode 100644 index 0000000000..8d53691747 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres @@ -0,0 +1,19 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE SEQUENCE thread_subscriptions_sequence + -- Synapse streams start at 2, because the default position is 1 + -- so any item inserted at position 1 is ignored. + -- This is also what existing streams do, except they use `setval(..., 1)` + -- which is semantically the same except less obvious. + START WITH 2; diff --git a/synapse/storage/schema/main/delta/92/05_fixup_max_depth_cap.sql b/synapse/storage/schema/main/delta/92/05_fixup_max_depth_cap.sql new file mode 100644 index 0000000000..c1ebf8b58b --- /dev/null +++ b/synapse/storage/schema/main/delta/92/05_fixup_max_depth_cap.sql @@ -0,0 +1,17 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Background update that fixes any events with a topological ordering above the +-- MAX_DEPTH value. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9205, 'fixup_max_depth_cap', '{}'); diff --git a/synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres b/synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres new file mode 100644 index 0000000000..b0729894c0 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +COMMENT ON TABLE thread_subscriptions IS 'Tracks local users that subscribe to threads'; + +COMMENT ON COLUMN thread_subscriptions.subscribed IS 'Whether the user is subscribed to the thread or not. We track unsubscribed threads because we need to stream the subscription change to the client.'; + +COMMENT ON COLUMN thread_subscriptions.automatic IS 'True if the user was subscribed to the thread automatically by their client, or false if the client manually requested the subscription.'; diff --git a/synapse/storage/schema/main/delta/92/06_device_federation_inbox_index.sql b/synapse/storage/schema/main/delta/92/06_device_federation_inbox_index.sql new file mode 100644 index 0000000000..0ba5b56148 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/06_device_federation_inbox_index.sql @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Background update that adds an index to `device_federation_inbox.received_ts` +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9206, 'device_federation_inbox_received_ts_index', '{}'); diff --git a/synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres b/synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres new file mode 100644 index 0000000000..3fc7e4b11e --- /dev/null +++ b/synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres @@ -0,0 +1,24 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +COMMENT ON COLUMN threads.latest_event_id IS + 'the ID of the event that is latest, ordered by (topological_ordering, stream_ordering)'; + +COMMENT ON COLUMN threads.topological_ordering IS + $$the topological ordering of the thread''s LATEST event. +Used as the primary way of ordering threads by recency in a room.$$; + +COMMENT ON COLUMN threads.stream_ordering IS + $$the stream ordering of the thread's LATEST event. +Used as a tie-breaker for ordering threads by recency in a room, when the topological order is a tie. +Also used for recency ordering in sliding sync.$$; diff --git a/synapse/storage/schema/main/delta/92/07_add_user_reports.sql b/synapse/storage/schema/main/delta/92/07_add_user_reports.sql new file mode 100644 index 0000000000..7439dad6d6 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/07_add_user_reports.sql @@ -0,0 +1,22 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE TABLE user_reports ( + id BIGINT NOT NULL PRIMARY KEY, + received_ts BIGINT NOT NULL, + target_user_id TEXT NOT NULL, + user_id TEXT NOT NULL, + reason TEXT NOT NULL +); +CREATE INDEX user_reports_target_user_id ON user_reports(target_user_id); -- for lookups +CREATE INDEX user_reports_user_id ON user_reports(user_id); -- for lookups diff --git a/synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql b/synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql new file mode 100644 index 0000000000..b568fc04f2 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9207, 'event_txn_id_device_id_txn_id2', '{}'); diff --git a/synapse/storage/schema/main/delta/92/08_room_ban_redactions.sql b/synapse/storage/schema/main/delta/92/08_room_ban_redactions.sql new file mode 100644 index 0000000000..566ddcbdd7 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/08_room_ban_redactions.sql @@ -0,0 +1,21 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE TABLE room_ban_redactions( + room_id text NOT NULL, + user_id text NOT NULL, + redacting_event_id text NOT NULL, + redact_end_ordering bigint DEFAULT NULL, -- stream ordering after which redactions are not applied + CONSTRAINT room_ban_redaction_uniqueness UNIQUE (room_id, user_id) +); + diff --git a/synapse/storage/schema/main/delta/92/08_thread_subscriptions_seq_fixup.sql.postgres b/synapse/storage/schema/main/delta/92/08_thread_subscriptions_seq_fixup.sql.postgres new file mode 100644 index 0000000000..d327d1e165 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/08_thread_subscriptions_seq_fixup.sql.postgres @@ -0,0 +1,19 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Work around https://github.com/element-hq/synapse/issues/18712 by advancing the +-- stream sequence. +-- This makes last_value of the sequence point to a position that will not get later +-- returned by nextval. +-- (For blank thread subscription streams, this means last_value = 2, nextval() = 3 after this line.) +SELECT nextval('thread_subscriptions_sequence'); diff --git a/synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql b/synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql new file mode 100644 index 0000000000..03b8a1a635 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql @@ -0,0 +1,20 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- The maximum stream_ordering in the room when the unsubscription was made. +ALTER TABLE thread_subscriptions + ADD COLUMN unsubscribed_at_stream_ordering BIGINT; + +-- The maximum topological_ordering in the room when the unsubscription was made. +ALTER TABLE thread_subscriptions + ADD COLUMN unsubscribed_at_topological_ordering BIGINT; diff --git a/synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql.postgres b/synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql.postgres new file mode 100644 index 0000000000..fc5d555db5 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +COMMENT ON COLUMN thread_subscriptions.unsubscribed_at_stream_ordering IS + $$The maximum stream_ordering in the room when the unsubscription was made.$$; + +COMMENT ON COLUMN thread_subscriptions.unsubscribed_at_topological_ordering IS + $$The maximum topological_ordering in the room when the unsubscription was made.$$; diff --git a/synapse/storage/schema/state/delta/89/01_state_groups_deletion.sql b/synapse/storage/schema/state/delta/89/01_state_groups_deletion.sql new file mode 100644 index 0000000000..d4cb27a3a2 --- /dev/null +++ b/synapse/storage/schema/state/delta/89/01_state_groups_deletion.sql @@ -0,0 +1,39 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- See the `StateDeletionDataStore` for details of these tables. + +-- We add state groups to this table when we want to later delete them. The +-- `insertion_ts` column indicates when the state group was proposed for +-- deletion (rather than when it should be deleted). +CREATE TABLE IF NOT EXISTS state_groups_pending_deletion ( + sequence_number $%AUTO_INCREMENT_PRIMARY_KEY%$, + state_group BIGINT NOT NULL, + insertion_ts BIGINT NOT NULL +); + +CREATE UNIQUE INDEX state_groups_pending_deletion_state_group ON state_groups_pending_deletion(state_group); +CREATE INDEX state_groups_pending_deletion_insertion_ts ON state_groups_pending_deletion(insertion_ts); + + +-- Holds the state groups the worker is currently persisting. +-- +-- The `sequence_number` column of the `state_groups_pending_deletion` table +-- *must* be updated whenever a state group may have become referenced. +CREATE TABLE IF NOT EXISTS state_groups_persisting ( + state_group BIGINT NOT NULL, + instance_name TEXT NOT NULL, + PRIMARY KEY (state_group, instance_name) +); + +CREATE INDEX state_groups_persisting_instance_name ON state_groups_persisting(instance_name); diff --git a/synapse/storage/schema/state/delta/90/02_delete_unreferenced_state_groups.sql b/synapse/storage/schema/state/delta/90/02_delete_unreferenced_state_groups.sql new file mode 100644 index 0000000000..55a038e2b8 --- /dev/null +++ b/synapse/storage/schema/state/delta/90/02_delete_unreferenced_state_groups.sql @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add a background update to delete any unreferenced state groups +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9002, 'mark_unreferenced_state_groups_for_deletion_bg_update', '{}'); diff --git a/synapse/storage/schema/state/delta/90/03_remove_old_deletion_bg_update.sql b/synapse/storage/schema/state/delta/90/03_remove_old_deletion_bg_update.sql new file mode 100644 index 0000000000..1cc6d612b6 --- /dev/null +++ b/synapse/storage/schema/state/delta/90/03_remove_old_deletion_bg_update.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Remove the old unreferenced state group deletion background update if it exists +DELETE FROM background_updates WHERE update_name = 'delete_unreferenced_state_groups_bg_update'; diff --git a/synapse/storage/types.py b/synapse/storage/types.py index 74f60cc590..4329d88c9a 100644 --- a/synapse/storage/types.py +++ b/synapse/storage/types.py @@ -26,14 +26,13 @@ from typing import ( List, Mapping, Optional, + Protocol, Sequence, Tuple, Type, Union, ) -from typing_extensions import Protocol - """ Some very basic protocol definitions for the DB-API2 classes specified in PEP-249 """ diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index e8588f33cf..1b7c5dac7a 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -184,11 +184,23 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): Note: Only works with Postgres. + Warning: Streams using this generator start at ID 2, because ID 1 is always assumed + to have been 'seen as persisted'. + Unclear if this extant behaviour is desirable for some reason. + When creating a new sequence for a new stream, it will be necessary to advance it + so that position 1 is consumed. + DO NOT USE `START WITH 2` FOR THIS PURPOSE: + see https://github.com/element-hq/synapse/issues/18712 + Instead, use `SELECT nextval('sequence_name');` immediately after the + `CREATE SEQUENCE` statement. + Args: db_conn db stream_name: A name for the stream, for use in the `stream_positions` table. (Does not need to be the same as the replication stream name) + server_name: The homeserver name of the server (used to label metrics) + (this should be `hs.hostname`). instance_name: The name of this instance. tables: List of tables associated with the stream. Tuple of table name, column name that stores the writer's instance name, and @@ -204,10 +216,12 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): def __init__( self, + *, db_conn: LoggingDatabaseConnection, db: DatabasePool, notifier: "ReplicationNotifier", stream_name: str, + server_name: str, instance_name: str, tables: List[Tuple[str, str, str]], sequence_name: str, @@ -217,6 +231,7 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): self._db = db self._notifier = notifier self._stream_name = stream_name + self.server_name = server_name self._instance_name = instance_name self._positive = positive self._writers = writers @@ -269,6 +284,9 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): self._known_persisted_positions: List[int] = [] # The maximum stream ID that we have seen been allocated across any writer. + # Since this defaults to 1, this means that ID 1 is assumed to have already + # been 'seen'. In other words, multi-writer streams start at 2. + # Unclear if this is desirable behaviour. self._max_seen_allocated_stream_id = 1 # The maximum position of the local instance. This can be higher than @@ -552,6 +570,7 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): txn.call_after( run_as_background_process, "MultiWriterIdGenerator._update_table", + self.server_name, self._db.runInteraction, "MultiWriterIdGenerator._update_table", self._update_stream_positions_table_txn, @@ -588,6 +607,7 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): txn.call_after( run_as_background_process, "MultiWriterIdGenerator._update_table", + self.server_name, self._db.runInteraction, "MultiWriterIdGenerator._update_table", self._update_stream_positions_table_txn, @@ -794,20 +814,16 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): # We upsert the value, ensuring on conflict that we always increase the # value (or decrease if stream goes backwards). - if isinstance(self._db.engine, PostgresEngine): - agg = "GREATEST" if self._positive else "LEAST" - else: - agg = "MAX" if self._positive else "MIN" + cmp = "<" if self._positive else ">" - sql = """ + sql = f""" INSERT INTO stream_positions (stream_name, instance_name, stream_id) VALUES (?, ?, ?) ON CONFLICT (stream_name, instance_name) DO UPDATE SET - stream_id = %(agg)s(stream_positions.stream_id, EXCLUDED.stream_id) - """ % { - "agg": agg, - } + stream_id = EXCLUDED.stream_id + WHERE stream_positions.stream_id {cmp} EXCLUDED.stream_id + """ pos = self.get_current_token_for_writer(self._instance_name) txn.execute(sql, (self._stream_name, self._instance_name, pos)) diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 856f646795..1e4bebe46d 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -33,7 +33,6 @@ from synapse.logging.opentracing import trace from synapse.streams import EventSource from synapse.types import ( AbstractMultiWriterStreamToken, - MultiWriterStreamToken, StreamKeyType, StreamToken, ) @@ -84,6 +83,7 @@ class EventSources: un_partial_stated_rooms_key = self.store.get_un_partial_stated_rooms_token( self._instance_name ) + thread_subscriptions_key = self.store.get_max_thread_subscriptions_stream_id() token = StreamToken( room_key=self.sources.room.get_current_key(), @@ -97,6 +97,7 @@ class EventSources: # Groups key is unused. groups_key=0, un_partial_stated_rooms_key=un_partial_stated_rooms_key, + thread_subscriptions_key=thread_subscriptions_key, ) return token @@ -123,6 +124,7 @@ class EventSources: StreamKeyType.TO_DEVICE: self.store.get_to_device_id_generator(), StreamKeyType.DEVICE_LIST: self.store.get_device_stream_id_generator(), StreamKeyType.UN_PARTIAL_STATED_ROOMS: self.store.get_un_partial_stated_rooms_id_generator(), + StreamKeyType.THREAD_SUBSCRIPTIONS: self.store.get_thread_subscriptions_stream_id_generator(), } for _, key in StreamKeyType.__members__.items(): @@ -195,16 +197,7 @@ class EventSources: Returns: The current token for pagination. """ - token = StreamToken( - room_key=await self.sources.room.get_current_key_for_room(room_id), - presence_key=0, - typing_key=0, - receipt_key=MultiWriterStreamToken(stream=0), - account_data_key=0, - push_rules_key=0, - to_device_key=0, - device_list_key=0, - groups_key=0, - un_partial_stated_rooms_key=0, + return StreamToken.START.copy_and_replace( + StreamKeyType.ROOM, + await self.sources.room.get_current_key_for_room(room_id), ) - return token diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 7d3422572d..a82211283b 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -33,6 +33,9 @@ class EventInternalMetadata: proactively_send: bool redacted: bool + policy_server_spammy: bool + """whether the policy server indicated that this event is spammy""" + txn_id: str """The transaction ID, if it was set when the event was created.""" token_id: int diff --git a/synapse/synapse_rust/http_client.pyi b/synapse/synapse_rust/http_client.pyi new file mode 100644 index 0000000000..9fb7831e6b --- /dev/null +++ b/synapse/synapse_rust/http_client.pyi @@ -0,0 +1,28 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import Mapping + +from twisted.internet.defer import Deferred + +from synapse.types import ISynapseReactor + +class HttpClient: + def __init__(self, reactor: ISynapseReactor, user_agent: str) -> None: ... + def get(self, url: str, response_limit: int) -> Deferred[bytes]: ... + def post( + self, + url: str, + response_limit: int, + headers: Mapping[str, str], + request_body: str, + ) -> Deferred[bytes]: ... diff --git a/synapse/synapse_rust/push.pyi b/synapse/synapse_rust/push.pyi index 3f317c3288..a3e12ad648 100644 --- a/synapse/synapse_rust/push.pyi +++ b/synapse/synapse_rust/push.pyi @@ -49,6 +49,7 @@ class FilteredPushRules: msc3664_enabled: bool, msc4028_push_encrypted_events: bool, msc4210_enabled: bool, + msc4306_enabled: bool, ): ... def rules(self) -> Collection[Tuple[PushRule, bool]]: ... @@ -67,13 +68,19 @@ class PushRuleEvaluator: room_version_feature_flags: Tuple[str, ...], msc3931_enabled: bool, msc4210_enabled: bool, + msc4306_enabled: bool, ): ... def run( self, push_rules: FilteredPushRules, user_id: Optional[str], display_name: Optional[str], + msc4306_thread_subscription_state: Optional[bool], ) -> Collection[Union[Mapping, str]]: ... def matches( - self, condition: JsonDict, user_id: Optional[str], display_name: Optional[str] + self, + condition: JsonDict, + user_id: Optional[str], + display_name: Optional[str], + msc4306_thread_subscription_state: Optional[bool] = None, ) -> bool: ... diff --git a/synapse/synapse_rust/segmenter.pyi b/synapse/synapse_rust/segmenter.pyi new file mode 100644 index 0000000000..5f36765947 --- /dev/null +++ b/synapse/synapse_rust/segmenter.pyi @@ -0,0 +1,3 @@ +from typing import List + +def parse_words(text: str) -> List[str]: ... diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 26783c5622..2d5b07ab8f 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -40,6 +40,7 @@ from typing import ( Set, Tuple, Type, + TypedDict, TypeVar, Union, overload, @@ -49,7 +50,7 @@ import attr from immutabledict import immutabledict from signedjson.key import decode_verify_key_bytes from signedjson.types import VerifyKey -from typing_extensions import Self, TypedDict +from typing_extensions import Self from unpaddedbase64 import decode_base64 from zope.interface import Interface @@ -72,8 +73,10 @@ if TYPE_CHECKING: from typing_extensions import Self from synapse.appservice.api import ApplicationService + from synapse.events import EventBase from synapse.storage.databases.main import DataStore, PurgeEventsStore from synapse.storage.databases.main.appservice import ApplicationServiceWorkerStore + from synapse.storage.util.id_generators import MultiWriterIdGenerator logger = logging.getLogger(__name__) @@ -352,15 +355,82 @@ class RoomAlias(DomainSpecificString): @attr.s(slots=True, frozen=True, repr=False) -class RoomID(DomainSpecificString): - """Structure representing a room id.""" +class RoomIdWithDomain(DomainSpecificString): + """Structure representing a room ID with a domain suffix.""" SIGIL = "!" +# the set of urlsafe base64 characters, no padding. +ROOM_ID_PATTERN_DOMAINLESS = re.compile(r"^[A-Za-z0-9\-_]{43}$") + + +@attr.s(slots=True, frozen=True, auto_attribs=True, repr=False) +class RoomID: + """Structure representing a room id without a domain. + There are two forms of room IDs: + - "!localpart:domain" used in most room versions prior to MSC4291. + - "!event_id_base_64" used in room versions post MSC4291. + This class will accept any room ID which meets either of these two criteria. + """ + + SIGIL = "!" + id: str + room_id_with_domain: Optional[RoomIdWithDomain] + + @classmethod + def is_valid(cls: Type["RoomID"], s: str) -> bool: + if ":" in s: + return RoomIdWithDomain.is_valid(s) + try: + cls.from_string(s) + return True + except Exception: + return False + + def get_domain(self) -> Optional[str]: + if not self.room_id_with_domain: + return None + return self.room_id_with_domain.domain + + def to_string(self) -> str: + if self.room_id_with_domain: + return self.room_id_with_domain.to_string() + return self.id + + __repr__ = to_string + + @classmethod + def from_string(cls: Type["RoomID"], s: str) -> "RoomID": + # sigil check + if len(s) < 1 or s[0] != cls.SIGIL: + raise SynapseError( + 400, + "Expected %s string to start with '%s'" % (cls.__name__, cls.SIGIL), + Codes.INVALID_PARAM, + ) + + room_id_with_domain: Optional[RoomIdWithDomain] = None + if ":" in s: + room_id_with_domain = RoomIdWithDomain.from_string(s) + else: + # MSC4291 room IDs must be valid urlsafe unpadded base64 + val = s[1:] + if not ROOM_ID_PATTERN_DOMAINLESS.match(val): + raise SynapseError( + 400, + "Expected %s string to be valid urlsafe unpadded base64 '%s'" + % (cls.__name__, val), + Codes.INVALID_PARAM, + ) + + return cls(id=s, room_id_with_domain=room_id_with_domain) + + @attr.s(slots=True, frozen=True, repr=False) class EventID(DomainSpecificString): - """Structure representing an event id.""" + """Structure representing an event ID which is namespaced to a homeserver. + Room versions 3 and above are not supported by this grammar.""" SIGIL = "$" @@ -569,6 +639,25 @@ class AbstractMultiWriterStreamToken(metaclass=abc.ABCMeta): ), ) + @classmethod + def from_generator(cls, generator: "MultiWriterIdGenerator") -> Self: + """Get the current token out of a MultiWriterIdGenerator""" + + # The `min_pos` is the minimum position that we know all instances + # have finished persisting to, so we only care about instances whose + # positions are ahead of that. (Instance positions can be behind the + # min position as there are times we can work out that the minimum + # position is ahead of the naive minimum across all current + # positions. See MultiWriterIdGenerator for details) + min_pos = generator.get_current_token() + positions = { + instance: position + for instance, position in generator.get_positions().items() + if position > min_pos + } + + return cls(stream=min_pos, instance_map=immutabledict(positions)) + @attr.s(frozen=True, slots=True, order=False) class RoomStreamToken(AbstractMultiWriterStreamToken): @@ -664,6 +753,11 @@ class RoomStreamToken(AbstractMultiWriterStreamToken): @classmethod async def parse(cls, store: "PurgeEventsStore", string: str) -> "RoomStreamToken": + # Check that it looks like a Synapse token first. We do this so that + # we don't log at the exception-level for obviously incorrect tokens. + if not string or string[0] not in ("s", "t", "m"): + raise SynapseError(400, f"Invalid room stream token {string:!r}") + try: if string[0] == "s": return cls(topological=None, stream=int(string[1:])) @@ -883,8 +977,7 @@ class MultiWriterStreamToken(AbstractMultiWriterStreamToken): def __str__(self) -> str: instances = ", ".join(f"{k}: {v}" for k, v in sorted(self.instance_map.items())) return ( - f"MultiWriterStreamToken(stream: {self.stream}, " - f"instances: {{{instances}}})" + f"MultiWriterStreamToken(stream: {self.stream}, instances: {{{instances}}})" ) @@ -903,6 +996,7 @@ class StreamKeyType(Enum): TO_DEVICE = "to_device_key" DEVICE_LIST = "device_list_key" UN_PARTIAL_STATED_ROOMS = "un_partial_stated_rooms_key" + THREAD_SUBSCRIPTIONS = "thread_subscriptions_key" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -910,7 +1004,7 @@ class StreamToken: """A collection of keys joined together by underscores in the following order and which represent the position in their respective streams. - ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379` + ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242` 1. `room_key`: `s2633508` which is a `RoomStreamToken` - `RoomStreamToken`'s can also look like `t426-2633508` or `m56~2.58~3.59` - See the docstring for `RoomStreamToken` for more details. @@ -923,6 +1017,7 @@ class StreamToken: 8. `device_list_key`: `265584` 9. `groups_key`: `1` (note that this key is now unused) 10. `un_partial_stated_rooms_key`: `379` + 11. `thread_subscriptions_key`: 4242 You can see how many of these keys correspond to the various fields in a "/sync" response: @@ -975,10 +1070,13 @@ class StreamToken: account_data_key: int push_rules_key: int to_device_key: int - device_list_key: int + device_list_key: MultiWriterStreamToken = attr.ib( + validator=attr.validators.instance_of(MultiWriterStreamToken) + ) # Note that the groups key is no longer used and may have bogus values. groups_key: int un_partial_stated_rooms_key: int + thread_subscriptions_key: int _SEPARATOR = "_" START: ClassVar["StreamToken"] @@ -1006,6 +1104,7 @@ class StreamToken: device_list_key, groups_key, un_partial_stated_rooms_key, + thread_subscriptions_key, ) = keys return cls( @@ -1016,9 +1115,12 @@ class StreamToken: account_data_key=int(account_data_key), push_rules_key=int(push_rules_key), to_device_key=int(to_device_key), - device_list_key=int(device_list_key), + device_list_key=await MultiWriterStreamToken.parse( + store, device_list_key + ), groups_key=int(groups_key), un_partial_stated_rooms_key=int(un_partial_stated_rooms_key), + thread_subscriptions_key=int(thread_subscriptions_key), ) except CancelledError: raise @@ -1035,12 +1137,13 @@ class StreamToken: str(self.account_data_key), str(self.push_rules_key), str(self.to_device_key), - str(self.device_list_key), + await self.device_list_key.to_string(store), # Note that the groups key is no longer used, but it is still # serialized so that there will not be confusion in the future # if additional tokens are added. str(self.groups_key), str(self.un_partial_stated_rooms_key), + str(self.thread_subscriptions_key), ] ) @@ -1064,6 +1167,12 @@ class StreamToken: StreamKeyType.RECEIPT, self.receipt_key.copy_and_advance(new_value) ) return new_token + elif key == StreamKeyType.DEVICE_LIST: + new_token = self.copy_and_replace( + StreamKeyType.DEVICE_LIST, + self.device_list_key.copy_and_advance(new_value), + ) + return new_token new_token = self.copy_and_replace(key, new_value) new_id = new_token.get_field(key) @@ -1082,7 +1191,11 @@ class StreamToken: @overload def get_field( - self, key: Literal[StreamKeyType.RECEIPT] + self, + key: Literal[ + StreamKeyType.RECEIPT, + StreamKeyType.DEVICE_LIST, + ], ) -> MultiWriterStreamToken: ... @overload @@ -1090,12 +1203,12 @@ class StreamToken: self, key: Literal[ StreamKeyType.ACCOUNT_DATA, - StreamKeyType.DEVICE_LIST, StreamKeyType.PRESENCE, StreamKeyType.PUSH_RULES, StreamKeyType.TO_DEVICE, StreamKeyType.TYPING, StreamKeyType.UN_PARTIAL_STATED_ROOMS, + StreamKeyType.THREAD_SUBSCRIPTIONS, ], ) -> int: ... @@ -1151,12 +1264,23 @@ class StreamToken: f"typing: {self.typing_key}, receipt: {self.receipt_key}, " f"account_data: {self.account_data_key}, push_rules: {self.push_rules_key}, " f"to_device: {self.to_device_key}, device_list: {self.device_list_key}, " - f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key})" + f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}," + f"thread_subscriptions: {self.thread_subscriptions_key})" ) StreamToken.START = StreamToken( - RoomStreamToken(stream=0), 0, 0, MultiWriterStreamToken(stream=0), 0, 0, 0, 0, 0, 0 + room_key=RoomStreamToken(stream=0), + presence_key=0, + typing_key=0, + receipt_key=MultiWriterStreamToken(stream=0), + account_data_key=0, + push_rules_key=0, + to_device_key=0, + device_list_key=MultiWriterStreamToken(stream=0), + groups_key=0, + un_partial_stated_rooms_key=0, + thread_subscriptions_key=0, ) @@ -1203,6 +1327,27 @@ class SlidingSyncStreamToken: return f"{self.connection_position}/{stream_token_str}" +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ThreadSubscriptionsToken: + """ + Token for a position in the thread subscriptions stream. + + Format: `ts` + """ + + stream_id: int + + @staticmethod + def from_string(s: str) -> "ThreadSubscriptionsToken": + if not s.startswith("ts"): + raise ValueError("thread subscription token must start with `ts`") + + return ThreadSubscriptionsToken(stream_id=int(s[2:])) + + def to_string(self) -> str: + return f"ts{self.stream_id}" + + @attr.s(slots=True, frozen=True, auto_attribs=True) class PersistedPosition: """Position of a newly persisted row with instance that persisted it.""" @@ -1416,3 +1561,31 @@ class ScheduledTask: result: Optional[JsonMapping] # Optional error that should be assigned a value when the status is FAILED error: Optional[str] + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class EventOrderings: + stream: int + """ + The stream_ordering of the event. + Negative numbers mean the event was backfilled. + """ + + topological: int + """ + The topological_ordering of the event. + Currently this is equivalent to the `depth` attributes of + the PDU. + """ + + @staticmethod + def from_event(event: "EventBase") -> "EventOrderings": + """ + Get the orderings from an event. + + Preconditions: + - the event must have been persisted (otherwise it won't have a stream ordering) + """ + stream = event.internal_metadata.stream_ordering + assert stream is not None + return EventOrderings(stream, event.depth) diff --git a/synapse/types/handlers/policy_server.py b/synapse/types/handlers/policy_server.py new file mode 100644 index 0000000000..bfc09dabf4 --- /dev/null +++ b/synapse/types/handlers/policy_server.py @@ -0,0 +1,16 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +RECOMMENDATION_OK = "ok" +RECOMMENDATION_SPAM = "spam" diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index aae60fddea..b7bc565464 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -50,6 +50,7 @@ from synapse.types import ( SlidingSyncStreamToken, StrCollection, StreamToken, + ThreadSubscriptionsToken, UserID, ) from synapse.types.rest.client import SlidingSyncBody @@ -357,11 +358,50 @@ class SlidingSyncResult: def __bool__(self) -> bool: return bool(self.room_id_to_typing_map) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class ThreadSubscriptionsExtension: + """The Thread Subscriptions extension (MSC4308) + + Attributes: + subscribed: map (room_id -> thread_root_id -> info) of new or changed subscriptions + unsubscribed: map (room_id -> thread_root_id -> info) of new unsubscriptions + prev_batch: if present, there is a gap and the client can use this token to backpaginate + """ + + @attr.s(slots=True, frozen=True, auto_attribs=True) + class ThreadSubscription: + # always present when `subscribed` + automatic: Optional[bool] + + # the same as our stream_id; useful for clients to resolve + # race conditions locally + bump_stamp: int + + @attr.s(slots=True, frozen=True, auto_attribs=True) + class ThreadUnsubscription: + # the same as our stream_id; useful for clients to resolve + # race conditions locally + bump_stamp: int + + # room_id -> event_id (of thread root) -> the subscription change + subscribed: Optional[Mapping[str, Mapping[str, ThreadSubscription]]] + # room_id -> event_id (of thread root) -> the unsubscription + unsubscribed: Optional[Mapping[str, Mapping[str, ThreadUnsubscription]]] + prev_batch: Optional[ThreadSubscriptionsToken] + + def __bool__(self) -> bool: + return ( + bool(self.subscribed) + or bool(self.unsubscribed) + or bool(self.prev_batch) + ) + to_device: Optional[ToDeviceExtension] = None e2ee: Optional[E2eeExtension] = None account_data: Optional[AccountDataExtension] = None receipts: Optional[ReceiptsExtension] = None typing: Optional[TypingExtension] = None + thread_subscriptions: Optional[ThreadSubscriptionsExtension] = None def __bool__(self) -> bool: return bool( @@ -370,6 +410,7 @@ class SlidingSyncResult: or self.account_data or self.receipts or self.typing + or self.thread_subscriptions ) next_pos: SlidingSyncStreamToken @@ -407,8 +448,8 @@ class StateValues: # Include all state events of the given type WILDCARD: Final = "*" # Lazy-load room membership events (include room membership events for any event - # `sender` in the timeline). We only give special meaning to this value when it's a - # `state_key`. + # `sender` or membership change target in the timeline). We only give special + # meaning to this value when it's a `state_key`. LAZY: Final = "$LAZY" # Subsitute with the requester's user ID. Typically used by clients to get # the user's membership. @@ -641,9 +682,10 @@ class RoomSyncConfig: if user_id == StateValues.ME: continue # We're lazy-loading membership so we can just return the state we have. - # Lazy-loading means we include membership for any event `sender` in the - # timeline but since we had to auth those timeline events, we will have the - # membership state for them (including from remote senders). + # Lazy-loading means we include membership for any event `sender` or + # membership change target in the timeline but since we had to auth those + # timeline events, we will have the membership state for them (including + # from remote senders). elif user_id == StateValues.LAZY: continue elif user_id == StateValues.WILDCARD: diff --git a/synapse/types/rest/__init__.py b/synapse/types/rest/__init__.py index 183831e79a..a02836deee 100644 --- a/synapse/types/rest/__init__.py +++ b/synapse/types/rest/__init__.py @@ -18,26 +18,8 @@ # [This file includes modifications made by New Vector Limited] # # -from synapse._pydantic_compat import BaseModel, Extra +from synapse.util.pydantic_models import ParseModel -class RequestBodyModel(BaseModel): - """A custom version of Pydantic's BaseModel which - - - ignores unknown fields and - - does not allow fields to be overwritten after construction, - - but otherwise uses Pydantic's default behaviour. - - Ignoring unknown fields is a useful default. It means that clients can provide - unstable field not known to the server without the request being refused outright. - - Subclassing in this way is recommended by - https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally - """ - - class Config: - # By default, ignore fields that we don't recognise. - extra = Extra.ignore - # By default, don't allow fields to be reassigned after parsing. - allow_mutation = False +class RequestBodyModel(ParseModel): + pass diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index c739bd16b0..11d7e59b43 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from synapse._pydantic_compat import ( Extra, + Field, StrictBool, StrictInt, StrictStr, @@ -364,11 +365,25 @@ class SlidingSyncBody(RequestBodyModel): # Process all room subscriptions defined in the Room Subscription API. (This is the default.) rooms: Optional[List[StrictStr]] = ["*"] + class ThreadSubscriptionsExtension(RequestBodyModel): + """The Thread Subscriptions extension (MSC4308) + + Attributes: + enabled + limit: maximum number of subscription changes to return (default 100) + """ + + enabled: Optional[StrictBool] = False + limit: StrictInt = 100 + to_device: Optional[ToDeviceExtension] = None e2ee: Optional[E2eeExtension] = None account_data: Optional[AccountDataExtension] = None receipts: Optional[ReceiptsExtension] = None typing: Optional[TypingExtension] = None + thread_subscriptions: Optional[ThreadSubscriptionsExtension] = Field( + alias="io.element.msc4308.thread_subscriptions" + ) conn_id: Optional[StrictStr] diff --git a/synapse/types/state.py b/synapse/types/state.py index e641215f18..6420e050a5 100644 --- a/synapse/types/state.py +++ b/synapse/types/state.py @@ -462,7 +462,7 @@ class StateFilter: new_types.update({state_type: set() for state_type in minus_wildcards}) # insert the plus wildcards - new_types.update({state_type: None for state_type in plus_wildcards}) + new_types.update(dict.fromkeys(plus_wildcards)) # insert the specific state keys for state_type, state_key in plus_state_keys: diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index b5fa20a41a..b01653246a 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -38,6 +38,16 @@ class _BackgroundUpdates: EVENTS_JUMP_TO_DATE_INDEX = "events_jump_to_date_index" + CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "current_state_events_stream_ordering_idx" + ) + ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "room_memberships_stream_ordering_idx" + ) + LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "local_current_membership_stream_ordering_idx" + ) + SLIDING_SYNC_PREFILL_JOINED_ROOMS_TO_RECALCULATE_TABLE_BG_UPDATE = ( "sliding_sync_prefill_joined_rooms_to_recalculate_table_bg_update" ) @@ -48,3 +58,9 @@ class _BackgroundUpdates: SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE = ( "sliding_sync_membership_snapshots_fix_forgotten_column_bg_update" ) + + MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE = ( + "mark_unreferenced_state_groups_for_deletion_bg_update" + ) + + FIXUP_MAX_DEPTH_CAP = "fixup_max_depth_cap" diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index e0d876e84b..36129c3a67 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -19,10 +19,21 @@ # # +import collections.abc import json import logging import typing -from typing import Any, Callable, Dict, Generator, Optional, Sequence +from typing import ( + Any, + Callable, + Dict, + Iterator, + Mapping, + Optional, + Sequence, + Set, + TypeVar, +) import attr from immutabledict import immutabledict @@ -30,7 +41,6 @@ from matrix_common.versionstring import get_distribution_version_string from typing_extensions import ParamSpec from twisted.internet import defer, task -from twisted.internet.defer import Deferred from twisted.internet.interfaces import IDelayedCall, IReactorTime from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -43,6 +53,15 @@ if typing.TYPE_CHECKING: logger = logging.getLogger(__name__) +class Duration: + """Helper class that holds constants for common time durations in + milliseconds.""" + + MINUTE_MS = 60 * 1000 + HOUR_MS = 60 * MINUTE_MS + DAY_MS = 24 * HOUR_MS + + def _reject_invalid_json(val: Any) -> None: """Do not allow Infinity, -Infinity, or NaN values in JSON.""" raise ValueError("Invalid JSON value: '%s'" % val) @@ -100,13 +119,11 @@ class Clock: _reactor: IReactorTime = attr.ib() - @defer.inlineCallbacks - def sleep(self, seconds: float) -> "Generator[Deferred[float], Any, Any]": + async def sleep(self, seconds: float) -> None: d: defer.Deferred[float] = defer.Deferred() with context.PreserveLoggingContext(): self._reactor.callLater(seconds, d.callback, seconds) - res = yield d - return res + await d def time(self) -> float: """Returns the current system time in seconds since epoch.""" @@ -251,3 +268,72 @@ class ExceptionBundle(Exception): parts.append(str(e)) super().__init__("\n - ".join(parts)) self.exceptions = exceptions + + +K = TypeVar("K") +V = TypeVar("V") + + +@attr.s(slots=True, auto_attribs=True) +class MutableOverlayMapping(collections.abc.MutableMapping[K, V]): + """A mutable mapping that allows changes to a read-only underlying + mapping. Supports deletions. + + This is useful for cases where you want to allow modifications to a mapping + without changing or copying the original mapping. + + Note: the underlying mapping must not change while this proxy is in use. + """ + + _underlying_map: Mapping[K, V] + _mutable_map: Dict[K, V] = attr.ib(factory=dict) + _deletions: Set[K] = attr.ib(factory=set) + + def __getitem__(self, key: K) -> V: + if key in self._deletions: + raise KeyError(key) + if key in self._mutable_map: + return self._mutable_map[key] + return self._underlying_map[key] + + def __setitem__(self, key: K, value: V) -> None: + self._deletions.discard(key) + self._mutable_map[key] = value + + def __delitem__(self, key: K) -> None: + if key not in self: + raise KeyError(key) + + self._deletions.add(key) + self._mutable_map.pop(key, None) + + def __iter__(self) -> Iterator[K]: + for key in self._mutable_map: + if key not in self._deletions: + yield key + + for key in self._underlying_map: + if key not in self._deletions and key not in self._mutable_map: + # `key` should not be in both _mutable_map and _deletions + assert key not in self._mutable_map + yield key + + def __len__(self) -> int: + count = len(self._underlying_map) + for key in self._deletions: + if key in self._underlying_map: + count -= 1 + + for key in self._mutable_map: + # `key` should not be in both _mutable_map and _deletions + assert key not in self._deletions + + if key not in self._underlying_map: + count += 1 + + return count + + def clear(self) -> None: + self._underlying_map = {} + self._mutable_map.clear() + self._deletions.clear() diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index e1eb8a4863..c21b7887f9 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -41,6 +41,7 @@ from typing import ( Hashable, Iterable, List, + Literal, Optional, Set, Tuple, @@ -51,7 +52,7 @@ from typing import ( ) import attr -from typing_extensions import Concatenate, Literal, ParamSpec, Unpack +from typing_extensions import Concatenate, ParamSpec, Unpack from twisted.internet import defer from twisted.internet.defer import CancelledError @@ -346,6 +347,7 @@ T2 = TypeVar("T2") T3 = TypeVar("T3") T4 = TypeVar("T4") T5 = TypeVar("T5") +T6 = TypeVar("T6") @overload @@ -460,6 +462,23 @@ async def gather_optional_coroutines( ) -> Tuple[Optional[T1], Optional[T2], Optional[T3], Optional[T4], Optional[T5]]: ... +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + Tuple[ + Optional[Coroutine[Any, Any, T1]], + Optional[Coroutine[Any, Any, T2]], + Optional[Coroutine[Any, Any, T3]], + Optional[Coroutine[Any, Any, T4]], + Optional[Coroutine[Any, Any, T5]], + Optional[Coroutine[Any, Any, T6]], + ] + ], +) -> Tuple[ + Optional[T1], Optional[T2], Optional[T3], Optional[T4], Optional[T5], Optional[T6] +]: ... + + async def gather_optional_coroutines( *coroutines: Unpack[Tuple[Optional[Coroutine[Any, Any, T1]], ...]], ) -> Tuple[Optional[T1], ...]: diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index 3fb697751f..4c0f129423 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -37,6 +37,7 @@ from prometheus_client import Gauge from twisted.internet import defer from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util import Clock @@ -49,19 +50,19 @@ R = TypeVar("R") number_queued = Gauge( "synapse_util_batching_queue_number_queued", "The number of items waiting in the queue across all keys", - labelnames=("name",), + labelnames=("name", SERVER_NAME_LABEL), ) number_in_flight = Gauge( "synapse_util_batching_queue_number_pending", "The number of items across all keys either being processed or waiting in a queue", - labelnames=("name",), + labelnames=("name", SERVER_NAME_LABEL), ) number_of_keys = Gauge( "synapse_util_batching_queue_number_of_keys", "The number of distinct keys that have items queued", - labelnames=("name",), + labelnames=("name", SERVER_NAME_LABEL), ) @@ -85,6 +86,8 @@ class BatchingQueue(Generic[V, R]): Args: name: A name for the queue, used for logging contexts and metrics. This must be unique, otherwise the metrics will be wrong. + server_name: The homeserver name of the server (used to label metrics) + (this should be `hs.hostname`). clock: The clock to use to schedule work. process_batch_callback: The callback to to be run to process a batch of work. @@ -92,11 +95,14 @@ class BatchingQueue(Generic[V, R]): def __init__( self, + *, name: str, + server_name: str, clock: Clock, process_batch_callback: Callable[[List[V]], Awaitable[R]], ): self._name = name + self.server_name = server_name self._clock = clock # The set of keys currently being processed. @@ -109,14 +115,18 @@ class BatchingQueue(Generic[V, R]): # The function to call with batches of values. self._process_batch_callback = process_batch_callback - number_queued.labels(self._name).set_function( - lambda: sum(len(q) for q in self._next_values.values()) + number_queued.labels( + name=self._name, **{SERVER_NAME_LABEL: self.server_name} + ).set_function(lambda: sum(len(q) for q in self._next_values.values())) + + number_of_keys.labels( + name=self._name, **{SERVER_NAME_LABEL: self.server_name} + ).set_function(lambda: len(self._next_values)) + + self._number_in_flight_metric: Gauge = number_in_flight.labels( + name=self._name, **{SERVER_NAME_LABEL: self.server_name} ) - number_of_keys.labels(self._name).set_function(lambda: len(self._next_values)) - - self._number_in_flight_metric: Gauge = number_in_flight.labels(self._name) - async def add_to_queue(self, value: V, key: Hashable = ()) -> R: """Adds the value to the queue with the given key, returning the result of the processing function for the batch that included the given value. @@ -135,7 +145,9 @@ class BatchingQueue(Generic[V, R]): # If we're not currently processing the key fire off a background # process to start processing. if key not in self._processing_keys: - run_as_background_process(self._name, self._process_queue, key) + run_as_background_process( + self._name, self.server_name, self._process_queue, key + ) with self._number_in_flight_metric.track_inprogress(): return await make_deferred_yieldable(d) diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index df8829baeb..710a29e3f0 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -31,6 +31,7 @@ from prometheus_client import REGISTRY from prometheus_client.core import Gauge from synapse.config.cache import add_resizable_cache +from synapse.metrics import SERVER_NAME_LABEL from synapse.util.metrics import DynamicCollectorRegistry logger = logging.getLogger(__name__) @@ -41,55 +42,71 @@ TRACK_MEMORY_USAGE = False # We track cache metrics in a special registry that lets us update the metrics # just before they are returned from the scrape endpoint. -CACHE_METRIC_REGISTRY = DynamicCollectorRegistry() - -caches_by_name: Dict[str, Sized] = {} +# +# The `SERVER_NAME_LABEL` is included in the individual metrics added to this registry, +# so we don't need to worry about it on the collector itself. +CACHE_METRIC_REGISTRY = DynamicCollectorRegistry() # type: ignore[missing-server-name-label] cache_size = Gauge( - "synapse_util_caches_cache_size", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache_size", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_hits = Gauge( - "synapse_util_caches_cache_hits", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache_hits", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_evicted = Gauge( "synapse_util_caches_cache_evicted_size", "", - ["name", "reason"], + labelnames=["name", "reason", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) cache_total = Gauge( - "synapse_util_caches_cache", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_max_size = Gauge( - "synapse_util_caches_cache_max_size", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache_max_size", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_memory_usage = Gauge( "synapse_util_caches_cache_size_bytes", "Estimated memory usage of the caches", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_size = Gauge( "synapse_util_caches_response_cache_size", "", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_hits = Gauge( "synapse_util_caches_response_cache_hits", "", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_evicted = Gauge( "synapse_util_caches_response_cache_evicted_size", "", - ["name", "reason"], + labelnames=["name", "reason", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_total = Gauge( - "synapse_util_caches_response_cache", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_response_cache", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) @@ -103,12 +120,17 @@ class EvictionReason(Enum): invalidation = auto() -@attr.s(slots=True, auto_attribs=True) +@attr.s(slots=True, auto_attribs=True, kw_only=True) class CacheMetric: + """ + Used to track cache metrics + """ + _cache: Sized _cache_type: str _cache_name: str _collect_callback: Optional[Callable] + _server_name: str hits: int = 0 misses: int = 0 @@ -145,34 +167,34 @@ class CacheMetric: def collect(self) -> None: try: + labels_base = { + "name": self._cache_name, + SERVER_NAME_LABEL: self._server_name, + } if self._cache_type == "response_cache": - response_cache_size.labels(self._cache_name).set(len(self._cache)) - response_cache_hits.labels(self._cache_name).set(self.hits) + response_cache_size.labels(**labels_base).set(len(self._cache)) + response_cache_hits.labels(**labels_base).set(self.hits) for reason in EvictionReason: - response_cache_evicted.labels(self._cache_name, reason.name).set( - self.eviction_size_by_reason[reason] - ) - response_cache_total.labels(self._cache_name).set( - self.hits + self.misses - ) + response_cache_evicted.labels( + **{**labels_base, "reason": reason.name} + ).set(self.eviction_size_by_reason[reason]) + response_cache_total.labels(**labels_base).set(self.hits + self.misses) else: - cache_size.labels(self._cache_name).set(len(self._cache)) - cache_hits.labels(self._cache_name).set(self.hits) + cache_size.labels(**labels_base).set(len(self._cache)) + cache_hits.labels(**labels_base).set(self.hits) for reason in EvictionReason: - cache_evicted.labels(self._cache_name, reason.name).set( + cache_evicted.labels(**{**labels_base, "reason": reason.name}).set( self.eviction_size_by_reason[reason] ) - cache_total.labels(self._cache_name).set(self.hits + self.misses) + cache_total.labels(**labels_base).set(self.hits + self.misses) max_size = getattr(self._cache, "max_size", None) if max_size: - cache_max_size.labels(self._cache_name).set(max_size) + cache_max_size.labels(**labels_base).set(max_size) if TRACK_MEMORY_USAGE: # self.memory_usage can be None if nothing has been inserted # into the cache yet. - cache_memory_usage.labels(self._cache_name).set( - self.memory_usage or 0 - ) + cache_memory_usage.labels(**labels_base).set(self.memory_usage or 0) if self._collect_callback: self._collect_callback() except Exception as e: @@ -181,9 +203,11 @@ class CacheMetric: def register_cache( + *, cache_type: str, cache_name: str, cache: Sized, + server_name: str, collect_callback: Optional[Callable] = None, resizable: bool = True, resize_callback: Optional[Callable] = None, @@ -196,6 +220,8 @@ def register_cache( cache_name: name of the cache cache: cache itself, which must implement __len__(), and may optionally implement a max_size property + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). collect_callback: If given, a function which is called during metric collection to update additional metrics. resizable: Whether this cache supports being resized, in which case either @@ -210,9 +236,14 @@ def register_cache( resize_callback = cache.set_cache_factor # type: ignore add_resizable_cache(cache_name, resize_callback) - metric = CacheMetric(cache, cache_type, cache_name, collect_callback) - metric_name = "cache_%s_%s" % (cache_type, cache_name) - caches_by_name[cache_name] = cache + metric = CacheMetric( + cache=cache, + cache_type=cache_type, + cache_name=cache_name, + server_name=server_name, + collect_callback=collect_callback, + ) + metric_name = "cache_%s_%s_%s" % (cache_type, cache_name, server_name) CACHE_METRIC_REGISTRY.register_hook(metric_name, metric.collect) return metric @@ -225,7 +256,7 @@ KNOWN_KEYS = { "depth", "event_id", "hashes", - "origin", + "origin", # old events were created with an origin field. "origin_server_ts", "prev_events", "room_id", diff --git a/synapse/util/caches/deferred_cache.py b/synapse/util/caches/deferred_cache.py index 14868fa4d3..92d446ce2a 100644 --- a/synapse/util/caches/deferred_cache.py +++ b/synapse/util/caches/deferred_cache.py @@ -43,6 +43,7 @@ from prometheus_client import Gauge from twisted.internet import defer from twisted.python.failure import Failure +from synapse.metrics import SERVER_NAME_LABEL from synapse.util.async_helpers import ObservableDeferred from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache, iterate_tree_cache_entry @@ -50,7 +51,7 @@ from synapse.util.caches.treecache import TreeCache, iterate_tree_cache_entry cache_pending_metric = Gauge( "synapse_util_caches_cache_pending", "Number of lookups currently pending for this cache", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], ) T = TypeVar("T") @@ -79,7 +80,9 @@ class DeferredCache(Generic[KT, VT]): def __init__( self, + *, name: str, + server_name: str, max_entries: int = 1000, tree: bool = False, iterable: bool = False, @@ -89,6 +92,8 @@ class DeferredCache(Generic[KT, VT]): """ Args: name: The name of the cache + server_name: server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). max_entries: Maximum amount of entries that the cache will hold tree: Use a TreeCache instead of a dict as the underlying cache type iterable: If True, count each item in the cached object as an entry, @@ -107,12 +112,15 @@ class DeferredCache(Generic[KT, VT]): ] = cache_type() def metrics_cb() -> None: - cache_pending_metric.labels(name).set(len(self._pending_deferred_cache)) + cache_pending_metric.labels( + name=name, **{SERVER_NAME_LABEL: server_name} + ).set(len(self._pending_deferred_cache)) # cache is used for completed results and maps to the result itself, rather than # a Deferred. self.cache: LruCache[KT, VT] = LruCache( max_size=max_entries, + server_name=server_name, cache_name=name, cache_type=cache_type, size_callback=( diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 29a9586710..47b8f4ddc8 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -33,6 +33,7 @@ from typing import ( List, Mapping, Optional, + Protocol, Sequence, Tuple, Type, @@ -153,6 +154,14 @@ class _CacheDescriptorBase: ) +class HasServerName(Protocol): + server_name: str + """ + The homeserver name that this cache is associated with (used to label the metric) + (`hs.hostname`). + """ + + class DeferredCacheDescriptor(_CacheDescriptorBase): """A method decorator that applies a memoizing cache around the function. @@ -200,6 +209,7 @@ class DeferredCacheDescriptor(_CacheDescriptorBase): def __init__( self, + *, orig: Callable[..., Any], max_entries: int = 1000, num_args: Optional[int] = None, @@ -229,10 +239,20 @@ class DeferredCacheDescriptor(_CacheDescriptorBase): self.prune_unread_entries = prune_unread_entries def __get__( - self, obj: Optional[Any], owner: Optional[Type] + self, obj: Optional[HasServerName], owner: Optional[Type] ) -> Callable[..., "defer.Deferred[Any]"]: + # We need access to instance-level `obj.server_name` attribute + assert obj is not None, ( + "Cannot call cached method from class (❌ `MyClass.cached_method()`) " + "and must be called from an instance (✅ `MyClass().cached_method()`). " + ) + assert obj.server_name is not None, ( + "The `server_name` attribute must be set on the object where `@cached` decorator is used." + ) + cache: DeferredCache[CacheKey, Any] = DeferredCache( name=self.name, + server_name=obj.server_name, max_entries=self.max_entries, tree=self.tree, iterable=self.iterable, @@ -490,7 +510,7 @@ class _CachedFunctionDescriptor: def __call__(self, orig: F) -> CachedFunction[F]: d = DeferredCacheDescriptor( - orig, + orig=orig, max_entries=self.max_entries, num_args=self.num_args, uncached_args=self.uncached_args, @@ -559,9 +579,12 @@ def cachedList( Used to do batch lookups for an already created cache. One of the arguments is specified as a list that is iterated through to lookup keys in the original cache. A new tuple consisting of the (deduplicated) keys that weren't in - the cache gets passed to the original function, which is expected to results + the cache gets passed to the original function, which is expected to result in a map of key to value for each passed value. The new results are stored in the - original cache. Note that any missing values are cached as None. + original cache. + + Note that any values in the input that end up being missing from both the + cache and the returned dictionary will be cached as `None`. Args: cached_method_name: The name of the single-item lookup method. diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index 1e6696332f..168ddc51cd 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -21,10 +21,19 @@ import enum import logging import threading -from typing import Dict, Generic, Iterable, Optional, Set, Tuple, TypeVar, Union +from typing import ( + Dict, + Generic, + Iterable, + Literal, + Optional, + Set, + Tuple, + TypeVar, + Union, +) import attr -from typing_extensions import Literal from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache @@ -118,7 +127,15 @@ class DictionaryCache(Generic[KT, DKT, DV]): for the '2' dict key. """ - def __init__(self, name: str, max_entries: int = 1000): + def __init__(self, *, name: str, server_name: str, max_entries: int = 1000): + """ + Args: + name + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). + max_entries + """ + # We use a single LruCache to store two different types of entries: # 1. Map from (key, dict_key) -> dict value (or sentinel, indicating # the key doesn't exist in the dict); and @@ -143,6 +160,7 @@ class DictionaryCache(Generic[KT, DKT, DV]): Union[_PerKeyValue, Dict[DKT, DV]], ] = LruCache( max_size=max_entries, + server_name=server_name, cache_name=name, cache_type=TreeCache, size_callback=len, diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 8017c031ee..1962a3fdfa 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -21,10 +21,9 @@ import logging from collections import OrderedDict -from typing import Any, Generic, Iterable, Optional, TypeVar, Union, overload +from typing import Any, Generic, Iterable, Literal, Optional, TypeVar, Union, overload import attr -from typing_extensions import Literal from twisted.internet import defer @@ -47,7 +46,9 @@ VT = TypeVar("VT") class ExpiringCache(Generic[KT, VT]): def __init__( self, + *, cache_name: str, + server_name: str, clock: Clock, max_len: int = 0, expiry_ms: int = 0, @@ -57,6 +58,8 @@ class ExpiringCache(Generic[KT, VT]): """ Args: cache_name: Name of this cache, used for logging. + server_name: The homeserver name that this cache is associated + with (used to label the metric) (`hs.hostname`). clock max_len: Max size of dict. If the dict grows larger than this then the oldest items get automatically evicted. Default is 0, @@ -84,14 +87,21 @@ class ExpiringCache(Generic[KT, VT]): self.iterable = iterable - self.metrics = register_cache("expiring", cache_name, self) + self.metrics = register_cache( + cache_type="expiring", + cache_name=cache_name, + cache=self, + server_name=server_name, + ) if not self._expiry_ms: # Don't bother starting the loop if things never expire return def f() -> "defer.Deferred[None]": - return run_as_background_process("prune_cache", self._prune_cache) + return run_as_background_process( + "prune_cache", server_name, self._prune_cache + ) self._clock.looping_call(f, self._expiry_ms / 2) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 481a1a621e..927162700a 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -34,6 +34,7 @@ from typing import ( Generic, Iterable, List, + Literal, Optional, Set, Tuple, @@ -44,13 +45,13 @@ from typing import ( overload, ) -from typing_extensions import Literal - -from twisted.internet import reactor +from twisted.internet import defer, reactor from twisted.internet.interfaces import IReactorTime from synapse.config import cache as cache_config -from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.metrics.background_process_metrics import ( + run_as_background_process, +) from synapse.metrics.jemalloc import get_jemalloc_stats from synapse.util import Clock, caches from synapse.util.caches import CacheMetric, EvictionReason, register_cache @@ -119,103 +120,121 @@ USE_GLOBAL_LIST = False GLOBAL_ROOT = ListNode["_Node"].create_root_node() -@wrap_as_background_process("LruCache._expire_old_entries") -async def _expire_old_entries( - clock: Clock, expiry_seconds: float, autotune_config: Optional[dict] -) -> None: +def _expire_old_entries( + server_name: str, + clock: Clock, + expiry_seconds: float, + autotune_config: Optional[dict], +) -> "defer.Deferred[None]": """Walks the global cache list to find cache entries that haven't been accessed in the given number of seconds, or if a given memory threshold has been breached. """ - if autotune_config: - max_cache_memory_usage = autotune_config["max_cache_memory_usage"] - target_cache_memory_usage = autotune_config["target_cache_memory_usage"] - min_cache_ttl = autotune_config["min_cache_ttl"] / 1000 - now = int(clock.time()) - node = GLOBAL_ROOT.prev_node - assert node is not None + async def _internal_expire_old_entries( + clock: Clock, expiry_seconds: float, autotune_config: Optional[dict] + ) -> None: + if autotune_config: + max_cache_memory_usage = autotune_config["max_cache_memory_usage"] + target_cache_memory_usage = autotune_config["target_cache_memory_usage"] + min_cache_ttl = autotune_config["min_cache_ttl"] / 1000 - i = 0 + now = int(clock.time()) + node = GLOBAL_ROOT.prev_node + assert node is not None - logger.debug("Searching for stale caches") + i = 0 - evicting_due_to_memory = False + logger.debug("Searching for stale caches") - # determine if we're evicting due to memory - jemalloc_interface = get_jemalloc_stats() - if jemalloc_interface and autotune_config: - try: - jemalloc_interface.refresh_stats() - mem_usage = jemalloc_interface.get_stat("allocated") - if mem_usage > max_cache_memory_usage: - logger.info("Begin memory-based cache eviction.") - evicting_due_to_memory = True - except Exception: - logger.warning( - "Unable to read allocated memory, skipping memory-based cache eviction." - ) + evicting_due_to_memory = False - while node is not GLOBAL_ROOT: - # Only the root node isn't a `_TimedListNode`. - assert isinstance(node, _TimedListNode) - - # if node has not aged past expiry_seconds and we are not evicting due to memory usage, there's - # nothing to do here - if ( - node.last_access_ts_secs > now - expiry_seconds - and not evicting_due_to_memory - ): - break - - # if entry is newer than min_cache_entry_ttl then do not evict and don't evict anything newer - if evicting_due_to_memory and now - node.last_access_ts_secs < min_cache_ttl: - break - - cache_entry = node.get_cache_entry() - next_node = node.prev_node - - # The node should always have a reference to a cache entry and a valid - # `prev_node`, as we only drop them when we remove the node from the - # list. - assert next_node is not None - assert cache_entry is not None - cache_entry.drop_from_cache() - - # Check mem allocation periodically if we are evicting a bunch of caches - if jemalloc_interface and evicting_due_to_memory and (i + 1) % 100 == 0: + # determine if we're evicting due to memory + jemalloc_interface = get_jemalloc_stats() + if jemalloc_interface and autotune_config: try: jemalloc_interface.refresh_stats() mem_usage = jemalloc_interface.get_stat("allocated") - if mem_usage < target_cache_memory_usage: - evicting_due_to_memory = False - logger.info("Stop memory-based cache eviction.") + if mem_usage > max_cache_memory_usage: + logger.info("Begin memory-based cache eviction.") + evicting_due_to_memory = True except Exception: logger.warning( - "Unable to read allocated memory, this may affect memory-based cache eviction." + "Unable to read allocated memory, skipping memory-based cache eviction." ) - # If we've failed to read the current memory usage then we - # should stop trying to evict based on memory usage - evicting_due_to_memory = False - # If we do lots of work at once we yield to allow other stuff to happen. - if (i + 1) % 10000 == 0: - logger.debug("Waiting during drop") - if node.last_access_ts_secs > now - expiry_seconds: - await clock.sleep(0.5) - else: - await clock.sleep(0) - logger.debug("Waking during drop") + while node is not GLOBAL_ROOT: + # Only the root node isn't a `_TimedListNode`. + assert isinstance(node, _TimedListNode) - node = next_node + # if node has not aged past expiry_seconds and we are not evicting due to memory usage, there's + # nothing to do here + if ( + node.last_access_ts_secs > now - expiry_seconds + and not evicting_due_to_memory + ): + break - # If we've yielded then our current node may have been evicted, so we - # need to check that its still valid. - if node.prev_node is None: - break + # if entry is newer than min_cache_entry_ttl then do not evict and don't evict anything newer + if ( + evicting_due_to_memory + and now - node.last_access_ts_secs < min_cache_ttl + ): + break - i += 1 + cache_entry = node.get_cache_entry() + next_node = node.prev_node - logger.info("Dropped %d items from caches", i) + # The node should always have a reference to a cache entry and a valid + # `prev_node`, as we only drop them when we remove the node from the + # list. + assert next_node is not None + assert cache_entry is not None + cache_entry.drop_from_cache() + + # Check mem allocation periodically if we are evicting a bunch of caches + if jemalloc_interface and evicting_due_to_memory and (i + 1) % 100 == 0: + try: + jemalloc_interface.refresh_stats() + mem_usage = jemalloc_interface.get_stat("allocated") + if mem_usage < target_cache_memory_usage: + evicting_due_to_memory = False + logger.info("Stop memory-based cache eviction.") + except Exception: + logger.warning( + "Unable to read allocated memory, this may affect memory-based cache eviction." + ) + # If we've failed to read the current memory usage then we + # should stop trying to evict based on memory usage + evicting_due_to_memory = False + + # If we do lots of work at once we yield to allow other stuff to happen. + if (i + 1) % 10000 == 0: + logger.debug("Waiting during drop") + if node.last_access_ts_secs > now - expiry_seconds: + await clock.sleep(0.5) + else: + await clock.sleep(0) + logger.debug("Waking during drop") + + node = next_node + + # If we've yielded then our current node may have been evicted, so we + # need to check that its still valid. + if node.prev_node is None: + break + + i += 1 + + logger.info("Dropped %d items from caches", i) + + return run_as_background_process( + "LruCache._expire_old_entries", + server_name, + _internal_expire_old_entries, + clock, + expiry_seconds, + autotune_config, + ) def setup_expire_lru_cache_entries(hs: "HomeServer") -> None: @@ -235,10 +254,12 @@ def setup_expire_lru_cache_entries(hs: "HomeServer") -> None: global USE_GLOBAL_LIST USE_GLOBAL_LIST = True + server_name = hs.hostname clock = hs.get_clock() clock.looping_call( _expire_old_entries, 30 * 1000, + server_name, clock, expiry_time, hs.config.caches.cache_autotuning, @@ -377,9 +398,43 @@ class LruCache(Generic[KT, VT]): If cache_type=TreeCache, all keys must be tuples. """ + @overload def __init__( self, + *, max_size: int, + server_name: str, + cache_name: str, + cache_type: Type[Union[dict, TreeCache]] = dict, + size_callback: Optional[Callable[[VT], int]] = None, + metrics_collection_callback: Optional[Callable[[], None]] = None, + apply_cache_factor_from_config: bool = True, + clock: Optional[Clock] = None, + prune_unread_entries: bool = True, + extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, + ): ... + + @overload + def __init__( + self, + *, + max_size: int, + server_name: Literal[None] = None, + cache_name: Literal[None] = None, + cache_type: Type[Union[dict, TreeCache]] = dict, + size_callback: Optional[Callable[[VT], int]] = None, + metrics_collection_callback: Optional[Callable[[], None]] = None, + apply_cache_factor_from_config: bool = True, + clock: Optional[Clock] = None, + prune_unread_entries: bool = True, + extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, + ): ... + + def __init__( + self, + *, + max_size: int, + server_name: Optional[str] = None, cache_name: Optional[str] = None, cache_type: Type[Union[dict, TreeCache]] = dict, size_callback: Optional[Callable[[VT], int]] = None, @@ -393,8 +448,13 @@ class LruCache(Generic[KT, VT]): Args: max_size: The maximum amount of entries the cache can hold - cache_name: The name of this cache, for the prometheus metrics. If unset, - no metrics will be reported on this cache. + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). Must be set if `cache_name` is + set. If unset, no metrics will be reported on this cache. + + cache_name: The name of this cache, for the prometheus metrics. Must be set + if `server_name` is set. If unset, no metrics will be reported on this + cache. cache_type: type of underlying cache to be used. Typically one of dict @@ -458,11 +518,12 @@ class LruCache(Generic[KT, VT]): # do yet when we get resized. self._on_resize: Optional[Callable[[], None]] = None - if cache_name is not None: + if cache_name is not None and server_name is not None: metrics: Optional[CacheMetric] = register_cache( - "lru_cache", - cache_name, - self, + cache_type="lru_cache", + cache_name=cache_name, + cache=self, + server_name=server_name, collect_callback=metrics_collection_callback, ) else: diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 96b7ca83dc..49a9151916 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -101,14 +101,38 @@ class ResponseCache(Generic[KV]): used rather than trying to compute a new response. """ - def __init__(self, clock: Clock, name: str, timeout_ms: float = 0): + def __init__( + self, + *, + clock: Clock, + name: str, + server_name: str, + timeout_ms: float = 0, + enable_logging: bool = True, + ): + """ + Args: + clock + name + server_name: The homeserver name that this cache is associated + with (used to label the metric) (`hs.hostname`). + timeout_ms + enable_logging + """ self._result_cache: Dict[KV, ResponseCacheEntry] = {} self.clock = clock self.timeout_sec = timeout_ms / 1000.0 self._name = name - self._metrics = register_cache("response_cache", name, self, resizable=False) + self._metrics = register_cache( + cache_type="response_cache", + cache_name=name, + cache=self, + server_name=server_name, + resizable=False, + ) + self._enable_logging = enable_logging def size(self) -> int: return len(self._result_cache) @@ -246,9 +270,12 @@ class ResponseCache(Generic[KV]): """ entry = self._get(key) if not entry: - logger.debug( - "[%s]: no cached result for [%s], calculating new one", self._name, key - ) + if self._enable_logging: + logger.debug( + "[%s]: no cached result for [%s], calculating new one", + self._name, + key, + ) context = ResponseCacheContext(cache_key=key) if cache_context: kwargs["cache_context"] = context @@ -269,12 +296,15 @@ class ResponseCache(Generic[KV]): return await make_deferred_yieldable(entry.result.observe()) result = entry.result.observe() - if result.called: - logger.info("[%s]: using completed cached result for [%s]", self._name, key) - else: - logger.info( - "[%s]: using incomplete cached result for [%s]", self._name, key - ) + if self._enable_logging: + if result.called: + logger.info( + "[%s]: using completed cached result for [%s]", self._name, key + ) + else: + logger.info( + "[%s]: using incomplete cached result for [%s]", self._name, key + ) span_context = entry.opentracing_span_context with start_active_span_follows_from( diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 03503abe0f..2cffd352d8 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -73,11 +73,23 @@ class StreamChangeCache: def __init__( self, + *, name: str, + server_name: str, current_stream_pos: int, max_size: int = 10000, prefilled_cache: Optional[Mapping[EntityType, int]] = None, ) -> None: + """ + Args: + name + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). + current_stream_pos + max_size + prefilled_cache + """ + self._original_max_size: int = max_size self._max_size = math.floor(max_size) @@ -96,7 +108,11 @@ class StreamChangeCache: self.name = name self.metrics = caches.register_cache( - "cache", self.name, self._cache, resize_callback=self.set_cache_factor + cache_type="cache", + cache_name=self.name, + server_name=server_name, + cache=self._cache, + resize_callback=self.set_cache_factor, ) if prefilled_cache: @@ -314,6 +330,15 @@ class StreamChangeCache: self._entity_to_key[entity] = stream_pos self._evict() + def all_entities_changed(self, stream_pos: int) -> None: + """ + Mark all entities as changed. This is useful when the cache is invalidated and + there may be some potential change for all of the entities. + """ + self._cache.clear() + self._entity_to_key.clear() + self._earliest_known_stream_pos = stream_pos + def _evict(self) -> None: """ Ensure the cache has not exceeded the maximum size. diff --git a/synapse/util/caches/ttlcache.py b/synapse/util/caches/ttlcache.py index 26a088603a..18c3a1e51c 100644 --- a/synapse/util/caches/ttlcache.py +++ b/synapse/util/caches/ttlcache.py @@ -40,7 +40,21 @@ VT = TypeVar("VT") class TTLCache(Generic[KT, VT]): """A key/value cache implementation where each entry has its own TTL""" - def __init__(self, cache_name: str, timer: Callable[[], float] = time.time): + def __init__( + self, + *, + cache_name: str, + server_name: str, + timer: Callable[[], float] = time.time, + ): + """ + Args: + cache_name + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). + timer: Function used to get the current time in seconds since the epoch. + """ + # map from key to _CacheEntry self._data: Dict[KT, _CacheEntry[KT, VT]] = {} @@ -49,7 +63,13 @@ class TTLCache(Generic[KT, VT]): self._timer = timer - self._metrics = register_cache("ttl", cache_name, self, resizable=False) + self._metrics = register_cache( + cache_type="ttl", + cache_name=cache_name, + cache=self, + server_name=server_name, + resizable=False, + ) def set(self, key: KT, value: VT, ttl: float) -> None: """Add/update an entry in the cache diff --git a/synapse/util/check_dependencies.py b/synapse/util/check_dependencies.py index 68336814c0..1c79c0be48 100644 --- a/synapse/util/check_dependencies.py +++ b/synapse/util/check_dependencies.py @@ -37,6 +37,8 @@ DISTRIBUTION_NAME = "matrix-synapse" __all__ = ["check_requirements"] +logger = logging.getLogger(__name__) + class DependencyException(Exception): @property @@ -72,7 +74,7 @@ def _is_dev_dependency(req: Requirement) -> bool: def _should_ignore_runtime_requirement(req: Requirement) -> bool: # This is a build-time dependency. Irritatingly, `poetry build` ignores the # requirements listed in the [build-system] section of pyproject.toml, so in order - # to support `poetry install --no-dev` we have to mark it as a runtime dependency. + # to support `poetry install --without dev` we have to mark it as a runtime dependency. # See discussion on https://github.com/python-poetry/poetry/issues/6154 (it sounds # like the poetry authors don't consider this a bug?) # @@ -211,6 +213,6 @@ def check_requirements(extra: Optional[str] = None) -> None: if deps_unfulfilled: for err in errors: - logging.error(err) + logger.error(err) raise DependencyException(deps_unfulfilled) diff --git a/synapse/util/constants.py b/synapse/util/constants.py new file mode 100644 index 0000000000..9986017147 --- /dev/null +++ b/synapse/util/constants.py @@ -0,0 +1,20 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +# Time-based constants. +# +# Laying these out incrementally, even if only some are required, helps with +# readability and catching bugs. +ONE_MINUTE_SECONDS = 60 +ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS diff --git a/synapse/util/daemonize.py b/synapse/util/daemonize.py index 52549f53c4..e653abff97 100644 --- a/synapse/util/daemonize.py +++ b/synapse/util/daemonize.py @@ -29,6 +29,11 @@ import sys from types import FrameType, TracebackType from typing import NoReturn, Optional, Type +from synapse.logging.context import ( + LoggingContext, + PreserveLoggingContext, +) + def daemonize_process(pid_file: str, logger: logging.Logger, chdir: str = "/") -> None: """daemonize the current process @@ -64,8 +69,14 @@ def daemonize_process(pid_file: str, logger: logging.Logger, chdir: str = "/") - pid_fh.write(old_pid) sys.exit(1) - # Fork, creating a new process for the child. - process_id = os.fork() + # Stop the existing context *before* we fork the process. Otherwise the cputime + # metrics get confused about the per-thread resource usage appearing to go backwards + # because we're comparing the resource usage from the original process to the forked + # process. `PreserveLoggingContext` already takes care of restarting the original + # context *after* the block. + with PreserveLoggingContext(): + # Fork, creating a new process for the child. + process_id = os.fork() if process_id != 0: # parent process: exit. @@ -133,16 +144,17 @@ def daemonize_process(pid_file: str, logger: logging.Logger, chdir: str = "/") - # write a log line on SIGTERM. def sigterm(signum: int, frame: Optional[FrameType]) -> NoReturn: - logger.warning("Caught signal %s. Stopping daemon." % signum) + logger.warning("Caught signal %s. Stopping daemon.", signum) sys.exit(0) signal.signal(signal.SIGTERM, sigterm) # Cleanup pid file at exit. def exit() -> None: - logger.warning("Stopping daemon.") - os.remove(pid_file) - sys.exit(0) + with LoggingContext("atexit"): + logger.warning("Stopping daemon.") + os.remove(pid_file) + sys.exit(0) atexit.register(exit) diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 95786bd3dd..f48ae3373c 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -58,7 +58,13 @@ class Distributor: model will do for today. """ - def __init__(self) -> None: + def __init__(self, server_name: str) -> None: + """ + Args: + server_name: The homeserver name of the server (used to label metrics) + (this should be `hs.hostname`). + """ + self.server_name = server_name self.signals: Dict[str, Signal] = {} self.pre_registration: Dict[str, List[Callable]] = {} @@ -91,7 +97,9 @@ class Distributor: if name not in self.signals: raise KeyError("%r does not have a signal named %s" % (self, name)) - run_as_background_process(name, self.signals[name].fire, *args, **kwargs) + run_as_background_process( + name, self.server_name, self.signals[name].fire, *args, **kwargs + ) P = ParamSpec("P") diff --git a/synapse/util/events.py b/synapse/util/events.py index ad9b946578..4808268702 100644 --- a/synapse/util/events.py +++ b/synapse/util/events.py @@ -13,6 +13,11 @@ # # +from typing import Any, List, Optional + +from synapse._pydantic_compat import Field, StrictStr, ValidationError, validator +from synapse.types import JsonDict +from synapse.util.pydantic_models import ParseModel from synapse.util.stringutils import random_string @@ -27,3 +32,100 @@ def generate_fake_event_id() -> str: A string intended to look like an event ID, but with no actual meaning. """ return "$" + random_string(43) + + +class MTextRepresentation(ParseModel): + """ + See `TextualRepresentation` in the Matrix specification. + """ + + body: StrictStr + mimetype: Optional[StrictStr] + + +class MTopic(ParseModel): + """ + `m.room.topic` -> `content` -> `m.topic` + + Textual representation of the room topic in different mimetypes. Added in Matrix v1.15. + + See `TopicContentBlock` in the Matrix specification. + """ + + m_text: Optional[List[MTextRepresentation]] = Field(alias="m.text") + """ + An ordered array of textual representations in different mimetypes. + """ + + # Because "Receivers SHOULD use the first representation in the array that they + # understand.", we ignore invalid representations in the `m.text` field and use + # what we can. + @validator("m_text", pre=True) + def ignore_invalid_representations( + cls, m_text: Any + ) -> Optional[List[MTextRepresentation]]: + if not isinstance(m_text, list): + raise ValueError("m.text must be a list") + representations = [] + for element in m_text: + try: + representations.append(MTextRepresentation.parse_obj(element)) + except ValidationError: + continue + return representations + + +class TopicContent(ParseModel): + """ + Represents the `content` field of an `m.room.topic` event + """ + + topic: StrictStr + """ + The topic in plain text. + """ + + m_topic: Optional[MTopic] = Field(alias="m.topic") + """ + Textual representation of the room topic in different mimetypes. + """ + + # We ignore invalid `m.topic` fields as we can always fall back to the plain-text + # `topic` field. + @validator("m_topic", pre=True) + def ignore_invalid_m_topic(cls, m_topic: Any) -> Optional[MTopic]: + try: + return MTopic.parse_obj(m_topic) + except ValidationError: + return None + + +def get_plain_text_topic_from_event_content(content: JsonDict) -> Optional[str]: + """ + Given the `content` of an `m.room.topic` event, returns the plain-text topic + representation. Prefers pulling plain-text from the newer `m.topic` field if + available with a fallback to `topic`. + + Args: + content: The `content` field of an `m.room.topic` event. + + Returns: + A string representing the plain text topic. + """ + + try: + topic_content = TopicContent.parse_obj(content) + except ValidationError: + return None + + # Find the first `text/plain` topic ("Receivers SHOULD use the first + # representationin the array that they understand.") + if topic_content.m_topic and topic_content.m_topic.m_text: + for representation in topic_content.m_topic.m_text: + # The mimetype property defaults to `text/plain` if omitted. + if not representation.mimetype or representation.mimetype == "text/plain": + return representation.body + + # Fallback to the plain-old `topic` field if there isn't any `text/plain` topic + # representation available. + return topic_content.topic diff --git a/synapse/util/gai_resolver.py b/synapse/util/gai_resolver.py index fecf829ade..3c7a966e87 100644 --- a/synapse/util/gai_resolver.py +++ b/synapse/util/gai_resolver.py @@ -97,7 +97,7 @@ _GETADDRINFO_RESULT = List[ SocketKind, int, str, - Union[Tuple[str, int], Tuple[str, int, int, int]], + Union[Tuple[str, int], Tuple[str, int, int, int], Tuple[int, bytes]], ] ] diff --git a/synapse/util/iterutils.py b/synapse/util/iterutils.py index b73f690b88..0a6a30aab2 100644 --- a/synapse/util/iterutils.py +++ b/synapse/util/iterutils.py @@ -30,14 +30,13 @@ from typing import ( Iterator, List, Mapping, + Protocol, Set, Sized, Tuple, TypeVar, ) -from typing_extensions import Protocol - T = TypeVar("T") S = TypeVar("S", bound="_SelfSlice") @@ -115,7 +114,7 @@ def sorted_topologically( # This is implemented by Kahn's algorithm. - degree_map = {node: 0 for node in nodes} + degree_map = dict.fromkeys(nodes, 0) reverse_graph: Dict[T, Set[T]] = {} for node, edges in graph.items(): @@ -165,7 +164,7 @@ def sorted_topologically_batched( persisted. """ - degree_map = {node: 0 for node in nodes} + degree_map = dict.fromkeys(nodes, 0) reverse_graph: Dict[T, Set[T]] = {} for node, edges in graph.items(): diff --git a/synapse/util/macaroons.py b/synapse/util/macaroons.py index 84ae226207..6fa15543ec 100644 --- a/synapse/util/macaroons.py +++ b/synapse/util/macaroons.py @@ -22,12 +22,11 @@ """Utilities for manipulating macaroons""" -from typing import Callable, Optional +from typing import Callable, Literal, Optional import attr import pymacaroons from pymacaroons.exceptions import MacaroonVerificationFailedException -from typing_extensions import Literal from synapse.util import Clock, stringutils diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 020618598c..608a4d4848 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -22,69 +22,119 @@ import logging from functools import wraps from types import TracebackType -from typing import Awaitable, Callable, Dict, Generator, Optional, Type, TypeVar +from typing import ( + Awaitable, + Callable, + Dict, + Generator, + Optional, + Protocol, + Type, + TypeVar, +) from prometheus_client import CollectorRegistry, Counter, Metric -from typing_extensions import Concatenate, ParamSpec, Protocol +from typing_extensions import Concatenate, ParamSpec from synapse.logging.context import ( ContextResourceUsage, LoggingContext, current_context, ) -from synapse.metrics import InFlightGauge +from synapse.metrics import SERVER_NAME_LABEL, InFlightGauge from synapse.util import Clock logger = logging.getLogger(__name__) -block_counter = Counter("synapse_util_metrics_block_count", "", ["block_name"]) +# Metrics to see the number of and how much time is spend in various blocks of code. +# +block_counter = Counter( + "synapse_util_metrics_block_count", + documentation="The number of times this block has been called.", + labelnames=["block_name", SERVER_NAME_LABEL], +) +"""The number of times this block has been called.""" -block_timer = Counter("synapse_util_metrics_block_time_seconds", "", ["block_name"]) +block_timer = Counter( + "synapse_util_metrics_block_time_seconds", + documentation="The cumulative time spent executing this block across all calls, in seconds.", + labelnames=["block_name", SERVER_NAME_LABEL], +) +"""The cumulative time spent executing this block across all calls, in seconds.""" block_ru_utime = Counter( - "synapse_util_metrics_block_ru_utime_seconds", "", ["block_name"] + "synapse_util_metrics_block_ru_utime_seconds", + documentation="Resource usage: user CPU time in seconds used in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Resource usage: user CPU time in seconds used in this block""" block_ru_stime = Counter( - "synapse_util_metrics_block_ru_stime_seconds", "", ["block_name"] + "synapse_util_metrics_block_ru_stime_seconds", + documentation="Resource usage: system CPU time in seconds used in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Resource usage: system CPU time in seconds used in this block""" block_db_txn_count = Counter( - "synapse_util_metrics_block_db_txn_count", "", ["block_name"] + "synapse_util_metrics_block_db_txn_count", + documentation="Number of database transactions completed in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Number of database transactions completed in this block""" # seconds spent waiting for db txns, excluding scheduling time, in this block block_db_txn_duration = Counter( - "synapse_util_metrics_block_db_txn_duration_seconds", "", ["block_name"] + "synapse_util_metrics_block_db_txn_duration_seconds", + documentation="Seconds spent waiting for database txns, excluding scheduling time, in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Seconds spent waiting for database txns, excluding scheduling time, in this block""" # seconds spent waiting for a db connection, in this block block_db_sched_duration = Counter( - "synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"] + "synapse_util_metrics_block_db_sched_duration_seconds", + documentation="Seconds spent waiting for a db connection, in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Seconds spent waiting for a db connection, in this block""" # This is dynamically created in InFlightGauge.__init__. -class _InFlightMetric(Protocol): +class _BlockInFlightMetric(Protocol): + """ + Sub-metrics used for the `InFlightGauge` for blocks. + """ + real_time_max: float + """The longest observed duration of any single execution of this block, in seconds.""" real_time_sum: float + """The cumulative time spent executing this block across all calls, in seconds.""" -# Tracks the number of blocks currently active -in_flight: InFlightGauge[_InFlightMetric] = InFlightGauge( +in_flight: InFlightGauge[_BlockInFlightMetric] = InFlightGauge( "synapse_util_metrics_block_in_flight", - "", - labels=["block_name"], + desc="Tracks the number of blocks currently active", + labels=["block_name", SERVER_NAME_LABEL], + # Matches the fields in the `_BlockInFlightMetric` sub_metrics=["real_time_max", "real_time_sum"], ) - +"""Tracks the number of blocks currently active""" P = ParamSpec("P") R = TypeVar("R") -class HasClock(Protocol): +class HasClockAndServerName(Protocol): clock: Clock + """ + Used to measure functions + """ + server_name: str + """ + The homeserver name that this Measure is associated with (used to label the metric) + (`hs.hostname`). + """ def measure_func( @@ -92,8 +142,9 @@ def measure_func( ) -> Callable[[Callable[P, Awaitable[R]]], Callable[P, Awaitable[R]]]: """Decorate an async method with a `Measure` context manager. - The Measure is created using `self.clock`; it should only be used to decorate - methods in classes defining an instance-level `clock` attribute. + The Measure is created using `self.clock` and `self.server_name; it should only be + used to decorate methods in classes defining an instance-level `clock` and + `server_name` attributes. Usage: @@ -107,16 +158,21 @@ def measure_func( with Measure(...): ... + Args: + name: The name of the metric to report (the block name) (used to label the + metric). Defaults to the name of the decorated function. """ def wrapper( - func: Callable[Concatenate[HasClock, P], Awaitable[R]], + func: Callable[Concatenate[HasClockAndServerName, P], Awaitable[R]], ) -> Callable[P, Awaitable[R]]: block_name = func.__name__ if name is None else name @wraps(func) - async def measured_func(self: HasClock, *args: P.args, **kwargs: P.kwargs) -> R: - with Measure(self.clock, block_name): + async def measured_func( + self: HasClockAndServerName, *args: P.args, **kwargs: P.kwargs + ) -> R: + with Measure(self.clock, name=block_name, server_name=self.server_name): r = await func(self, *args, **kwargs) return r @@ -133,19 +189,24 @@ class Measure: __slots__ = [ "clock", "name", + "server_name", "_logging_context", "start", ] - def __init__(self, clock: Clock, name: str) -> None: + def __init__(self, clock: Clock, *, name: str, server_name: str) -> None: """ Args: clock: An object with a "time()" method, which returns the current time in seconds. - name: The name of the metric to report. + name: The name of the metric to report (the block name) (used to label the + metric). + server_name: The homeserver name that this Measure is associated with (used to + label the metric) (`hs.hostname`). """ self.clock = clock self.name = name + self.server_name = server_name curr_context = current_context() if not curr_context: logger.warning( @@ -165,7 +226,7 @@ class Measure: self.start = self.clock.time() self._logging_context.__enter__() - in_flight.register((self.name,), self._update_in_flight) + in_flight.register((self.name, self.server_name), self._update_in_flight) logger.debug("Entering block %s", self.name) @@ -185,19 +246,20 @@ class Measure: duration = self.clock.time() - self.start usage = self.get_resource_usage() - in_flight.unregister((self.name,), self._update_in_flight) + in_flight.unregister((self.name, self.server_name), self._update_in_flight) self._logging_context.__exit__(exc_type, exc_val, exc_tb) try: - block_counter.labels(self.name).inc() - block_timer.labels(self.name).inc(duration) - block_ru_utime.labels(self.name).inc(usage.ru_utime) - block_ru_stime.labels(self.name).inc(usage.ru_stime) - block_db_txn_count.labels(self.name).inc(usage.db_txn_count) - block_db_txn_duration.labels(self.name).inc(usage.db_txn_duration_sec) - block_db_sched_duration.labels(self.name).inc(usage.db_sched_duration_sec) - except ValueError: - logger.warning("Failed to save metrics! Usage: %s", usage) + labels = {"block_name": self.name, SERVER_NAME_LABEL: self.server_name} + block_counter.labels(**labels).inc() + block_timer.labels(**labels).inc(duration) + block_ru_utime.labels(**labels).inc(usage.ru_utime) + block_ru_stime.labels(**labels).inc(usage.ru_stime) + block_db_txn_count.labels(**labels).inc(usage.db_txn_count) + block_db_txn_duration.labels(**labels).inc(usage.db_txn_duration_sec) + block_db_sched_duration.labels(**labels).inc(usage.db_sched_duration_sec) + except ValueError as exc: + logger.warning("Failed to save metrics! Usage: %s Error: %s", usage, exc) def get_resource_usage(self) -> ContextResourceUsage: """Get the resources used within this Measure block @@ -206,7 +268,7 @@ class Measure: """ return self._logging_context.get_resource_usage() - def _update_in_flight(self, metrics: _InFlightMetric) -> None: + def _update_in_flight(self, metrics: _BlockInFlightMetric) -> None: """Gets called when processing in flight metrics""" assert self.start is not None duration = self.clock.time() - self.start diff --git a/synapse/util/msisdn.py b/synapse/util/msisdn.py index b6a784f0bc..dce8da5e18 100644 --- a/synapse/util/msisdn.py +++ b/synapse/util/msisdn.py @@ -21,7 +21,7 @@ import phonenumbers -from synapse.api.errors import SynapseError +from synapse.api.errors import Codes, SynapseError def phone_number_to_msisdn(country: str, number: str) -> str: @@ -45,7 +45,7 @@ def phone_number_to_msisdn(country: str, number: str) -> str: try: phoneNumber = phonenumbers.parse(number, country) except phonenumbers.NumberParseException: - raise SynapseError(400, "Unable to parse phone number") + raise SynapseError(400, "Unable to parse phone number", Codes.INVALID_PARAM) return phonenumbers.format_number(phoneNumber, phonenumbers.PhoneNumberFormat.E164)[ 1: ] diff --git a/synapse/util/patch_inline_callbacks.py b/synapse/util/patch_inline_callbacks.py index 56bdf451da..c776ad65b3 100644 --- a/synapse/util/patch_inline_callbacks.py +++ b/synapse/util/patch_inline_callbacks.py @@ -20,6 +20,7 @@ import functools import sys +from types import GeneratorType from typing import Any, Callable, Generator, List, TypeVar, cast from typing_extensions import ParamSpec @@ -151,6 +152,12 @@ def _check_yield_points( ) -> Generator["Deferred[object]", object, T]: gen = f(*args, **kwargs) + # We only patch if we have a native generator function, as we rely on + # `gen.gi_frame`. + if not isinstance(gen, GeneratorType): + ret = yield from gen + return ret + last_yield_line_no = gen.gi_frame.f_lineno result: Any = None while True: @@ -162,7 +169,7 @@ def _check_yield_points( d = result.throwExceptionIntoGenerator(gen) else: d = gen.send(result) - except (StopIteration, defer._DefGen_Return) as e: + except StopIteration as e: if current_context() != expected_context: # This happens when the context is lost sometime *after* the # final yield and returning. E.g. we forgot to yield on a @@ -183,7 +190,7 @@ def _check_yield_points( ) ) changes.append(err) - # The `StopIteration` or `_DefGen_Return` contains the return value from the + # The `StopIteration` contains the return value from the # generator. return cast(T, e.value) diff --git a/synapse/util/pydantic_models.py b/synapse/util/pydantic_models.py new file mode 100644 index 0000000000..4880709501 --- /dev/null +++ b/synapse/util/pydantic_models.py @@ -0,0 +1,83 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import re +from typing import Any, Callable, Generator + +from synapse._pydantic_compat import BaseModel, Extra, StrictStr +from synapse.types import EventID + + +class ParseModel(BaseModel): + """A custom version of Pydantic's BaseModel which + + - ignores unknown fields and + - does not allow fields to be overwritten after construction, + + but otherwise uses Pydantic's default behaviour. + + For now, ignore unknown fields. In the future, we could change this so that unknown + config values cause a ValidationError, provided the error messages are meaningful to + server operators. + + Subclassing in this way is recommended by + https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally + """ + + class Config: + # By default, ignore fields that we don't recognise. + extra = Extra.ignore + # By default, don't allow fields to be reassigned after parsing. + allow_mutation = False + + +class AnyEventId(StrictStr): + """ + A validator for strings that need to be an Event ID. + + Accepts any valid grammar of Event ID from any room version. + """ + + EVENT_ID_HASH_ROOM_VERSION_3_PLUS = re.compile( + r"^([a-zA-Z0-9-_]{43}|[a-zA-Z0-9+/]{43})$" + ) + + @classmethod + def __get_validators__(cls) -> Generator[Callable[..., Any], Any, Any]: + yield from super().__get_validators__() # type: ignore + yield cls.validate_event_id + + @classmethod + def validate_event_id(cls, value: str) -> str: + if not value.startswith("$"): + raise ValueError("Event ID must start with `$`") + + if ":" in value: + # Room versions 1 and 2 + EventID.from_string(value) # throws on fail + else: + # Room versions 3+: event ID is $ + a base64 sha256 hash + # Room version 3 is base64, 4+ are base64Url + # In both cases, the base64 is unpadded. + # refs: + # - https://spec.matrix.org/v1.15/rooms/v3/ e.g. $acR1l0raoZnm60CBwAVgqbZqoO/mYU81xysh1u7XcJk + # - https://spec.matrix.org/v1.15/rooms/v4/ e.g. $Rqnc-F-dvnEYJTyHq_iKxU2bZ1CI92-kuZq3a5lr5Zg + b64_hash = value[1:] + if cls.EVENT_ID_HASH_ROOM_VERSION_3_PLUS.fullmatch(b64_hash) is None: + raise ValueError( + "Event ID must either have a domain part or be a valid hash" + ) + + return value diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 3f067b792c..88edc07161 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -52,7 +52,7 @@ from synapse.logging.context import ( run_in_background, ) from synapse.logging.opentracing import start_active_span -from synapse.metrics import Histogram, LaterGauge +from synapse.metrics import SERVER_NAME_LABEL, Histogram, LaterGauge from synapse.util import Clock if typing.TYPE_CHECKING: @@ -65,17 +65,17 @@ logger = logging.getLogger(__name__) rate_limit_sleep_counter = Counter( "synapse_rate_limit_sleep", "Number of requests slept by the rate limiter", - ["rate_limiter_name"], + labelnames=["rate_limiter_name", SERVER_NAME_LABEL], ) rate_limit_reject_counter = Counter( "synapse_rate_limit_reject", "Number of requests rejected by the rate limiter", - ["rate_limiter_name"], + labelnames=["rate_limiter_name", SERVER_NAME_LABEL], ) queue_wait_timer = Histogram( "synapse_rate_limit_queue_wait_time_seconds", "Amount of time spent waiting for the rate limiter to let our request through.", - ["rate_limiter_name"], + labelnames=["rate_limiter_name", SERVER_NAME_LABEL], buckets=( 0.005, 0.01, @@ -119,7 +119,10 @@ def _get_counts_from_rate_limiter_instance( # Only track metrics if they provided a `metrics_name` to # differentiate this instance of the rate limiter. if rate_limiter_instance.metrics_name: - key = (rate_limiter_instance.metrics_name,) + key = ( + rate_limiter_instance.metrics_name, + rate_limiter_instance.our_server_name, + ) counts[key] = count_func(rate_limiter_instance) return counts @@ -128,22 +131,28 @@ def _get_counts_from_rate_limiter_instance( # We track the number of affected hosts per time-period so we can # differentiate one really noisy homeserver from a general # ratelimit tuning problem across the federation. -LaterGauge( - "synapse_rate_limit_sleep_affected_hosts", - "Number of hosts that had requests put to sleep", - ["rate_limiter_name"], - lambda: _get_counts_from_rate_limiter_instance( +sleep_affected_hosts_gauge = LaterGauge( + name="synapse_rate_limit_sleep_affected_hosts", + desc="Number of hosts that had requests put to sleep", + labelnames=["rate_limiter_name", SERVER_NAME_LABEL], +) +sleep_affected_hosts_gauge.register_hook( + homeserver_instance_id=None, + hook=lambda: _get_counts_from_rate_limiter_instance( lambda rate_limiter_instance: sum( ratelimiter.should_sleep() for ratelimiter in rate_limiter_instance.ratelimiters.values() ) ), ) -LaterGauge( - "synapse_rate_limit_reject_affected_hosts", - "Number of hosts that had requests rejected", - ["rate_limiter_name"], - lambda: _get_counts_from_rate_limiter_instance( +reject_affected_hosts_gauge = LaterGauge( + name="synapse_rate_limit_reject_affected_hosts", + desc="Number of hosts that had requests rejected", + labelnames=["rate_limiter_name", SERVER_NAME_LABEL], +) +reject_affected_hosts_gauge.register_hook( + homeserver_instance_id=None, + hook=lambda: _get_counts_from_rate_limiter_instance( lambda rate_limiter_instance: sum( ratelimiter.should_reject() for ratelimiter in rate_limiter_instance.ratelimiters.values() @@ -157,6 +166,7 @@ class FederationRateLimiter: def __init__( self, + our_server_name: str, clock: Clock, config: FederationRatelimitSettings, metrics_name: Optional[str] = None, @@ -170,11 +180,15 @@ class FederationRateLimiter: for this rate limiter. """ + self.our_server_name = our_server_name self.metrics_name = metrics_name def new_limiter() -> "_PerHostRatelimiter": return _PerHostRatelimiter( - clock=clock, config=config, metrics_name=metrics_name + our_server_name=our_server_name, + clock=clock, + config=config, + metrics_name=metrics_name, ) self.ratelimiters: DefaultDict[str, "_PerHostRatelimiter"] = ( @@ -205,6 +219,7 @@ class FederationRateLimiter: class _PerHostRatelimiter: def __init__( self, + our_server_name: str, clock: Clock, config: FederationRatelimitSettings, metrics_name: Optional[str] = None, @@ -218,6 +233,7 @@ class _PerHostRatelimiter: for this rate limiter. from the rest in the metrics """ + self.our_server_name = our_server_name self.clock = clock self.metrics_name = metrics_name @@ -279,7 +295,10 @@ class _PerHostRatelimiter: async def _on_enter_with_tracing(self, request_id: object) -> None: maybe_metrics_cm: ContextManager = contextlib.nullcontext() if self.metrics_name: - maybe_metrics_cm = queue_wait_timer.labels(self.metrics_name).time() + maybe_metrics_cm = queue_wait_timer.labels( + rate_limiter_name=self.metrics_name, + **{SERVER_NAME_LABEL: self.our_server_name}, + ).time() with start_active_span("ratelimit wait"), maybe_metrics_cm: await self._on_enter(request_id) @@ -296,7 +315,10 @@ class _PerHostRatelimiter: if self.should_reject(): logger.debug("Ratelimiter(%s): rejecting request", self.host) if self.metrics_name: - rate_limit_reject_counter.labels(self.metrics_name).inc() + rate_limit_reject_counter.labels( + rate_limiter_name=self.metrics_name, + **{SERVER_NAME_LABEL: self.our_server_name}, + ).inc() raise LimitExceededError( limiter_name="rc_federation", retry_after_ms=int(self.window_size / self.sleep_limit), @@ -333,7 +355,10 @@ class _PerHostRatelimiter: self.sleep_sec, ) if self.metrics_name: - rate_limit_sleep_counter.labels(self.metrics_name).inc() + rate_limit_sleep_counter.labels( + rate_limiter_name=self.metrics_name, + **{SERVER_NAME_LABEL: self.our_server_name}, + ).inc() ret_defer = run_in_background(self.clock.sleep, self.sleep_sec) self.sleeping_requests.add(request_id) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 42be1c8d28..149df405b3 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -59,7 +59,9 @@ class NotRetryingDestination(Exception): async def get_retry_limiter( + *, destination: str, + our_server_name: str, clock: Clock, store: DataStore, ignore_backoff: bool = False, @@ -74,6 +76,7 @@ async def get_retry_limiter( Args: destination: name of homeserver + our_server_name: Our homeserver name (used to label metrics) (`hs.hostname`) clock: timing source store: datastore ignore_backoff: true to ignore the historical backoff data and @@ -82,7 +85,12 @@ async def get_retry_limiter( Example usage: try: - limiter = await get_retry_limiter(destination, clock, store) + limiter = await get_retry_limiter( + destination=destination, + our_server_name=self.server_name, + clock=clock, + store=store, + ) with limiter: response = await do_request() except NotRetryingDestination: @@ -114,11 +122,12 @@ async def get_retry_limiter( backoff_on_failure = not ignore_backoff return RetryDestinationLimiter( - destination, - clock, - store, - failure_ts, - retry_interval, + destination=destination, + our_server_name=our_server_name, + clock=clock, + store=store, + failure_ts=failure_ts, + retry_interval=retry_interval, backoff_on_failure=backoff_on_failure, **kwargs, ) @@ -151,7 +160,9 @@ async def filter_destinations_by_retry_limiter( class RetryDestinationLimiter: def __init__( self, + *, destination: str, + our_server_name: str, clock: Clock, store: DataStore, failure_ts: Optional[int], @@ -169,6 +180,7 @@ class RetryDestinationLimiter: Args: destination + our_server_name: Our homeserver name (used to label metrics) (`hs.hostname`) clock store failure_ts: when this destination started failing (in ms since @@ -184,6 +196,7 @@ class RetryDestinationLimiter: backoff_on_all_error_codes: Whether we should back off on any error code. """ + self.our_server_name = our_server_name self.clock = clock self.store = store self.destination = destination @@ -318,4 +331,6 @@ class RetryDestinationLimiter: logger.exception("Failed to store destination_retry_timings") # we deliberately do this in the background. - run_as_background_process("store_retry_timings", store_retry_timings) + run_as_background_process( + "store_retry_timings", self.our_server_name, store_retry_timings + ) diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index 13ff54b669..32b5bc00c9 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -43,6 +43,14 @@ CLIENT_SECRET_REGEX = re.compile(r"^[0-9a-zA-Z\.=_\-]+$") # MXC_REGEX = re.compile("^mxc://([^/]+)/([^/#?]+)$") +# https://spec.matrix.org/v1.13/appendices/#common-namespaced-identifier-grammar +# +# At least one character, less than or equal to 255 characters. Must start with +# a-z, the rest is a-z, 0-9, -, _, or .. +# +# This doesn't check anything about validity of namespaces. +NAMESPACED_GRAMMAR = re.compile(r"^[a-z][a-z0-9_.-]{0,254}$") + def random_string(length: int) -> str: """Generate a cryptographically secure string of random letters. @@ -68,6 +76,10 @@ def is_ascii(s: bytes) -> bool: return True +def is_namedspaced_grammar(s: str) -> bool: + return bool(NAMESPACED_GRAMMAR.match(s)) + + def assert_valid_client_secret(client_secret: str) -> None: """Validate that a given string matches the client_secret defined by the spec""" if ( diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 448960b297..0539989320 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -30,7 +30,7 @@ from synapse.logging.context import ( nested_logging_context, set_current_context, ) -from synapse.metrics import LaterGauge +from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import ( run_as_background_process, wrap_as_background_process, @@ -44,35 +44,52 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +running_tasks_gauge = LaterGauge( + name="synapse_scheduler_running_tasks", + desc="The number of concurrent running tasks handled by the TaskScheduler", + labelnames=[SERVER_NAME_LABEL], +) + + class TaskScheduler: """ - This is a simple task sheduler aimed at resumable tasks: usually we use `run_in_background` - to launch a background task, or Twisted `deferLater` if we want to do so later on. + This is a simple task scheduler designed for resumable tasks. Normally, + you'd use `run_in_background` to start a background task or Twisted's + `deferLater` if you want to run it later. - The problem with that is that the tasks will just stop and never be resumed if synapse - is stopped for whatever reason. + The issue is that these tasks stop completely and won't resume if Synapse is + shut down for any reason. - How this works: - - A function mapped to a named action should first be registered with `register_action`. - This function will be called when trying to resuming tasks after a synapse shutdown, - so this registration should happen when synapse is initialised, NOT right before scheduling - a task. - - A task can then be launched using this named action with `schedule_task`. A `params` dict - can be passed, and it will be available to the registered function when launched. This task - can be launch either now-ish, or later on by giving a `timestamp` parameter. + Here's how it works: - The function may call `update_task` at any time to update the `result` of the task, - and this can be used to resume the task at a specific point and/or to convey a result to - the code launching the task. - You can also specify the `result` (and/or an `error`) when returning from the function. + - Register an Action: First, you need to register a function to a named + action using `register_action`. This function will be called to resume tasks + after a Synapse shutdown. Make sure to register it when Synapse initializes, + not right before scheduling the task. - The reconciliation loop runs every minute, so this is not a precise scheduler. - There is a limit of 10 concurrent tasks, so tasks may be delayed if the pool is already - full. In this regard, please take great care that scheduled tasks can actually finished. - For now there is no mechanism to stop a running task if it is stuck. + - Schedule a Task: You can launch a task linked to the named action + using `schedule_task`. You can pass a `params` dictionary, which will be + passed to the registered function when it's executed. Tasks can be scheduled + to run either immediately or later by specifying a `timestamp`. - Tasks will be run on the worker specified with `run_background_tasks_on` config, - or the main one by default. + - Update Task: The function handling the task can call `update_task` at + any point to update the task's `result`. This lets you resume the task from + a specific point or pass results back to the code that scheduled it. When + the function completes, you can also return a `result` or an `error`. + + Things to keep in mind: + + - The reconciliation loop runs every minute, so this is not a high-precision + scheduler. + + - Only 10 tasks can run at the same time. If the pool is full, tasks may be + delayed. Make sure your scheduled tasks can actually finish. + + - Currently, there's no way to stop a task if it gets stuck. + + - Tasks will run on the worker defined by the `run_background_tasks_on` + setting in your configuration. If no worker is specified, they'll run on + the main one by default. """ # Precision of the scheduler, evaluation of tasks to run will only happen @@ -91,6 +108,9 @@ class TaskScheduler: def __init__(self, hs: "HomeServer"): self._hs = hs + self.server_name = ( + hs.hostname + ) # nb must be called this for @wrap_as_background_process self._store = hs.get_datastores().main self._clock = hs.get_clock() self._running_tasks: Set[str] = set() @@ -117,11 +137,9 @@ class TaskScheduler: TaskScheduler.SCHEDULE_INTERVAL_MS, ) - LaterGauge( - "synapse_scheduler_running_tasks", - "The number of concurrent running tasks handled by the TaskScheduler", - labels=None, - caller=lambda: len(self._running_tasks), + running_tasks_gauge.register_hook( + homeserver_instance_id=hs.get_instance_id(), + hook=lambda: {(self.server_name,): len(self._running_tasks)}, ) def register_action( @@ -157,7 +175,7 @@ class TaskScheduler: params: Optional[JsonMapping] = None, ) -> str: """Schedule a new potentially resumable task. A function matching the specified - `action` should have be registered with `register_action` before the task is run. + `action` should've been registered with `register_action` before the task is run. Args: action: the name of a previously registered action @@ -174,9 +192,10 @@ class TaskScheduler: The id of the scheduled task """ status = TaskStatus.SCHEDULED + start_now = False if timestamp is None or timestamp < self._clock.time_msec(): timestamp = self._clock.time_msec() - status = TaskStatus.ACTIVE + start_now = True task = ScheduledTask( random_string(16), @@ -190,9 +209,11 @@ class TaskScheduler: ) await self._store.insert_scheduled_task(task) - if status == TaskStatus.ACTIVE: + # If the task is ready to run immediately, run the scheduling algorithm now + # rather than waiting + if start_now: if self._run_background_tasks: - await self._launch_task(task) + self._launch_scheduled_tasks() else: self._hs.get_replication_command_handler().send_new_active_task(task.id) @@ -207,15 +228,15 @@ class TaskScheduler: result: Optional[JsonMapping] = None, error: Optional[str] = None, ) -> bool: - """Update some task associated values. This is exposed publicly so it can - be used inside task functions, mainly to update the result and be able to - resume a task at a specific step after a restart of synapse. + """Update some task-associated values. This is exposed publicly so it can + be used inside task functions, mainly to update the result or resume + a task at a specific step after a restart of synapse. It can also be used to stage a task, by setting the `status` to `SCHEDULED` with a new timestamp. - The `status` can only be set to `ACTIVE` or `SCHEDULED`, `COMPLETE` and `FAILED` - are terminal status and can only be set by returning it in the function. + The `status` can only be set to `ACTIVE` or `SCHEDULED`. `COMPLETE` and `FAILED` + are terminal statuses and can only be set by returning them from the function. Args: id: the id of the task to update @@ -223,6 +244,12 @@ class TaskScheduler: status: the new `TaskStatus` of the task result: the new result of the task error: the new error of the task + + Returns: + True if the update was successful, False otherwise. + + Raises: + Exception: If a status other than `ACTIVE` or `SCHEDULED` was passed. """ if status == TaskStatus.COMPLETE or status == TaskStatus.FAILED: raise Exception( @@ -260,9 +287,9 @@ class TaskScheduler: max_timestamp: Optional[int] = None, limit: Optional[int] = None, ) -> List[ScheduledTask]: - """Get a list of tasks. Returns all the tasks if no args is provided. + """Get a list of tasks. Returns all the tasks if no args are provided. - If an arg is `None` all tasks matching the other args will be selected. + If an arg is `None`, all tasks matching the other args will be selected. If an arg is an empty list, the corresponding value of the task needs to be `None` to be selected. @@ -274,8 +301,8 @@ class TaskScheduler: a timestamp inferior to the specified one limit: Only return `limit` number of rows if set. - Returns - A list of `ScheduledTask`, ordered by increasing timestamps + Returns: + A list of `ScheduledTask`, ordered by increasing timestamps. """ return await self._store.get_scheduled_tasks( actions=actions, @@ -300,23 +327,13 @@ class TaskScheduler: raise Exception(f"Task {id} is currently ACTIVE and can't be deleted") await self._store.delete_scheduled_task(id) - def launch_task_by_id(self, id: str) -> None: - """Try launching the task with the given ID.""" - # Don't bother trying to launch new tasks if we're already at capacity. - if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: - return + def on_new_task(self, task_id: str) -> None: + """Handle a notification that a new ready-to-run task has been added to the queue""" + # Just run the scheduler + self._launch_scheduled_tasks() - run_as_background_process("launch_task_by_id", self._launch_task_by_id, id) - - async def _launch_task_by_id(self, id: str) -> None: - """Helper async function for `launch_task_by_id`.""" - task = await self.get_task(id) - if task: - await self._launch_task(task) - - @wrap_as_background_process("launch_scheduled_tasks") - async def _launch_scheduled_tasks(self) -> None: - """Retrieve and launch scheduled tasks that should be running at that time.""" + def _launch_scheduled_tasks(self) -> None: + """Retrieve and launch scheduled tasks that should be running at this time.""" # Don't bother trying to launch new tasks if we're already at capacity. if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: return @@ -326,20 +343,26 @@ class TaskScheduler: self._launching_new_tasks = True - try: - for task in await self.get_tasks( - statuses=[TaskStatus.ACTIVE], limit=self.MAX_CONCURRENT_RUNNING_TASKS - ): - await self._launch_task(task) - for task in await self.get_tasks( - statuses=[TaskStatus.SCHEDULED], - max_timestamp=self._clock.time_msec(), - limit=self.MAX_CONCURRENT_RUNNING_TASKS, - ): - await self._launch_task(task) + async def inner() -> None: + try: + for task in await self.get_tasks( + statuses=[TaskStatus.ACTIVE], + limit=self.MAX_CONCURRENT_RUNNING_TASKS, + ): + # _launch_task will ignore tasks that we're already running, and + # will also do nothing if we're already at the maximum capacity. + await self._launch_task(task) + for task in await self.get_tasks( + statuses=[TaskStatus.SCHEDULED], + max_timestamp=self._clock.time_msec(), + limit=self.MAX_CONCURRENT_RUNNING_TASKS, + ): + await self._launch_task(task) - finally: - self._launching_new_tasks = False + finally: + self._launching_new_tasks = False + + run_as_background_process("launch_scheduled_tasks", self.server_name, inner) @wrap_as_background_process("clean_scheduled_tasks") async def _clean_scheduled_tasks(self) -> None: @@ -425,7 +448,8 @@ class TaskScheduler: except Exception: f = Failure() logger.error( - f"scheduled task {task.id} failed", + "scheduled task %s failed", + task.id, exc_info=(f.type, f.value, f.getTracebackObject()), ) status = TaskStatus.FAILED @@ -458,8 +482,10 @@ class TaskScheduler: self._clock.time_msec() > task.timestamp + TaskScheduler.LAST_UPDATE_BEFORE_WARNING_MS ): - logger.warn( - f"Task {task.id} (action {task.action}) has seen no update for more than 24h and may be stuck" + logger.warning( + "Task %s (action %s) has seen no update for more than 24h and may be stuck", + task.id, + task.action, ) if task.id in self._running_tasks: @@ -467,4 +493,4 @@ class TaskScheduler: self._running_tasks.add(task.id) await self.update_task(task.id, status=TaskStatus.ACTIVE) - run_as_background_process(f"task-{task.action}", wrapper) + run_as_background_process(f"task-{task.action}", self.server_name, wrapper) diff --git a/synapse/visibility.py b/synapse/visibility.py index dc7b6e4065..d460d8f4c2 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -48,7 +48,12 @@ from synapse.logging.opentracing import trace from synapse.storage.controllers import StorageControllers from synapse.storage.databases.main import DataStore from synapse.synapse_rust.events import event_visible_to_server -from synapse.types import RetentionPolicy, StateMap, StrCollection, get_domain_from_id +from synapse.types import ( + RetentionPolicy, + StateMap, + StrCollection, + get_domain_from_id, +) from synapse.types.state import StateFilter from synapse.util import Clock @@ -106,9 +111,32 @@ async def filter_events_for_client( of `user_id` at each event. """ # Filter out events that have been soft failed so that we don't relay them - # to clients. - events_before_filtering = events + # to clients, unless they're a server admin and want that to happen. + # + # We copy the events list to guarantee any modifications we make will only + # happen within the function. + events_before_filtering = events.copy() + # Default case is to *exclude* soft-failed events events = [e for e in events if not e.internal_metadata.is_soft_failed()] + client_config = await storage.main.get_admin_client_config_for_user(user_id) + if filter_send_to_client and await storage.main.is_server_admin(user_id): + if client_config.return_soft_failed_events: + # The user has requested that all events be included, so do that. + # We copy the list for mutation safety. + events = events_before_filtering.copy() + elif client_config.return_policy_server_spammy_events: + # Include events that were soft failed by a policy server (marked spammy), + # but exclude all other soft failed events. We also want to include all + # not-soft-failed events, per usual operation. + events = [ + e + for e in events_before_filtering + if not e.internal_metadata.is_soft_failed() + or e.internal_metadata.policy_server_spammy + ] + # else - no change in behaviour; use default case + # else - no change in behaviour; use default case + if len(events_before_filtering) != len(events): if filtered_event_logger.isEnabledFor(logging.DEBUG): filtered_event_logger.debug( diff --git a/synmark/suites/lrucache.py b/synmark/suites/lrucache.py index 49d200c43b..d109441e55 100644 --- a/synmark/suites/lrucache.py +++ b/synmark/suites/lrucache.py @@ -29,7 +29,7 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: """ Benchmark `loops` number of insertions into LruCache without eviction. """ - cache: LruCache[int, bool] = LruCache(loops) + cache: LruCache[int, bool] = LruCache(max_size=loops) start = perf_counter() diff --git a/synmark/suites/lrucache_evict.py b/synmark/suites/lrucache_evict.py index 77061625a9..00cfdd0447 100644 --- a/synmark/suites/lrucache_evict.py +++ b/synmark/suites/lrucache_evict.py @@ -30,7 +30,7 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: Benchmark `loops` number of insertions into LruCache where half of them are evicted. """ - cache: LruCache[int, bool] = LruCache(loops // 2) + cache: LruCache[int, bool] = LruCache(max_size=loops // 2) start = perf_counter() diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index bd229cf7e9..b8fb21ab0d 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -23,7 +23,7 @@ from unittest.mock import AsyncMock, Mock import pymacaroons -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.auth.internal import InternalAuth from synapse.api.auth_blocking import AuthBlocking @@ -60,7 +60,7 @@ class AuthTestCase(unittest.HomeserverTestCase): # modify its config instead of the hs' self.auth_blocking = AuthBlocking(hs) - self.test_user = "@foo:bar" + self.test_user_id = UserID.from_string("@foo:bar") self.test_token = b"_test_token_" # this is overridden for the appservice tests @@ -71,7 +71,7 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_user_valid_token(self) -> None: user_info = TokenLookupResult( - user_id=self.test_user, token_id=5, device_id="device" + user_id=self.test_user_id.to_string(), token_id=5, device_id="device" ) self.store.get_user_by_access_token = AsyncMock(return_value=user_info) self.store.mark_access_token_as_used = AsyncMock(return_value=None) @@ -81,7 +81,7 @@ class AuthTestCase(unittest.HomeserverTestCase): request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(requester.user.to_string(), self.test_user) + self.assertEqual(requester.user, self.test_user_id) def test_get_user_by_req_user_bad_token(self) -> None: self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -96,7 +96,7 @@ class AuthTestCase(unittest.HomeserverTestCase): self.assertEqual(f.errcode, "M_UNKNOWN_TOKEN") def test_get_user_by_req_user_missing_token(self) -> None: - user_info = TokenLookupResult(user_id=self.test_user, token_id=5) + user_info = TokenLookupResult(user_id=self.test_user_id.to_string(), token_id=5) self.store.get_user_by_access_token = AsyncMock(return_value=user_info) request = Mock(args={}) @@ -109,7 +109,10 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_appservice_valid_token(self) -> None: app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -119,7 +122,7 @@ class AuthTestCase(unittest.HomeserverTestCase): request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(requester.user.to_string(), self.test_user) + self.assertEqual(requester.user, self.test_user_id) def test_get_user_by_req_appservice_valid_token_good_ip(self) -> None: from netaddr import IPSet @@ -127,7 +130,7 @@ class AuthTestCase(unittest.HomeserverTestCase): app_service = Mock( token="foobar", url="a_url", - sender=self.test_user, + sender=self.test_user_id.to_string(), ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -138,7 +141,7 @@ class AuthTestCase(unittest.HomeserverTestCase): request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(requester.user.to_string(), self.test_user) + self.assertEqual(requester.user, self.test_user_id) def test_get_user_by_req_appservice_valid_token_bad_ip(self) -> None: from netaddr import IPSet @@ -146,7 +149,7 @@ class AuthTestCase(unittest.HomeserverTestCase): app_service = Mock( token="foobar", url="a_url", - sender=self.test_user, + sender=self.test_user_id, ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -176,7 +179,7 @@ class AuthTestCase(unittest.HomeserverTestCase): self.assertEqual(f.errcode, "M_UNKNOWN_TOKEN") def test_get_user_by_req_appservice_missing_token(self) -> None: - app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + app_service = Mock(token="foobar", url="a_url", sender=self.test_user_id) self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -191,7 +194,10 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_appservice_valid_token_valid_user_id(self) -> None: masquerading_user_id = b"@doppelganger:matrix.org" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -215,7 +221,10 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_appservice_valid_token_bad_user_id(self) -> None: masquerading_user_id = b"@doppelganger:matrix.org" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=False) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -238,7 +247,10 @@ class AuthTestCase(unittest.HomeserverTestCase): masquerading_user_id = b"@doppelganger:matrix.org" masquerading_device_id = b"DOPPELDEVICE" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -270,7 +282,10 @@ class AuthTestCase(unittest.HomeserverTestCase): masquerading_user_id = b"@doppelganger:matrix.org" masquerading_device_id = b"NOT_A_REAL_DEVICE_ID" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -436,7 +451,7 @@ class AuthTestCase(unittest.HomeserverTestCase): namespaces={ "users": [{"regex": "@_appservice.*:sender", "exclusive": True}] }, - sender="@appservice:sender", + sender=UserID.from_string("@appservice:server"), ) requester = Requester( user=UserID.from_string("@appservice:server"), @@ -467,7 +482,7 @@ class AuthTestCase(unittest.HomeserverTestCase): namespaces={ "users": [{"regex": "@_appservice.*:sender", "exclusive": True}] }, - sender="@appservice:sender", + sender=UserID.from_string("@appservice:server"), ) requester = Requester( user=UserID.from_string("@appservice:server"), diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 743c52d969..8ad9a5a6f7 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -25,7 +25,7 @@ from unittest.mock import patch import jsonschema -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EduTypes, EventContentFields from synapse.api.errors import SynapseError diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py index a59e168db1..2e45d4e4d2 100644 --- a/tests/api/test_ratelimiting.py +++ b/tests/api/test_ratelimiting.py @@ -1,7 +1,11 @@ +from typing import Optional + from synapse.api.ratelimiting import LimitExceededError, Ratelimiter from synapse.appservice import ApplicationService from synapse.config.ratelimiting import RatelimitSettings -from synapse.types import create_requester +from synapse.module_api import RatelimitOverride +from synapse.module_api.callbacks.ratelimit_callbacks import RatelimitModuleApiCallbacks +from synapse.types import UserID, create_requester from tests import unittest @@ -36,7 +40,7 @@ class TestRatelimiter(unittest.HomeserverTestCase): token="fake_token", id="foo", rate_limited=True, - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), ) as_requester = create_requester("@user:example.com", app_service=appservice) @@ -72,7 +76,7 @@ class TestRatelimiter(unittest.HomeserverTestCase): token="fake_token", id="foo", rate_limited=False, - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), ) as_requester = create_requester("@user:example.com", app_service=appservice) @@ -220,9 +224,7 @@ class TestRatelimiter(unittest.HomeserverTestCase): self.assertIn("test_id_1", limiter.actions) - self.get_success_or_raise( - limiter.can_do_action(None, key="test_id_2", _time_now_s=10) - ) + self.reactor.advance(60) self.assertNotIn("test_id_1", limiter.actions) @@ -442,3 +444,49 @@ class TestRatelimiter(unittest.HomeserverTestCase): limiter.can_do_action(requester=None, key="a", _time_now_s=20.0) ) self.assertTrue(success) + + def test_get_ratelimit_override_for_user_callback(self) -> None: + test_user_id = "@user:test" + test_limiter_name = "name" + callbacks = RatelimitModuleApiCallbacks(self.hs) + requester = create_requester(test_user_id) + limiter = Ratelimiter( + store=self.hs.get_datastores().main, + clock=self.clock, + cfg=RatelimitSettings( + test_limiter_name, + per_second=0.1, + burst_count=3, + ), + ratelimit_callbacks=callbacks, + ) + + # Observe four actions, exceeding the burst_count. + limiter.record_action(requester=requester, n_actions=4, _time_now_s=0.0) + + # We should be prevented from taking a new action now. + success, _ = self.get_success_or_raise( + limiter.can_do_action(requester=requester, _time_now_s=0.0) + ) + self.assertFalse(success) + + # Now register a callback that overrides the ratelimit for this user + # and limiter name. + async def get_ratelimit_override_for_user( + user_id: str, limiter_name: str + ) -> Optional[RatelimitOverride]: + if user_id == test_user_id: + return RatelimitOverride( + per_second=0.1, + burst_count=10, + ) + return None + + callbacks.register_callbacks( + get_ratelimit_override_for_user=get_ratelimit_override_for_user + ) + + success, _ = self.get_success_or_raise( + limiter.can_do_action(requester=requester, _time_now_s=0.0) + ) + self.assertTrue(success) diff --git a/tests/api/test_urls.py b/tests/api/test_urls.py new file mode 100644 index 0000000000..bb46008ad2 --- /dev/null +++ b/tests/api/test_urls.py @@ -0,0 +1,81 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +from twisted.internet.testing import MemoryReactor + +from synapse.api.urls import LoginSSORedirectURIBuilder +from synapse.server import HomeServer +from synapse.util import Clock + +from tests.unittest import HomeserverTestCase + +# a (valid) url with some annoying characters in. %3D is =, %26 is &, %2B is + +TRICKY_TEST_CLIENT_REDIRECT_URL = 'https://x?&q"+%3D%2B"="fö%26=o"' + + +class LoginSSORedirectURIBuilderTestCase(HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.login_sso_redirect_url_builder = LoginSSORedirectURIBuilder(hs.config) + + def test_no_idp_id(self) -> None: + self.assertEqual( + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id=None, client_redirect_url="http://example.com/redirect" + ), + "https://test/_matrix/client/v3/login/sso/redirect?redirectUrl=http%3A%2F%2Fexample.com%2Fredirect", + ) + + def test_explicit_idp_id(self) -> None: + self.assertEqual( + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="oidc-github", client_redirect_url="http://example.com/redirect" + ), + "https://test/_matrix/client/v3/login/sso/redirect/oidc-github?redirectUrl=http%3A%2F%2Fexample.com%2Fredirect", + ) + + def test_tricky_redirect_uri(self) -> None: + self.assertEqual( + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="oidc-github", + client_redirect_url=TRICKY_TEST_CLIENT_REDIRECT_URL, + ), + "https://test/_matrix/client/v3/login/sso/redirect/oidc-github?redirectUrl=https%3A%2F%2Fx%3F%3Cab+c%3E%26q%22%2B%253D%252B%22%3D%22f%C3%B6%2526%3Do%22", + ) + + def test_idp_id_with_slash_is_escaped(self) -> None: + """ + Test to make sure that we properly URL encode the IdP ID. + """ + self.assertEqual( + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="foo/bar", + client_redirect_url="http://example.com/redirect", + ), + "https://test/_matrix/client/v3/login/sso/redirect/foo%2Fbar?redirectUrl=http%3A%2F%2Fexample.com%2Fredirect", + ) + + def test_url_as_idp_id_is_escaped(self) -> None: + """ + Test to make sure that we properly URL encode the IdP ID. + + The IdP ID shouldn't be a URL. + """ + self.assertEqual( + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="http://should-not-be-url.com/", + client_redirect_url="http://example.com/redirect", + ), + "https://test/_matrix/client/v3/login/sso/redirect/http%3A%2F%2Fshould-not-be-url.com%2F?redirectUrl=http%3A%2F%2Fexample.com%2Fredirect", + ) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 47d590ecea..63cb5ff46f 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -22,7 +22,7 @@ from unittest.mock import Mock, patch from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.app.generic_worker import GenericWorkerServer from synapse.app.homeserver import SynapseHomeServer diff --git a/tests/appservice/test_api.py b/tests/appservice/test_api.py index 0f19736540..5eba6d20c8 100644 --- a/tests/appservice/test_api.py +++ b/tests/appservice/test_api.py @@ -21,11 +21,11 @@ from typing import Any, List, Mapping, Optional, Sequence, Union from unittest.mock import Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.appservice import ApplicationService from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util import Clock from tests import unittest @@ -41,7 +41,7 @@ class ApplicationServiceApiTestCase(unittest.HomeserverTestCase): self.api = hs.get_application_service_api() self.service = ApplicationService( id="unique_identifier", - sender="@as:test", + sender=UserID.from_string("@as:test"), url=URL, token="unused", hs_token=TOKEN, diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 3fa4426638..620c2b907b 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -25,6 +25,7 @@ from unittest.mock import AsyncMock, Mock from twisted.internet import defer from synapse.appservice import ApplicationService, Namespace +from synapse.types import UserID from tests import unittest @@ -37,7 +38,7 @@ class ApplicationServiceTestCase(unittest.TestCase): def setUp(self) -> None: self.service = ApplicationService( id="unique_identifier", - sender="@as:test", + sender=UserID.from_string("@as:test"), url="some_url", token="some_token", ) @@ -226,11 +227,11 @@ class ApplicationServiceTestCase(unittest.TestCase): @defer.inlineCallbacks def test_interested_in_self(self) -> Generator["defer.Deferred[Any]", object, None]: # make sure invites get through - self.service.sender = "@appservice:name" + self.service.sender = UserID.from_string("@appservice:name") self.service.namespaces[ApplicationService.NS_USERS].append(_regex("@irc_.*")) self.event.type = "m.room.member" self.event.content = {"membership": "invite"} - self.event.state_key = self.service.sender + self.event.state_key = self.service.sender.to_string() self.assertTrue( ( yield self.service.is_interested_in_event( diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index 730b00a9fb..11319bc52d 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -2,7 +2,7 @@ # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright 2015, 2016 OpenMarket Ltd -# Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2023, 2025 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -24,7 +24,7 @@ from unittest.mock import AsyncMock, Mock from typing_extensions import TypeAlias from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.appservice import ( ApplicationService, @@ -53,11 +53,24 @@ class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): self.clock = MockClock() self.store = Mock() self.as_api = Mock() + + self.hs = Mock( + spec_set=[ + "get_datastores", + "get_clock", + "get_application_service_api", + "hostname", + ] + ) + self.hs.get_clock.return_value = self.clock + self.hs.get_datastores.return_value = Mock( + main=self.store, + ) + self.hs.get_application_service_api.return_value = self.as_api + self.recoverer = Mock() self.recoverer_fn = Mock(return_value=self.recoverer) - self.txnctrl = _TransactionController( - clock=cast(Clock, self.clock), store=self.store, as_api=self.as_api - ) + self.txnctrl = _TransactionController(self.hs) self.txnctrl.RECOVERER_CLASS = self.recoverer_fn def test_single_service_up_txn_sent(self) -> None: @@ -163,6 +176,7 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.service = Mock() self.callback = AsyncMock() self.recoverer = _Recoverer( + server_name="test_server", clock=cast(Clock, self.clock), as_api=self.as_api, store=self.store, @@ -234,6 +248,41 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.assertEqual(1, txn.complete.call_count) self.callback.assert_called_once_with(self.recoverer) + def test_recover_force_retry(self) -> None: + txn = Mock() + txns = [txn, None] + pop_txn = False + + def take_txn( + *args: object, **kwargs: object + ) -> "defer.Deferred[Optional[Mock]]": + if pop_txn: + return defer.succeed(txns.pop(0)) + else: + return defer.succeed(txn) + + self.store.get_oldest_unsent_txn = Mock(side_effect=take_txn) + + # Start the recovery, and then fail the first attempt. + self.recoverer.recover() + self.assertEqual(0, self.store.get_oldest_unsent_txn.call_count) + txn.send = AsyncMock(return_value=False) + txn.complete = AsyncMock(return_value=None) + self.clock.advance_time(2) + self.assertEqual(1, txn.send.call_count) + self.assertEqual(0, txn.complete.call_count) + self.assertEqual(0, self.callback.call_count) + + # Now allow the send to succeed, and force a retry. + pop_txn = True # returns the txn the first time, then no more. + txn.send = AsyncMock(return_value=True) # successfully send the txn + self.recoverer.force_retry() + self.assertEqual(1, txn.send.call_count) # new mock reset call count + self.assertEqual(1, txn.complete.call_count) + + # Ensure we call the callback to say we're done! + self.callback.assert_called_once_with(self.recoverer) + # Corresponds to synapse.appservice.scheduler._TransactionController.send TxnCtrlArgs: TypeAlias = """ diff --git a/tests/config/test_api.py b/tests/config/test_api.py index 6773c9a277..e6cc3e21ed 100644 --- a/tests/config/test_api.py +++ b/tests/config/test_api.py @@ -3,6 +3,7 @@ from unittest import TestCase as StdlibTestCase import yaml from synapse.config import ConfigError +from synapse.config._base import RootConfig from synapse.config.api import ApiConfig from synapse.types.state import StateFilter @@ -19,7 +20,7 @@ DEFAULT_PREJOIN_STATE_PAIRS = { class TestRoomPrejoinState(StdlibTestCase): def read_config(self, source: str) -> ApiConfig: - config = ApiConfig() + config = ApiConfig(RootConfig()) config.read_config(yaml.safe_load(source)) return config diff --git a/tests/config/test_appservice.py b/tests/config/test_appservice.py index e3021b59d8..2572681224 100644 --- a/tests/config/test_appservice.py +++ b/tests/config/test_appservice.py @@ -19,6 +19,7 @@ # # +from synapse.config._base import RootConfig from synapse.config.appservice import AppServiceConfig, ConfigError from tests.unittest import TestCase @@ -36,12 +37,12 @@ class AppServiceConfigTest(TestCase): ["foo", "bar", False], ]: with self.assertRaises(ConfigError): - AppServiceConfig().read_config( + AppServiceConfig(RootConfig()).read_config( {"app_service_config_files": invalid_value} ) def test_valid_app_service_config_files(self) -> None: - AppServiceConfig().read_config({"app_service_config_files": []}) - AppServiceConfig().read_config( + AppServiceConfig(RootConfig()).read_config({"app_service_config_files": []}) + AppServiceConfig(RootConfig()).read_config( {"app_service_config_files": ["/not/a/real/path", "/not/a/real/path/2"]} ) diff --git a/tests/config/test_cache.py b/tests/config/test_cache.py index 631263b5ca..deb6bade46 100644 --- a/tests/config/test_cache.py +++ b/tests/config/test_cache.py @@ -19,6 +19,7 @@ # # +from synapse.config._base import RootConfig from synapse.config.cache import CacheConfig, add_resizable_cache from synapse.types import JsonDict from synapse.util.caches.lrucache import LruCache @@ -29,7 +30,7 @@ from tests.unittest import TestCase class CacheConfigTests(TestCase): def setUp(self) -> None: # Reset caches before each test since there's global state involved. - self.config = CacheConfig() + self.config = CacheConfig(RootConfig()) self.config.reset() def tearDown(self) -> None: @@ -74,7 +75,7 @@ class CacheConfigTests(TestCase): the default cache size in the interim, and then resized once the config is loaded. """ - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -95,7 +96,7 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 200) @@ -105,7 +106,7 @@ class CacheConfigTests(TestCase): the default cache size in the interim, and then resized to the new default cache size once the config is loaded. """ - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -125,7 +126,7 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 150) @@ -144,15 +145,15 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache_a: LruCache = LruCache(100) + cache_a: LruCache = LruCache(max_size=100) add_resizable_cache("*cache_a*", cache_resize_callback=cache_a.set_cache_factor) self.assertEqual(cache_a.max_size, 200) - cache_b: LruCache = LruCache(100) + cache_b: LruCache = LruCache(max_size=100) add_resizable_cache("*Cache_b*", cache_resize_callback=cache_b.set_cache_factor) self.assertEqual(cache_b.max_size, 300) - cache_c: LruCache = LruCache(100) + cache_c: LruCache = LruCache(max_size=100) add_resizable_cache("*cache_c*", cache_resize_callback=cache_c.set_cache_factor) self.assertEqual(cache_c.max_size, 200) diff --git a/tests/config/test_database.py b/tests/config/test_database.py index b46519f84a..3fa5fff2b2 100644 --- a/tests/config/test_database.py +++ b/tests/config/test_database.py @@ -20,6 +20,7 @@ import yaml +from synapse.config._base import RootConfig from synapse.config.database import DatabaseConfig from tests import unittest @@ -28,7 +29,9 @@ from tests import unittest class DatabaseConfigTestCase(unittest.TestCase): def test_database_configured_correctly(self) -> None: conf = yaml.safe_load( - DatabaseConfig().generate_config_section(data_dir_path="/data_dir_path") + DatabaseConfig(RootConfig()).generate_config_section( + data_dir_path="/data_dir_path" + ) ) expected_database_conf = { diff --git a/tests/config/test_load.py b/tests/config/test_load.py index c5dee06af5..b72365b6e3 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -21,6 +21,7 @@ # import tempfile from typing import Callable +from unittest import mock import yaml from parameterized import parameterized @@ -31,6 +32,11 @@ from synapse.config.homeserver import HomeServerConfig from tests.config.utils import ConfigFileTestCase +try: + import authlib +except ImportError: + authlib = None + try: import hiredis except ImportError: @@ -39,7 +45,7 @@ except ImportError: class ConfigLoadingFileTestCase(ConfigFileTestCase): def test_load_fails_if_server_name_missing(self) -> None: - self.generate_config_and_remove_lines_containing("server_name") + self.generate_config_and_remove_lines_containing(["server_name"]) with self.assertRaises(ConfigError): HomeServerConfig.load_config("", ["-c", self.config_file]) with self.assertRaises(ConfigError): @@ -76,7 +82,7 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): ) def test_load_succeeds_if_macaroon_secret_key_missing(self) -> None: - self.generate_config_and_remove_lines_containing("macaroon") + self.generate_config_and_remove_lines_containing(["macaroon"]) config1 = HomeServerConfig.load_config("", ["-c", self.config_file]) config2 = HomeServerConfig.load_config("", ["-c", self.config_file]) config3 = HomeServerConfig.load_or_generate_config("", ["-c", self.config_file]) @@ -111,7 +117,7 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): self.assertTrue(config3.registration.enable_registration) def test_stats_enabled(self) -> None: - self.generate_config_and_remove_lines_containing("enable_metrics") + self.generate_config_and_remove_lines_containing(["enable_metrics"]) self.add_lines_to_config(["enable_metrics: true"]) # The default Metrics Flags are off by default. @@ -131,6 +137,13 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): [ "turn_shared_secret_path: /does/not/exist", "registration_shared_secret_path: /does/not/exist", + "macaroon_secret_key_path: /does/not/exist", + "recaptcha_private_key_path: /does/not/exist", + "recaptcha_public_key_path: /does/not/exist", + "form_secret_path: /does/not/exist", + "worker_replication_secret_path: /does/not/exist", + "experimental_features:\n msc3861:\n client_secret_path: /does/not/exist", + "experimental_features:\n msc3861:\n admin_token_path: /does/not/exist", *["redis:\n enabled: true\n password_path: /does/not/exist"] * (hiredis is not None), ] @@ -146,16 +159,44 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): [ ( "turn_shared_secret_path: {}", - lambda c: c.voip.turn_shared_secret, + lambda c: c.voip.turn_shared_secret.encode("utf-8"), ), ( "registration_shared_secret_path: {}", - lambda c: c.registration.registration_shared_secret, + lambda c: c.registration.registration_shared_secret.encode("utf-8"), + ), + ( + "macaroon_secret_key_path: {}", + lambda c: c.key.macaroon_secret_key, + ), + ( + "recaptcha_private_key_path: {}", + lambda c: c.captcha.recaptcha_private_key.encode("utf-8"), + ), + ( + "recaptcha_public_key_path: {}", + lambda c: c.captcha.recaptcha_public_key.encode("utf-8"), + ), + ( + "form_secret_path: {}", + lambda c: c.key.form_secret.encode("utf-8"), + ), + ( + "worker_replication_secret_path: {}", + lambda c: c.worker.worker_replication_secret.encode("utf-8"), + ), + ( + "experimental_features:\n msc3861:\n client_secret_path: {}", + lambda c: c.experimental.msc3861.client_secret().encode("utf-8"), + ), + ( + "experimental_features:\n msc3861:\n admin_token_path: {}", + lambda c: c.experimental.msc3861.admin_token().encode("utf-8"), ), *[ ( "redis:\n enabled: true\n password_path: {}", - lambda c: c.redis.redis_password, + lambda c: c.redis.redis_password.encode("utf-8"), ) ] * (hiredis is not None), @@ -164,11 +205,111 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): def test_secret_files_existing( self, config_line: str, get_secret: Callable[[RootConfig], str] ) -> None: - self.generate_config_and_remove_lines_containing("registration_shared_secret") + self.generate_config_and_remove_lines_containing( + ["form_secret", "macaroon_secret_key", "registration_shared_secret"] + ) with tempfile.NamedTemporaryFile(buffering=0) as secret_file: secret_file.write(b"53C237") self.add_lines_to_config(["", config_line.format(secret_file.name)]) config = HomeServerConfig.load_config("", ["-c", self.config_file]) - self.assertEqual(get_secret(config), "53C237") + self.assertEqual(get_secret(config), b"53C237") + + @parameterized.expand( + [ + "turn_shared_secret: 53C237", + "registration_shared_secret: 53C237", + "macaroon_secret_key: 53C237", + "recaptcha_private_key: 53C237", + "recaptcha_public_key: ¬53C237", + "form_secret: 53C237", + "worker_replication_secret: 53C237", + *[ + "experimental_features:\n" + " msc3861:\n" + " enabled: true\n" + " client_secret: 53C237" + ] + * (authlib is not None), + *[ + "experimental_features:\n" + " msc3861:\n" + " enabled: true\n" + " client_auth_method: private_key_jwt\n" + ' jwk: {{"mock": "mock"}}' + ] + * (authlib is not None), + *[ + "experimental_features:\n" + " msc3861:\n" + " enabled: true\n" + " admin_token: 53C237\n" + " client_secret_path: {secret_file}" + ] + * (authlib is not None), + *["redis:\n enabled: true\n password: 53C237"] * (hiredis is not None), + ] + ) + def test_no_secrets_in_config(self, config_line: str) -> None: + if authlib is not None: + patcher = mock.patch("authlib.jose.rfc7517.JsonWebKey.import_key") + self.addCleanup(patcher.stop) + patcher.start() + + with tempfile.NamedTemporaryFile(buffering=0) as secret_file: + # Only used for less mocking with admin_token + secret_file.write(b"53C237") + + self.generate_config_and_remove_lines_containing( + ["form_secret", "macaroon_secret_key", "registration_shared_secret"] + ) + # Check strict mode with no offenders. + HomeServerConfig.load_config( + "", ["-c", self.config_file, "--no-secrets-in-config"] + ) + self.add_lines_to_config( + ["", config_line.format(secret_file=secret_file.name)] + ) + # Check strict mode with a single offender. + with self.assertRaises(ConfigError): + HomeServerConfig.load_config( + "", ["-c", self.config_file, "--no-secrets-in-config"] + ) + + # Check lenient mode with a single offender. + HomeServerConfig.load_config("", ["-c", self.config_file]) + + def test_no_secrets_in_config_but_in_files(self) -> None: + with tempfile.NamedTemporaryFile(buffering=0) as secret_file: + secret_file.write(b"53C237") + + self.generate_config_and_remove_lines_containing( + ["form_secret", "macaroon_secret_key", "registration_shared_secret"] + ) + self.add_lines_to_config( + [ + "", + f"turn_shared_secret_path: {secret_file.name}", + f"registration_shared_secret_path: {secret_file.name}", + f"macaroon_secret_key_path: {secret_file.name}", + f"recaptcha_private_key_path: {secret_file.name}", + f"recaptcha_public_key_path: {secret_file.name}", + f"form_secret_path: {secret_file.name}", + f"worker_replication_secret_path: {secret_file.name}", + *[ + "experimental_features:\n" + " msc3861:\n" + " enabled: true\n" + f" admin_token_path: {secret_file.name}\n" + f" client_secret_path: {secret_file.name}\n" + # f" jwk_path: {secret_file.name}" + ] + * (authlib is not None), + *[f"redis:\n enabled: true\n password_path: {secret_file.name}"] + * (hiredis is not None), + ] + ) + HomeServerConfig.load_config( + "", ["-c", self.config_file, "--no-secrets-in-config"] + ) diff --git a/tests/config/test_oauth_delegation.py b/tests/config/test_oauth_delegation.py index 713bddeb90..833cfe628b 100644 --- a/tests/config/test_oauth_delegation.py +++ b/tests/config/test_oauth_delegation.py @@ -20,6 +20,7 @@ # import os +import tempfile from unittest.mock import Mock from synapse.config import ConfigError @@ -275,3 +276,168 @@ class MSC3861OAuthDelegation(TestCase): self.config_dict["enable_3pid_changes"] = True with self.assertRaises(ConfigError): self.parse_config() + + +class MasAuthDelegation(TestCase): + """Test that the Homeserver fails to initialize if the config is invalid.""" + + def setUp(self) -> None: + self.config_dict: JsonDict = { + **default_config("test"), + "public_baseurl": BASE_URL, + "enable_registration": False, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:1324/", + "secret": "verysecret", + }, + } + + def parse_config(self) -> HomeServerConfig: + config = HomeServerConfig() + config.parse_config_dict(self.config_dict, "", "") + return config + + def test_endpoint_has_to_be_a_url(self) -> None: + self.config_dict["matrix_authentication_service"]["endpoint"] = "not a url" + with self.assertRaises(ConfigError): + self.parse_config() + + def test_secret_and_secret_path_are_mutually_exclusive(self) -> None: + with tempfile.NamedTemporaryFile() as f: + self.config_dict["matrix_authentication_service"]["secret"] = "verysecret" + self.config_dict["matrix_authentication_service"]["secret_path"] = f.name + with self.assertRaises(ConfigError): + self.parse_config() + + def test_secret_path_loads_secret(self) -> None: + with tempfile.NamedTemporaryFile(buffering=0) as f: + f.write(b"53C237") + del self.config_dict["matrix_authentication_service"]["secret"] + self.config_dict["matrix_authentication_service"]["secret_path"] = f.name + config = self.parse_config() + self.assertEqual(config.mas.secret(), "53C237") + + def test_secret_path_must_exist(self) -> None: + del self.config_dict["matrix_authentication_service"]["secret"] + self.config_dict["matrix_authentication_service"]["secret_path"] = ( + "/not/a/valid/file" + ) + with self.assertRaises(ConfigError): + self.parse_config() + + def test_registration_cannot_be_enabled(self) -> None: + self.config_dict["enable_registration"] = True + with self.assertRaises(ConfigError): + self.parse_config() + + def test_user_consent_cannot_be_enabled(self) -> None: + tmpdir = self.mktemp() + os.mkdir(tmpdir) + self.config_dict["user_consent"] = { + "require_at_registration": True, + "version": "1", + "template_dir": tmpdir, + "server_notice_content": { + "msgtype": "m.text", + "body": "foo", + }, + } + with self.assertRaises(ConfigError): + self.parse_config() + + def test_password_config_cannot_be_enabled(self) -> None: + self.config_dict["password_config"] = {"enabled": True} + with self.assertRaises(ConfigError): + self.parse_config() + + @skip_unless(HAS_AUTHLIB, "requires authlib") + def test_oidc_sso_cannot_be_enabled(self) -> None: + self.config_dict["oidc_providers"] = [ + { + "idp_id": "microsoft", + "idp_name": "Microsoft", + "issuer": "https://login.microsoftonline.com//v2.0", + "client_id": "", + "client_secret": "", + "scopes": ["openid", "profile"], + "authorization_endpoint": "https://login.microsoftonline.com//oauth2/v2.0/authorize", + "token_endpoint": "https://login.microsoftonline.com//oauth2/v2.0/token", + "userinfo_endpoint": "https://graph.microsoft.com/oidc/userinfo", + } + ] + + with self.assertRaises(ConfigError): + self.parse_config() + + def test_cas_sso_cannot_be_enabled(self) -> None: + self.config_dict["cas_config"] = { + "enabled": True, + "server_url": "https://cas-server.com", + "displayname_attribute": "name", + "required_attributes": {"userGroup": "staff", "department": "None"}, + } + + with self.assertRaises(ConfigError): + self.parse_config() + + def test_auth_providers_cannot_be_enabled(self) -> None: + self.config_dict["modules"] = [ + { + "module": f"{__name__}.{CustomAuthModule.__qualname__}", + "config": {}, + } + ] + + # This requires actually setting up an HS, as the module will be run on setup, + # which should raise as the module tries to register an auth provider + config = self.parse_config() + reactor, clock = get_clock() + with self.assertRaises(ConfigError): + setup_test_homeserver( + self.addCleanup, reactor=reactor, clock=clock, config=config + ) + + @skip_unless(HAS_AUTHLIB, "requires authlib") + def test_jwt_auth_cannot_be_enabled(self) -> None: + self.config_dict["jwt_config"] = { + "enabled": True, + "secret": "my-secret-token", + "algorithm": "HS256", + } + + with self.assertRaises(ConfigError): + self.parse_config() + + def test_login_via_existing_session_cannot_be_enabled(self) -> None: + self.config_dict["login_via_existing_session"] = {"enabled": True} + with self.assertRaises(ConfigError): + self.parse_config() + + def test_captcha_cannot_be_enabled(self) -> None: + self.config_dict.update( + enable_registration_captcha=True, + recaptcha_public_key="test", + recaptcha_private_key="test", + ) + with self.assertRaises(ConfigError): + self.parse_config() + + def test_refreshable_tokens_cannot_be_enabled(self) -> None: + self.config_dict.update( + refresh_token_lifetime="24h", + refreshable_access_token_lifetime="10m", + nonrefreshable_access_token_lifetime="24h", + ) + with self.assertRaises(ConfigError): + self.parse_config() + + def test_session_lifetime_cannot_be_set(self) -> None: + self.config_dict["session_lifetime"] = "24h" + with self.assertRaises(ConfigError): + self.parse_config() + + def test_enable_3pid_changes_cannot_be_enabled(self) -> None: + self.config_dict["enable_3pid_changes"] = True + with self.assertRaises(ConfigError): + self.parse_config() diff --git a/tests/config/test_room_directory.py b/tests/config/test_room_directory.py index e25f7787f4..5f3d8be2a5 100644 --- a/tests/config/test_room_directory.py +++ b/tests/config/test_room_directory.py @@ -19,11 +19,12 @@ # import yaml -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin import synapse.rest.client.login import synapse.rest.client.room +from synapse.config._base import RootConfig from synapse.config.room_directory import RoomDirectoryConfig from synapse.server import HomeServer from synapse.util import Clock @@ -63,7 +64,7 @@ class RoomDirectoryConfigTestCase(unittest.HomeserverTestCase): """ ) - rd_config = RoomDirectoryConfig() + rd_config = RoomDirectoryConfig(RootConfig()) rd_config.read_config(config) self.assertFalse( @@ -123,7 +124,7 @@ class RoomDirectoryConfigTestCase(unittest.HomeserverTestCase): """ ) - rd_config = RoomDirectoryConfig() + rd_config = RoomDirectoryConfig(RootConfig()) rd_config.read_config(config) self.assertFalse( diff --git a/tests/config/test_server.py b/tests/config/test_server.py index 74073cfdc5..5eb2540439 100644 --- a/tests/config/test_server.py +++ b/tests/config/test_server.py @@ -20,7 +20,7 @@ import yaml -from synapse.config._base import ConfigError +from synapse.config._base import ConfigError, RootConfig from synapse.config.server import ServerConfig, generate_ip_set, is_threepid_reserved from tests import unittest @@ -40,7 +40,7 @@ class ServerConfigTestCase(unittest.TestCase): def test_unsecure_listener_no_listeners_open_private_ports_false(self) -> None: conf = yaml.safe_load( - ServerConfig().generate_config_section( + ServerConfig(RootConfig()).generate_config_section( "CONFDIR", "/data_dir_path", "che.org", False, None ) ) @@ -60,7 +60,7 @@ class ServerConfigTestCase(unittest.TestCase): def test_unsecure_listener_no_listeners_open_private_ports_true(self) -> None: conf = yaml.safe_load( - ServerConfig().generate_config_section( + ServerConfig(RootConfig()).generate_config_section( "CONFDIR", "/data_dir_path", "che.org", True, None ) ) @@ -94,7 +94,7 @@ class ServerConfigTestCase(unittest.TestCase): ] conf = yaml.safe_load( - ServerConfig().generate_config_section( + ServerConfig(RootConfig()).generate_config_section( "CONFDIR", "/data_dir_path", "this.one.listens", True, listeners ) ) @@ -128,7 +128,7 @@ class ServerConfigTestCase(unittest.TestCase): expected_listeners[1]["bind_addresses"] = ["::1", "127.0.0.1"] conf = yaml.safe_load( - ServerConfig().generate_config_section( + ServerConfig(RootConfig()).generate_config_section( "CONFDIR", "/data_dir_path", "this.one.listens", True, listeners ) ) diff --git a/tests/config/test_workers.py b/tests/config/test_workers.py index 64c0285d01..3a21975b89 100644 --- a/tests/config/test_workers.py +++ b/tests/config/test_workers.py @@ -47,7 +47,7 @@ class WorkerDutyConfigTestCase(TestCase): "worker_app": worker_app, **extras, } - worker_config.read_config(worker_config_dict) + worker_config.read_config(worker_config_dict, allow_secrets_in_config=True) return worker_config def test_old_configs_master(self) -> None: diff --git a/tests/config/utils.py b/tests/config/utils.py index 11140ff979..3cba4ac588 100644 --- a/tests/config/utils.py +++ b/tests/config/utils.py @@ -51,12 +51,13 @@ class ConfigFileTestCase(unittest.TestCase): ], ) - def generate_config_and_remove_lines_containing(self, needle: str) -> None: + def generate_config_and_remove_lines_containing(self, needles: list[str]) -> None: self.generate_config() with open(self.config_file) as f: contents = f.readlines() - contents = [line for line in contents if needle not in line] + for needle in needles: + contents = [line for line in contents if needle not in line] with open(self.config_file, "w") as f: f.write("".join(contents)) diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index d7b9fb8bc6..9cdc1604da 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -48,7 +48,6 @@ class EventSigningTestCase(unittest.TestCase): def test_sign_minimal(self) -> None: event_dict = { "event_id": "$0:domain", - "origin": "domain", "origin_server_ts": 1000000, "signatures": {}, "type": "X", @@ -64,7 +63,7 @@ class EventSigningTestCase(unittest.TestCase): self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) self.assertEqual( - event.hashes["sha256"], "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI" + event.hashes["sha256"], "A6Nco6sqoy18PPfPDVdYvoowfc0PVBk9g9OiyT3ncRM" ) self.assertTrue(hasattr(event, "signatures")) @@ -72,15 +71,14 @@ class EventSigningTestCase(unittest.TestCase): self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEqual( event.signatures[HOSTNAME][KEY_NAME], - "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+" - "aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA", + "PBc48yDVszWB9TRaB/+CZC1B+pDAC10F8zll006j+NN" + "fe4PEMWcVuLaG63LFTK9e4rwJE8iLZMPtCKhDTXhpAQ", ) def test_sign_message(self) -> None: event_dict = { "content": {"body": "Here is the message content"}, "event_id": "$0:domain", - "origin": "domain", "origin_server_ts": 1000000, "type": "m.room.message", "room_id": "!r:domain", @@ -98,7 +96,7 @@ class EventSigningTestCase(unittest.TestCase): self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) self.assertEqual( - event.hashes["sha256"], "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g" + event.hashes["sha256"], "rDCeYBepPlI891h/RkI2/Lkf9bt7u0TxFku4tMs7WKk" ) self.assertTrue(hasattr(event, "signatures")) @@ -106,6 +104,6 @@ class EventSigningTestCase(unittest.TestCase): self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEqual( event.signatures[HOSTNAME][KEY_NAME], - "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUw" - "u6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA", + "Ay4aj2b5oJ1k8INYZ9n3KnszCflM0emwcmQQ7vxpbdc" + "Sv9bkJxIZdWX1IJllcZLq89+D3sSabE+vqPtZs9akDw", ) diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py index 3bfaf1c80d..80f9bd097e 100644 --- a/tests/crypto/test_keyring.py +++ b/tests/crypto/test_keyring.py @@ -31,7 +31,7 @@ from signedjson.types import SigningKey, VerifyKey from twisted.internet import defer from twisted.internet.defer import Deferred, ensureDeferred -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import SynapseError from synapse.crypto import keyring diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index 7fb4d4fa90..8f1dc86984 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -27,19 +27,20 @@ from unittest.mock import Mock import attr from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError +from synapse.config._base import RootConfig from synapse.config.auto_accept_invites import AutoAcceptInvitesConfig from synapse.events.auto_accept_invites import InviteAutoAccepter from synapse.federation.federation_base import event_from_pdu_json -from synapse.handlers.sync import JoinedSyncResult, SyncRequestKey, SyncVersion +from synapse.handlers.sync import JoinedSyncResult, SyncRequestKey from synapse.module_api import ModuleApi from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer -from synapse.types import StreamToken, create_requester +from synapse.types import StreamToken, UserID, UserInfo, create_requester from synapse.util import Clock from tests.handlers.test_sync import generate_sync_config @@ -349,6 +350,169 @@ class AutoAcceptInvitesTestCase(FederatingHomeserverTestCase): join_updates, _ = sync_join(self, invited_user_id) self.assertEqual(len(join_updates), 0) + @override_config( + { + "auto_accept_invites": { + "enabled": True, + }, + } + ) + async def test_ignore_invite_for_missing_user(self) -> None: + """Tests that receiving an invite for a missing user is ignored.""" + inviting_user_id = self.register_user("inviter", "pass") + inviting_user_tok = self.login("inviter", "pass") + + # A local user who receives an invite + invited_user_id = "@fake:" + self.hs.config.server.server_name + + # Create a room and send an invite to the other user + room_id = self.helper.create_room_as( + inviting_user_id, + tok=inviting_user_tok, + ) + + self.helper.invite( + room_id, + inviting_user_id, + invited_user_id, + tok=inviting_user_tok, + ) + + join_updates, _ = sync_join(self, inviting_user_id) + # Assert that the last event in the room was not a member event for the target user. + self.assertEqual( + join_updates[0].timeline.events[-1].content["membership"], "invite" + ) + + @override_config( + { + "auto_accept_invites": { + "enabled": True, + }, + } + ) + async def test_ignore_invite_for_deactivated_user(self) -> None: + """Tests that receiving an invite for a deactivated user is ignored.""" + inviting_user_id = self.register_user("inviter", "pass", admin=True) + inviting_user_tok = self.login("inviter", "pass") + + # A local user who receives an invite + invited_user_id = self.register_user("invitee", "pass") + + # Create a room and send an invite to the other user + room_id = self.helper.create_room_as( + inviting_user_id, + tok=inviting_user_tok, + ) + + channel = self.make_request( + "PUT", + "/_synapse/admin/v2/users/%s" % invited_user_id, + {"deactivated": True}, + access_token=inviting_user_tok, + ) + + assert channel.code == 200 + + self.helper.invite( + room_id, + inviting_user_id, + invited_user_id, + tok=inviting_user_tok, + ) + + join_updates, b = sync_join(self, inviting_user_id) + # Assert that the last event in the room was not a member event for the target user. + self.assertEqual( + join_updates[0].timeline.events[-1].content["membership"], "invite" + ) + + @override_config( + { + "auto_accept_invites": { + "enabled": True, + }, + } + ) + async def test_ignore_invite_for_suspended_user(self) -> None: + """Tests that receiving an invite for a suspended user is ignored.""" + inviting_user_id = self.register_user("inviter", "pass", admin=True) + inviting_user_tok = self.login("inviter", "pass") + + # A local user who receives an invite + invited_user_id = self.register_user("invitee", "pass") + + # Create a room and send an invite to the other user + room_id = self.helper.create_room_as( + inviting_user_id, + tok=inviting_user_tok, + ) + + channel = self.make_request( + "PUT", + f"/_synapse/admin/v1/suspend/{invited_user_id}", + {"suspend": True}, + access_token=inviting_user_tok, + ) + + assert channel.code == 200 + + self.helper.invite( + room_id, + inviting_user_id, + invited_user_id, + tok=inviting_user_tok, + ) + + join_updates, b = sync_join(self, inviting_user_id) + # Assert that the last event in the room was not a member event for the target user. + self.assertEqual( + join_updates[0].timeline.events[-1].content["membership"], "invite" + ) + + @override_config( + { + "auto_accept_invites": { + "enabled": True, + }, + } + ) + async def test_ignore_invite_for_locked_user(self) -> None: + """Tests that receiving an invite for a suspended user is ignored.""" + inviting_user_id = self.register_user("inviter", "pass", admin=True) + inviting_user_tok = self.login("inviter", "pass") + + # A local user who receives an invite + invited_user_id = self.register_user("invitee", "pass") + + # Create a room and send an invite to the other user + room_id = self.helper.create_room_as( + inviting_user_id, + tok=inviting_user_tok, + ) + + channel = self.make_request( + "PUT", + f"/_synapse/admin/v2/users/{invited_user_id}", + {"locked": True}, + access_token=inviting_user_tok, + ) + + assert channel.code == 200 + + self.helper.invite( + room_id, + inviting_user_id, + invited_user_id, + tok=inviting_user_tok, + ) + + join_updates, b = sync_join(self, inviting_user_id) + # Assert that the last event in the room was not a member event for the target user. + self.assertEqual( + join_updates[0].timeline.events[-1].content["membership"], "invite" + ) + _request_key = 0 @@ -384,7 +548,6 @@ def sync_join( testcase.hs.get_sync_handler().wait_for_sync_for_user( requester, sync_config, - SyncVersion.SYNC_V2, generate_request_key(), since_token, ) @@ -527,7 +690,7 @@ class InviteAutoAccepterInternalTestCase(TestCase): "only_from_local_users": True, } } - parsed_config = AutoAcceptInvitesConfig() + parsed_config = AutoAcceptInvitesConfig(RootConfig()) parsed_config.read_config(config) self.assertTrue(parsed_config.enabled) @@ -647,11 +810,27 @@ def create_module( module_api.is_mine.side_effect = lambda a: a.split(":")[1] == "test" module_api.worker_name = worker_name module_api.sleep.return_value = make_multiple_awaitable(None) + module_api.get_userinfo_by_id.return_value = UserInfo( + user_id=UserID.from_string("@user:test"), + is_admin=False, + is_guest=False, + consent_server_notice_sent=None, + consent_ts=None, + consent_version=None, + appservice_id=None, + creation_ts=0, + user_type=None, + is_deactivated=False, + locked=False, + is_shadow_banned=False, + approved=True, + suspended=False, + ) if config_override is None: config_override = {} - config = AutoAcceptInvitesConfig() + config = AutoAcceptInvitesConfig(RootConfig()) config.read_config(config_override) return InviteAutoAccepter(config, module_api) diff --git a/tests/events/test_presence_router.py b/tests/events/test_presence_router.py index e48983ddfe..f7d55223b1 100644 --- a/tests/events/test_presence_router.py +++ b/tests/events/test_presence_router.py @@ -23,7 +23,7 @@ from unittest.mock import AsyncMock, Mock import attr -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EduTypes from synapse.events.presence_router import PresenceRouter, load_legacy_presence_router @@ -36,7 +36,7 @@ from synapse.server import HomeServer from synapse.types import JsonDict, StreamToken, create_requester from synapse.util import Clock -from tests.handlers.test_sync import SyncRequestKey, SyncVersion, generate_sync_config +from tests.handlers.test_sync import SyncRequestKey, generate_sync_config from tests.unittest import ( FederatingHomeserverTestCase, HomeserverTestCase, @@ -532,7 +532,6 @@ def sync_presence( testcase.hs.get_sync_handler().wait_for_sync_for_user( requester, sync_config, - SyncVersion.SYNC_V2, generate_request_key(), since_token, ) diff --git a/tests/events/test_snapshot.py b/tests/events/test_snapshot.py index f96bbe7705..6d24730ed7 100644 --- a/tests/events/test_snapshot.py +++ b/tests/events/test_snapshot.py @@ -19,7 +19,7 @@ # # -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.events import EventBase from synapse.events.snapshot import EventContext diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 654e6521a2..c6ebefbf38 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -34,11 +34,13 @@ from synapse.events.utils import ( _split_field, clone_event, copy_and_fixup_power_levels_contents, + format_event_raw, + make_config_for_admin, maybe_upsert_event_field, prune_event, serialize_event, ) -from synapse.types import JsonDict +from synapse.types import JsonDict, create_requester from synapse.util.frozenutils import freeze @@ -49,7 +51,13 @@ def MockEvent(**kwargs: Any) -> EventBase: kwargs["type"] = "fake_type" if "content" not in kwargs: kwargs["content"] = {} - return make_event_from_dict(kwargs) + + # Move internal metadata out so we can call make_event properly + internal_metadata = kwargs.get("internal_metadata") + if internal_metadata is not None: + kwargs.pop("internal_metadata") + + return make_event_from_dict(kwargs, internal_metadata_dict=internal_metadata) class TestMaybeUpsertEventField(stdlib_unittest.TestCase): @@ -122,7 +130,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase): "prev_events": "prev_events", "prev_state": "prev_state", "auth_events": "auth_events", - "origin": "domain", + "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", # Also include a key that should be removed. @@ -139,7 +147,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase): "prev_events": "prev_events", "prev_state": "prev_state", "auth_events": "auth_events", - "origin": "domain", + "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", "content": {}, @@ -148,13 +156,12 @@ class PruneEventTestCase(stdlib_unittest.TestCase): }, ) - # As of room versions we now redact the membership, prev_states, and origin keys. + # As of room versions we now redact the membership and prev_states keys. self.run_test( { "type": "A", "prev_state": "prev_state", "membership": "join", - "origin": "example.com", }, {"type": "A", "content": {}, "signatures": {}, "unsigned": {}}, room_version=RoomVersions.V11, @@ -238,7 +245,6 @@ class PruneEventTestCase(stdlib_unittest.TestCase): { "type": "m.room.create", "content": {"not_a_real_key": True}, - "origin": "some_homeserver", "nonsense_field": "some_random_garbage", }, { @@ -639,9 +645,18 @@ class CloneEventTestCase(stdlib_unittest.TestCase): class SerializeEventTestCase(stdlib_unittest.TestCase): - def serialize(self, ev: EventBase, fields: Optional[List[str]]) -> JsonDict: + def serialize( + self, + ev: EventBase, + fields: Optional[List[str]], + include_admin_metadata: bool = False, + ) -> JsonDict: return serialize_event( - ev, 1479807801915, config=SerializeEventConfig(only_event_fields=fields) + ev, + 1479807801915, + config=SerializeEventConfig( + only_event_fields=fields, include_admin_metadata=include_admin_metadata + ), ) def test_event_fields_works_with_keys(self) -> None: @@ -760,6 +775,104 @@ class SerializeEventTestCase(stdlib_unittest.TestCase): ["room_id", 4], # type: ignore[list-item] ) + def test_default_serialize_config_excludes_admin_metadata(self) -> None: + # We just really don't want this to be set to True accidentally + self.assertFalse(SerializeEventConfig().include_admin_metadata) + + def test_event_flagged_for_admins(self) -> None: + # Default behaviour should be *not* to include it + self.assertEqual( + self.serialize( + MockEvent( + type="foo", + event_id="test", + room_id="!foo:bar", + content={"foo": "bar"}, + internal_metadata={"soft_failed": True}, + ), + [], + ), + { + "type": "foo", + "event_id": "test", + "room_id": "!foo:bar", + "content": {"foo": "bar"}, + "unsigned": {}, + }, + ) + + # When asked though, we should set it + self.assertEqual( + self.serialize( + MockEvent( + type="foo", + event_id="test", + room_id="!foo:bar", + content={"foo": "bar"}, + internal_metadata={"soft_failed": True}, + ), + [], + True, + ), + { + "type": "foo", + "event_id": "test", + "room_id": "!foo:bar", + "content": {"foo": "bar"}, + "unsigned": {"io.element.synapse.soft_failed": True}, + }, + ) + self.assertEqual( + self.serialize( + MockEvent( + type="foo", + event_id="test", + room_id="!foo:bar", + content={"foo": "bar"}, + internal_metadata={ + "soft_failed": True, + "policy_server_spammy": True, + }, + ), + [], + True, + ), + { + "type": "foo", + "event_id": "test", + "room_id": "!foo:bar", + "content": {"foo": "bar"}, + "unsigned": { + "io.element.synapse.soft_failed": True, + "io.element.synapse.policy_server_spammy": True, + }, + }, + ) + + def test_make_serialize_config_for_admin_retains_other_fields(self) -> None: + non_default_config = SerializeEventConfig( + include_admin_metadata=False, # should be True in a moment + as_client_event=False, # default True + event_format=format_event_raw, # default format_event_for_client_v1 + requester=create_requester("@example:example.org"), # default None + only_event_fields=["foo"], # default None + include_stripped_room_state=True, # default False + ) + admin_config = make_config_for_admin(non_default_config) + self.assertEqual( + admin_config.as_client_event, non_default_config.as_client_event + ) + self.assertEqual(admin_config.event_format, non_default_config.event_format) + self.assertEqual(admin_config.requester, non_default_config.requester) + self.assertEqual( + admin_config.only_event_fields, non_default_config.only_event_fields + ) + self.assertEqual( + admin_config.include_stripped_room_state, + admin_config.include_stripped_room_state, + ) + self.assertTrue(admin_config.include_admin_metadata) + class CopyPowerLevelsContentTestCase(stdlib_unittest.TestCase): def setUp(self) -> None: diff --git a/tests/federation/test_federation_catch_up.py b/tests/federation/test_federation_catch_up.py index 1e1ed8e642..f99911b102 100644 --- a/tests/federation/test_federation_catch_up.py +++ b/tests/federation/test_federation_catch_up.py @@ -2,7 +2,7 @@ from typing import Callable, Collection, List, Optional, Tuple from unittest import mock from unittest.mock import AsyncMock, Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes from synapse.events import EventBase diff --git a/tests/federation/test_federation_client.py b/tests/federation/test_federation_client.py index 585f3b798c..df688cd21f 100644 --- a/tests/federation/test_federation_client.py +++ b/tests/federation/test_federation_client.py @@ -23,7 +23,7 @@ from unittest import mock import twisted.web.client from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.room_versions import RoomVersions from synapse.events import EventBase diff --git a/tests/federation/test_federation_devices.py b/tests/federation/test_federation_devices.py new file mode 100644 index 0000000000..bf6204a7e3 --- /dev/null +++ b/tests/federation/test_federation_devices.py @@ -0,0 +1,161 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +import logging +from unittest.mock import AsyncMock, Mock + +from twisted.internet.testing import MemoryReactor + +from synapse.handlers.device import DeviceListUpdater +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock +from synapse.util.retryutils import NotRetryingDestination + +from tests import unittest + +logger = logging.getLogger(__name__) + + +class DeviceListResyncTestCase(unittest.HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = self.hs.get_datastores().main + + def test_retry_device_list_resync(self) -> None: + """Tests that device lists are marked as stale if they couldn't be synced, and + that stale device lists are retried periodically. + """ + remote_user_id = "@john:test_remote" + remote_origin = "test_remote" + + # Track the number of attempts to resync the user's device list. + self.resync_attempts = 0 + + # When this function is called, increment the number of resync attempts (only if + # we're querying devices for the right user ID), then raise a + # NotRetryingDestination error to fail the resync gracefully. + def query_user_devices( + destination: str, user_id: str, timeout: int = 30000 + ) -> JsonDict: + if user_id == remote_user_id: + self.resync_attempts += 1 + + raise NotRetryingDestination(0, 0, destination) + + # Register the mock on the federation client. + federation_client = self.hs.get_federation_client() + federation_client.query_user_devices = Mock(side_effect=query_user_devices) # type: ignore[method-assign] + + # Register a mock on the store so that the incoming update doesn't fail because + # we don't share a room with the user. + self.store.get_rooms_for_user = AsyncMock(return_value=["!someroom:test"]) + + # Manually inject a fake device list update. We need this update to include at + # least one prev_id so that the user's device list will need to be retried. + device_list_updater = self.hs.get_device_handler().device_list_updater + assert isinstance(device_list_updater, DeviceListUpdater) + self.get_success( + device_list_updater.incoming_device_list_update( + origin=remote_origin, + edu_content={ + "deleted": False, + "device_display_name": "Mobile", + "device_id": "QBUAZIFURK", + "prev_id": [5], + "stream_id": 6, + "user_id": remote_user_id, + }, + ) + ) + + # Check that there was one resync attempt. + self.assertEqual(self.resync_attempts, 1) + + # Check that the resync attempt failed and caused the user's device list to be + # marked as stale. + need_resync = self.get_success( + self.store.get_user_ids_requiring_device_list_resync() + ) + self.assertIn(remote_user_id, need_resync) + + # Check that waiting for 30 seconds caused Synapse to retry resyncing the device + # list. + self.reactor.advance(30) + self.assertEqual(self.resync_attempts, 2) + + def test_cross_signing_keys_retry(self) -> None: + """Tests that resyncing a device list correctly processes cross-signing keys from + the remote server. + """ + remote_user_id = "@john:test_remote" + remote_master_key = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" + remote_self_signing_key = "QeIiFEjluPBtI7WQdG365QKZcFs9kqmHir6RBD0//nQ" + + # Register mock device list retrieval on the federation client. + federation_client = self.hs.get_federation_client() + federation_client.query_user_devices = AsyncMock( # type: ignore[method-assign] + return_value={ + "user_id": remote_user_id, + "stream_id": 1, + "devices": [], + "master_key": { + "user_id": remote_user_id, + "usage": ["master"], + "keys": {"ed25519:" + remote_master_key: remote_master_key}, + }, + "self_signing_key": { + "user_id": remote_user_id, + "usage": ["self_signing"], + "keys": { + "ed25519:" + remote_self_signing_key: remote_self_signing_key + }, + }, + } + ) + + # Resync the device list. + device_handler = self.hs.get_device_handler() + self.get_success( + device_handler.device_list_updater.multi_user_device_resync( + [remote_user_id] + ), + ) + + # Retrieve the cross-signing keys for this user. + keys = self.get_success( + self.store.get_e2e_cross_signing_keys_bulk(user_ids=[remote_user_id]), + ) + self.assertIn(remote_user_id, keys) + key = keys[remote_user_id] + assert key is not None + + # Check that the master key is the one returned by the mock. + master_key = key["master"] + self.assertEqual(len(master_key["keys"]), 1) + self.assertTrue("ed25519:" + remote_master_key in master_key["keys"].keys()) + self.assertTrue(remote_master_key in master_key["keys"].values()) + + # Check that the self-signing key is the one returned by the mock. + self_signing_key = key["self_signing"] + self.assertEqual(len(self_signing_key["keys"]), 1) + self.assertTrue( + "ed25519:" + remote_self_signing_key in self_signing_key["keys"].keys(), + ) + self.assertTrue(remote_self_signing_key in self_signing_key["keys"].values()) diff --git a/tests/federation/test_federation_media.py b/tests/federation/test_federation_media.py index e66aae499b..b9ec2794a3 100644 --- a/tests/federation/test_federation_media.py +++ b/tests/federation/test_federation_media.py @@ -22,7 +22,7 @@ import os import shutil import tempfile -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.media.filepath import MediaFilePaths from synapse.media.media_storage import MediaStorage @@ -67,7 +67,7 @@ class FederationMediaDownloadsTest(unittest.FederatingHomeserverTestCase): def test_file_download(self) -> None: content = io.BytesIO(b"file_to_stream") content_uri = self.get_success( - self.media_repo.create_content( + self.media_repo.create_or_update_content( "text/plain", "test_upload", content, @@ -110,7 +110,7 @@ class FederationMediaDownloadsTest(unittest.FederatingHomeserverTestCase): content = io.BytesIO(SMALL_PNG) content_uri = self.get_success( - self.media_repo.create_content( + self.media_repo.create_or_update_content( "image/png", "test_png_upload", content, @@ -147,6 +147,45 @@ class FederationMediaDownloadsTest(unittest.FederatingHomeserverTestCase): found_file = any(SMALL_PNG in field for field in stripped_bytes) self.assertTrue(found_file) + def test_federation_etag(self) -> None: + """Test that federation ETags work""" + + content = io.BytesIO(b"file_to_stream") + content_uri = self.get_success( + self.media_repo.create_or_update_content( + "text/plain", + "test_upload", + content, + 46, + UserID.from_string("@user_id:whatever.org"), + ) + ) + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/media/download/{content_uri.media_id}", + ) + self.pump() + self.assertEqual(200, channel.code) + + # We expect exactly one ETag header. + etags = channel.headers.getRawHeaders("ETag") + self.assertIsNotNone(etags) + assert etags is not None # For mypy + self.assertEqual(len(etags), 1) + etag = etags[0] + + # Refetching with the etag should result in 304 and empty body. + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/media/download/{content_uri.media_id}", + custom_headers=[("If-None-Match", etag)], + ) + self.pump() + self.assertEqual(channel.code, 304) + self.assertEqual(channel.is_finished(), True) + self.assertNotIn("body", channel.result) + class FederationThumbnailTest(unittest.FederatingHomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: @@ -176,7 +215,7 @@ class FederationThumbnailTest(unittest.FederatingHomeserverTestCase): def test_thumbnail_download_scaled(self) -> None: content = io.BytesIO(small_png.data) content_uri = self.get_success( - self.media_repo.create_content( + self.media_repo.create_or_update_content( "image/png", "test_png_thumbnail", content, @@ -216,7 +255,7 @@ class FederationThumbnailTest(unittest.FederatingHomeserverTestCase): def test_thumbnail_download_cropped(self) -> None: content = io.BytesIO(small_png.data) content_uri = self.get_success( - self.media_repo.create_content( + self.media_repo.create_or_update_content( "image/png", "test_png_thumbnail", content, diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py new file mode 100644 index 0000000000..acf343930f --- /dev/null +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -0,0 +1,671 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2020 The Matrix.org Foundation C.I.C. +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +import logging +import time +import urllib.parse +from http import HTTPStatus +from typing import Any, Callable, Optional, Set, Tuple, TypeVar, Union +from unittest.mock import Mock + +import attr +from parameterized import parameterized + +from twisted.internet.testing import MemoryReactor + +from synapse.api.constants import EventContentFields, EventTypes, Membership +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.events import EventBase, make_event_from_dict +from synapse.events.utils import strip_event +from synapse.federation.federation_base import ( + event_from_pdu_json, +) +from synapse.federation.transport.client import SendJoinResponse +from synapse.http.matrixfederationclient import ( + ByteParser, +) +from synapse.http.types import QueryParams +from synapse.rest import admin +from synapse.rest.client import login, room, sync +from synapse.server import HomeServer +from synapse.types import JsonDict, MutableStateMap, StateMap +from synapse.types.handlers.sliding_sync import ( + StateValues, +) +from synapse.util import Clock + +from tests import unittest +from tests.utils import test_timeout + +logger = logging.getLogger(__name__) + + +def required_state_json_to_state_map(required_state: Any) -> StateMap[EventBase]: + state_map: MutableStateMap[EventBase] = {} + + # Scrutinize JSON values to ensure it's in the expected format + if isinstance(required_state, list): + for state_event_dict in required_state: + # Yell because we're in a test and this is unexpected + assert isinstance(state_event_dict, dict), ( + "`required_state` should be a list of event dicts" + ) + + event_type = state_event_dict["type"] + event_state_key = state_event_dict["state_key"] + + # Yell because we're in a test and this is unexpected + assert isinstance(event_type, str), ( + "Each event in `required_state` should have a string `type`" + ) + assert isinstance(event_state_key, str), ( + "Each event in `required_state` should have a string `state_key`" + ) + + state_map[(event_type, event_state_key)] = make_event_from_dict( + state_event_dict + ) + else: + # Yell because we're in a test and this is unexpected + raise AssertionError("`required_state` should be a list of event dicts") + + return state_map + + +@attr.s(slots=True, auto_attribs=True) +class RemoteRoomJoinResult: + remote_room_id: str + room_version: RoomVersion + remote_room_creator_user_id: str + local_user1_id: str + local_user1_tok: str + state_map: StateMap[EventBase] + + +class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): + """ + Tests to make sure that interactions with out-of-band membership (outliers) works as + expected. + + - invites received over federation, before we join the room + - *rejections* for said invites + + See the "Out-of-band membership events" section in + `docs/development/room-dag-concepts.md` for more information. + """ + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + sync.register_servlets, + ] + + sync_endpoint = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync" + + def default_config(self) -> JsonDict: + conf = super().default_config() + # Federation sending is disabled by default in the test environment + # so we need to enable it like this. + conf["federation_sender_instances"] = ["master"] + + return conf + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.federation_http_client = Mock( + # The problem with using `spec=MatrixFederationHttpClient` here is that it + # requires everything to be mocked which is a lot of work that I don't want + # to do when the code only uses a few methods (`get_json` and `put_json`). + ) + return self.setup_test_homeserver( + federation_http_client=self.federation_http_client + ) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + self.store = self.hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + + def do_sync( + self, sync_body: JsonDict, *, since: Optional[str] = None, tok: str + ) -> Tuple[JsonDict, str]: + """Do a sliding sync request with given body. + + Asserts the request was successful. + + Attributes: + sync_body: The full request body to use + since: Optional since token + tok: Access token to use + + Returns: + A tuple of the response body and the `pos` field. + """ + + sync_path = self.sync_endpoint + if since: + sync_path += f"?pos={since}" + + channel = self.make_request( + method="POST", + path=sync_path, + content=sync_body, + access_token=tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + return channel.json_body, channel.json_body["pos"] + + def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: + """ + Helper to reproduce this scenario: + + 1. The remote user invites our local user to a room on their remote server (which + creates an out-of-band invite membership for user1 on our local server). + 2. The local user notices the invite from `/sync`. + 3. The local user joins the room. + 4. The local user can see that they are now joined to the room from `/sync`. + """ + + # Create a local user + local_user1_id = self.register_user("user1", "pass") + local_user1_tok = self.login(local_user1_id, "pass") + + # Create a remote room + room_creator_user_id = f"@remote-user:{self.OTHER_SERVER_NAME}" + remote_room_id = f"!remote-room:{self.OTHER_SERVER_NAME}" + room_version = RoomVersions.V10 + + room_create_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": room_creator_user_id, + "depth": 1, + "origin_server_ts": 1, + "type": EventTypes.Create, + "state_key": "", + "content": { + # The `ROOM_CREATOR` field could be removed if we used a room + # version > 10 (in favor of relying on `sender`) + EventContentFields.ROOM_CREATOR: room_creator_user_id, + EventContentFields.ROOM_VERSION: room_version.identifier, + }, + "auth_events": [], + "prev_events": [], + } + ), + room_version=room_version, + ) + + creator_membership_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": room_creator_user_id, + "depth": 2, + "origin_server_ts": 2, + "type": EventTypes.Member, + "state_key": room_creator_user_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [room_create_event.event_id], + "prev_events": [room_create_event.event_id], + } + ), + room_version=room_version, + ) + + # From the remote homeserver, invite user1 on the local homserver + user1_invite_membership_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": room_creator_user_id, + "depth": 3, + "origin_server_ts": 3, + "type": EventTypes.Member, + "state_key": local_user1_id, + "content": {"membership": Membership.INVITE}, + "auth_events": [ + room_create_event.event_id, + creator_membership_event.event_id, + ], + "prev_events": [creator_membership_event.event_id], + } + ), + room_version=room_version, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/invite/{remote_room_id}/{user1_invite_membership_event.event_id}", + content={ + "event": user1_invite_membership_event.get_dict(), + "invite_room_state": [ + strip_event(room_create_event), + ], + "room_version": room_version.identifier, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [(EventTypes.Member, StateValues.WILDCARD)], + "timeline_limit": 0, + } + } + } + + # Sync until the local user1 can see the invite + with test_timeout( + 3, + "Unable to find user1's invite event in the room", + ): + while True: + response_body, _ = self.do_sync(sync_body, tok=local_user1_tok) + if ( + remote_room_id in response_body["rooms"].keys() + # If they have `invite_state` for the room, they are invited + and len( + response_body["rooms"][remote_room_id].get("invite_state", []) + ) + > 0 + ): + break + + # Prevent tight-looping to allow the `test_timeout` to work + time.sleep(0.1) + + user1_join_membership_event_template = make_event_from_dict( + { + "room_id": remote_room_id, + "sender": local_user1_id, + "depth": 4, + "origin_server_ts": 4, + "type": EventTypes.Member, + "state_key": local_user1_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + room_create_event.event_id, + user1_invite_membership_event.event_id, + ], + "prev_events": [user1_invite_membership_event.event_id], + }, + room_version=room_version, + ) + + T = TypeVar("T") + + # Mock the remote homeserver responding to our HTTP requests + # + # We're going to mock the following endpoints so that user1 can join the remote room: + # - GET /_matrix/federation/v1/make_join/{room_id}/{user_id} + # - PUT /_matrix/federation/v2/send_join/{room_id}/{user_id} + # + async def get_json( + destination: str, + path: str, + args: Optional[QueryParams] = None, + retry_on_dns_fail: bool = True, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + try_trailing_slash_on_400: bool = False, + parser: Optional[ByteParser[T]] = None, + ) -> Union[JsonDict, T]: + if ( + path + == f"/_matrix/federation/v1/make_join/{urllib.parse.quote_plus(remote_room_id)}/{urllib.parse.quote_plus(local_user1_id)}" + ): + return { + "event": user1_join_membership_event_template.get_pdu_json(), + "room_version": room_version.identifier, + } + + raise NotImplementedError( + "We have not mocked a response for `get_json(...)` for the following endpoint yet: " + + f"{destination}{path}" + ) + + self.federation_http_client.get_json.side_effect = get_json + + # PDU's that hs1 sent to hs2 + collected_pdus_from_hs1_federation_send: Set[str] = set() + + async def put_json( + destination: str, + path: str, + args: Optional[QueryParams] = None, + data: Optional[JsonDict] = None, + json_data_callback: Optional[Callable[[], JsonDict]] = None, + long_retries: bool = False, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + backoff_on_404: bool = False, + try_trailing_slash_on_400: bool = False, + parser: Optional[ByteParser[T]] = None, + backoff_on_all_error_codes: bool = False, + ) -> Union[JsonDict, T, SendJoinResponse]: + if ( + path.startswith( + f"/_matrix/federation/v2/send_join/{urllib.parse.quote_plus(remote_room_id)}/" + ) + and data is not None + and data.get("type") == EventTypes.Member + and data.get("state_key") == local_user1_id + # We're assuming this is a `ByteParser[SendJoinResponse]` + and parser is not None + ): + # As the remote server, we need to sign the event before sending it back + user1_join_membership_event_signed = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server(data), + room_version=room_version, + ) + + # Since they passed in a `parser`, we need to return the type that + # they're expecting instead of just a `JsonDict` + return SendJoinResponse( + auth_events=[ + room_create_event, + user1_invite_membership_event, + ], + state=[ + room_create_event, + creator_membership_event, + user1_invite_membership_event, + ], + event_dict=user1_join_membership_event_signed.get_pdu_json(), + event=user1_join_membership_event_signed, + members_omitted=False, + servers_in_room=[ + self.OTHER_SERVER_NAME, + ], + ) + + if path.startswith("/_matrix/federation/v1/send/") and data is not None: + for pdu in data.get("pdus", []): + event = event_from_pdu_json(pdu, room_version) + collected_pdus_from_hs1_federation_send.add(event.event_id) + + # Just acknowledge everything hs1 is trying to send hs2 + return { + event_from_pdu_json(pdu, room_version).event_id: {} + for pdu in data.get("pdus", []) + } + + raise NotImplementedError( + "We have not mocked a response for `put_json(...)` for the following endpoint yet: " + + f"{destination}{path} with the following body data: {data}" + ) + + self.federation_http_client.put_json.side_effect = put_json + + # User1 joins the room + self.helper.join(remote_room_id, local_user1_id, tok=local_user1_tok) + + # Reset the mocks now that user1 has joined the room + self.federation_http_client.get_json.side_effect = None + self.federation_http_client.put_json.side_effect = None + + # Sync until the local user1 can see that they are now joined to the room + with test_timeout( + 3, + "Unable to find user1's join event in the room", + ): + while True: + response_body, _ = self.do_sync(sync_body, tok=local_user1_tok) + if remote_room_id in response_body["rooms"].keys(): + required_state_map = required_state_json_to_state_map( + response_body["rooms"][remote_room_id]["required_state"] + ) + if ( + required_state_map.get((EventTypes.Member, local_user1_id)) + is not None + ): + break + + # Prevent tight-looping to allow the `test_timeout` to work + time.sleep(0.1) + + # Nothing needs to be sent from hs1 to hs2 since we already let the other + # homeserver know by doing the `/make_join` and `/send_join` dance. + self.assertIncludes( + collected_pdus_from_hs1_federation_send, + set(), + exact=True, + message="Didn't expect any events to be sent from hs1 over federation to hs2", + ) + + return RemoteRoomJoinResult( + remote_room_id=remote_room_id, + room_version=room_version, + remote_room_creator_user_id=room_creator_user_id, + local_user1_id=local_user1_id, + local_user1_tok=local_user1_tok, + state_map=self.get_success( + self.storage_controllers.state.get_current_state(remote_room_id) + ), + ) + + def test_can_join_from_out_of_band_invite(self) -> None: + """ + Test to make sure that we can join a room that we were invited to over + federation; even if our server has never participated in the room before. + """ + self._invite_local_user_to_remote_room_and_join() + + @parameterized.expand( + [("accept invite", Membership.JOIN), ("reject invite", Membership.LEAVE)] + ) + def test_can_x_from_out_of_band_invite_after_we_are_already_participating_in_the_room( + self, _test_description: str, membership_action: str + ) -> None: + """ + Test to make sure that we can do either a) join the room (accept the invite) or + b) reject the invite after being invited to over federation; even if we are + already participating in the room. + + This is a regression test to make sure we stress the scenario where even though + we are already participating in the room, local users can still react to invites + regardless of whether the remote server has told us about the invite event (via + a federation `/send` transaction) and we have de-outliered the invite event. + Previously, we would mistakenly throw an error saying the user wasn't in the + room when they tried to join or reject the invite. + """ + remote_room_join_result = self._invite_local_user_to_remote_room_and_join() + remote_room_id = remote_room_join_result.remote_room_id + room_version = remote_room_join_result.room_version + + # Create another local user + local_user2_id = self.register_user("user2", "pass") + local_user2_tok = self.login(local_user2_id, "pass") + + T = TypeVar("T") + + # PDU's that hs1 sent to hs2 + collected_pdus_from_hs1_federation_send: Set[str] = set() + + async def put_json( + destination: str, + path: str, + args: Optional[QueryParams] = None, + data: Optional[JsonDict] = None, + json_data_callback: Optional[Callable[[], JsonDict]] = None, + long_retries: bool = False, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + backoff_on_404: bool = False, + try_trailing_slash_on_400: bool = False, + parser: Optional[ByteParser[T]] = None, + backoff_on_all_error_codes: bool = False, + ) -> Union[JsonDict, T]: + if path.startswith("/_matrix/federation/v1/send/") and data is not None: + for pdu in data.get("pdus", []): + event = event_from_pdu_json(pdu, room_version) + collected_pdus_from_hs1_federation_send.add(event.event_id) + + # Just acknowledge everything hs1 is trying to send hs2 + return { + event_from_pdu_json(pdu, room_version).event_id: {} + for pdu in data.get("pdus", []) + } + + raise NotImplementedError( + "We have not mocked a response for `put_json(...)` for the following endpoint yet: " + + f"{destination}{path} with the following body data: {data}" + ) + + self.federation_http_client.put_json.side_effect = put_json + + # From the remote homeserver, invite user2 on the local homserver + user2_invite_membership_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": remote_room_join_result.remote_room_creator_user_id, + "depth": 5, + "origin_server_ts": 5, + "type": EventTypes.Member, + "state_key": local_user2_id, + "content": {"membership": Membership.INVITE}, + "auth_events": [ + remote_room_join_result.state_map[ + (EventTypes.Create, "") + ].event_id, + remote_room_join_result.state_map[ + ( + EventTypes.Member, + remote_room_join_result.remote_room_creator_user_id, + ) + ].event_id, + ], + "prev_events": [ + remote_room_join_result.state_map[ + (EventTypes.Member, remote_room_join_result.local_user1_id) + ].event_id + ], + } + ), + room_version=room_version, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/invite/{remote_room_id}/{user2_invite_membership_event.event_id}", + content={ + "event": user2_invite_membership_event.get_dict(), + "invite_room_state": [ + strip_event( + remote_room_join_result.state_map[(EventTypes.Create, "")] + ), + ], + "room_version": room_version.identifier, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [(EventTypes.Member, StateValues.WILDCARD)], + "timeline_limit": 0, + } + } + } + + # Sync until the local user2 can see the invite + with test_timeout( + 3, + "Unable to find user2's invite event in the room", + ): + while True: + response_body, _ = self.do_sync(sync_body, tok=local_user2_tok) + if ( + remote_room_id in response_body["rooms"].keys() + # If they have `invite_state` for the room, they are invited + and len( + response_body["rooms"][remote_room_id].get("invite_state", []) + ) + > 0 + ): + break + + # Prevent tight-looping to allow the `test_timeout` to work + time.sleep(0.1) + + if membership_action == Membership.JOIN: + # User2 joins the room + join_event = self.helper.join( + remote_room_join_result.remote_room_id, + local_user2_id, + tok=local_user2_tok, + ) + expected_pdu_event_id = join_event["event_id"] + elif membership_action == Membership.LEAVE: + # User2 rejects the invite + leave_event = self.helper.leave( + remote_room_join_result.remote_room_id, + local_user2_id, + tok=local_user2_tok, + ) + expected_pdu_event_id = leave_event["event_id"] + else: + raise NotImplementedError( + "This test does not support this membership action yet" + ) + + # Sync until the local user2 can see their new membership in the room + with test_timeout( + 3, + "Unable to find user2's new membership event in the room", + ): + while True: + response_body, _ = self.do_sync(sync_body, tok=local_user2_tok) + if membership_action == Membership.JOIN: + if remote_room_id in response_body["rooms"].keys(): + required_state_map = required_state_json_to_state_map( + response_body["rooms"][remote_room_id]["required_state"] + ) + if ( + required_state_map.get((EventTypes.Member, local_user2_id)) + is not None + ): + break + elif membership_action == Membership.LEAVE: + if remote_room_id not in response_body["rooms"].keys(): + break + else: + raise NotImplementedError( + "This test does not support this membership action yet" + ) + + # Prevent tight-looping to allow the `test_timeout` to work + time.sleep(0.1) + + # Make sure that we let hs2 know about the new membership event + self.assertIncludes( + collected_pdus_from_hs1_federation_send, + {expected_pdu_event_id}, + exact=True, + message="Expected to find the event ID of the user2 membership to be sent from hs1 over federation to hs2", + ) diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index 6a8887fe74..b8dd61d04f 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -24,16 +24,17 @@ from signedjson import key, sign from signedjson.types import BaseKey, SigningKey from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EduTypes, RoomEncryptionAlgorithms from synapse.api.presence import UserPresenceState from synapse.federation.sender.per_destination_queue import MAX_PRESENCE_STATES_PER_EDU from synapse.federation.units import Transaction -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceListUpdater, DeviceWriterHandler from synapse.rest import admin from synapse.rest.client import login from synapse.server import HomeServer +from synapse.storage.databases.main.events_worker import EventMetadata from synapse.types import JsonDict, ReadReceipt from synapse.util import Clock @@ -55,12 +56,15 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase): federation_transport_client=self.federation_transport_client, ) - hs.get_storage_controllers().state.get_current_hosts_in_room = AsyncMock( # type: ignore[method-assign] + self.main_store = hs.get_datastores().main + self.state_controller = hs.get_storage_controllers().state + + self.state_controller.get_current_hosts_in_room = AsyncMock( # type: ignore[method-assign] return_value={"test", "host2"} ) - hs.get_storage_controllers().state.get_current_hosts_in_room_or_partial_state_approximation = ( # type: ignore[method-assign] - hs.get_storage_controllers().state.get_current_hosts_in_room + self.state_controller.get_current_hosts_in_room_or_partial_state_approximation = ( # type: ignore[method-assign] + self.state_controller.get_current_hosts_in_room ) return hs @@ -185,12 +189,15 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase): ], ) - def test_send_receipts_with_backoff(self) -> None: - """Send two receipts in quick succession; the second should be flushed, but - only after 20ms""" + def test_send_receipts_with_backoff_small_room(self) -> None: + """Read receipt in small rooms should not be delayed""" mock_send_transaction = self.federation_transport_client.send_transaction mock_send_transaction.return_value = {} + self.state_controller.get_current_hosts_in_room_or_partial_state_approximation = AsyncMock( # type: ignore[method-assign] + return_value={"test", "host2"} + ) + sender = self.hs.get_federation_sender() receipt = ReadReceipt( "room_id", @@ -206,7 +213,104 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase): # expect a call to send_transaction mock_send_transaction.assert_called_once() - json_cb = mock_send_transaction.call_args[0][1] + self._assert_edu_in_call(mock_send_transaction.call_args[0][1]) + + def test_send_receipts_with_backoff_recent_event(self) -> None: + """Read receipt for a recent message should not be delayed""" + mock_send_transaction = self.federation_transport_client.send_transaction + mock_send_transaction.return_value = {} + + # Pretend this is a big room + self.state_controller.get_current_hosts_in_room_or_partial_state_approximation = AsyncMock( # type: ignore[method-assign] + return_value={"test"} | {f"host{i}" for i in range(20)} + ) + + self.main_store.get_metadata_for_event = AsyncMock( + return_value=EventMetadata( + received_ts=self.clock.time_msec(), + sender="@test:test", + ) + ) + + sender = self.hs.get_federation_sender() + receipt = ReadReceipt( + "room_id", + "m.read", + "user_id", + ["event_id"], + thread_id=None, + data={"ts": 1234}, + ) + self.get_success(sender.send_read_receipt(receipt)) + + self.pump() + + # expect a call to send_transaction for each host + self.assertEqual(mock_send_transaction.call_count, 20) + self._assert_edu_in_call(mock_send_transaction.call_args.args[1]) + + mock_send_transaction.reset_mock() + + def test_send_receipts_with_backoff_sender(self) -> None: + """Read receipt for a message should not be delayed to the sender, but + is delayed to everyone else""" + mock_send_transaction = self.federation_transport_client.send_transaction + mock_send_transaction.return_value = {} + + # Pretend this is a big room + self.state_controller.get_current_hosts_in_room_or_partial_state_approximation = AsyncMock( # type: ignore[method-assign] + return_value={"test"} | {f"host{i}" for i in range(20)} + ) + + self.main_store.get_metadata_for_event = AsyncMock( + return_value=EventMetadata( + received_ts=self.clock.time_msec() - 5 * 60_000, + sender="@test:host1", + ) + ) + + sender = self.hs.get_federation_sender() + receipt = ReadReceipt( + "room_id", + "m.read", + "user_id", + ["event_id"], + thread_id=None, + data={"ts": 1234}, + ) + self.get_success(sender.send_read_receipt(receipt)) + + self.pump() + + # First, expect a call to send_transaction for the sending host + mock_send_transaction.assert_called() + + transaction = mock_send_transaction.call_args_list[0].args[0] + self.assertEqual(transaction.destination, "host1") + self._assert_edu_in_call(mock_send_transaction.call_args_list[0].args[1]) + + # We also expect a call to one of the other hosts, as the first + # destination to wake up. + self.assertEqual(mock_send_transaction.call_count, 2) + self._assert_edu_in_call(mock_send_transaction.call_args.args[1]) + + mock_send_transaction.reset_mock() + + # We now expect to see 18 more transactions to the remaining hosts + # periodically. + for _ in range(18): + self.reactor.advance( + 1.0 + / self.hs.config.ratelimiting.federation_rr_transactions_per_room_per_second + ) + + mock_send_transaction.assert_called_once() + self._assert_edu_in_call(mock_send_transaction.call_args.args[1]) + mock_send_transaction.reset_mock() + + def _assert_edu_in_call(self, json_cb: Callable[[], JsonDict]) -> None: + """Assert that the given `json_cb` from a `send_transaction` has a + receipt in it.""" data = json_cb() self.assertEqual( data["edus"], @@ -226,46 +330,6 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase): } ], ) - mock_send_transaction.reset_mock() - - # send the second RR - receipt = ReadReceipt( - "room_id", - "m.read", - "user_id", - ["other_id"], - thread_id=None, - data={"ts": 1234}, - ) - self.successResultOf(defer.ensureDeferred(sender.send_read_receipt(receipt))) - self.pump() - mock_send_transaction.assert_not_called() - - self.reactor.advance(19) - mock_send_transaction.assert_not_called() - - self.reactor.advance(10) - mock_send_transaction.assert_called_once() - json_cb = mock_send_transaction.call_args[0][1] - data = json_cb() - self.assertEqual( - data["edus"], - [ - { - "edu_type": EduTypes.RECEIPT, - "content": { - "room_id": { - "m.read": { - "user_id": { - "event_ids": ["other_id"], - "data": {"ts": 1234}, - } - } - } - }, - } - ], - ) class FederationSenderPresenceTestCases(HomeserverTestCase): @@ -436,7 +500,7 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): hs.get_datastores().main.get_current_hosts_in_room = get_current_hosts_in_room # type: ignore[assignment] device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) self.device_handler = device_handler # whenever send_transaction is called, record the edu data @@ -490,6 +554,8 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): "devices": [{"device_id": "D1"}], } + assert isinstance(self.device_handler.device_list_updater, DeviceListUpdater) + self.get_success( self.device_handler.device_list_updater.incoming_device_list_update( "host2", @@ -608,7 +674,7 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): self.assertEqual(edu["edu_type"], EduTypes.DEVICE_LIST_UPDATE) c = edu["content"] if stream_id is not None: - self.assertEqual(c["prev_id"], [stream_id]) # type: ignore[unreachable] + self.assertEqual(c["prev_id"], [stream_id]) self.assertGreaterEqual(c["stream_id"], stream_id) stream_id = c["stream_id"] devices = {edu["content"]["device_id"] for edu in self.edus} diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 88261450b1..52fd32ba85 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -20,14 +20,21 @@ # import logging from http import HTTPStatus +from typing import Optional, Union +from unittest.mock import Mock from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor -from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.api.constants import EventTypes, Membership +from synapse.api.errors import FederationError +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.events import EventBase, make_event_from_dict +from synapse.federation.federation_base import event_from_pdu_json +from synapse.http.types import QueryParams +from synapse.logging.context import LoggingContext from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer @@ -38,6 +45,8 @@ from synapse.util import Clock from tests import unittest from tests.unittest import override_config +logger = logging.getLogger(__name__) + class FederationServerTests(unittest.FederatingHomeserverTestCase): servlets = [ @@ -85,10 +94,167 @@ class FederationServerTests(unittest.FederatingHomeserverTestCase): self.assertEqual(500, channel.code, channel.result) +def _create_acl_event(content: JsonDict) -> EventBase: + return make_event_from_dict( + { + "room_id": "!a:b", + "event_id": "$a:b", + "type": "m.room.server_acls", + "sender": "@a:b", + "content": content, + } + ) + + +class MessageAcceptTests(unittest.FederatingHomeserverTestCase): + """ + Tests to make sure that we don't accept flawed events from federation (incoming). + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.http_client = Mock() + return self.setup_test_homeserver(federation_http_client=self.http_client) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + self.store = self.hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + self.federation_event_handler = self.hs.get_federation_event_handler() + + # Create a local room + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + self.room_id = self.helper.create_room_as( + user1_id, tok=user1_tok, is_public=True + ) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(self.room_id) + ) + + # Figure out what the forward extremities in the room are (the most recent + # events that aren't tied into the DAG) + forward_extremity_event_ids = self.get_success( + self.hs.get_datastores().main.get_latest_event_ids_in_room(self.room_id) + ) + + # Join a remote user to the room that will attempt to send bad events + self.remote_bad_user_id = f"@baduser:{self.OTHER_SERVER_NAME}" + self.remote_bad_user_join_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_bad_user_id, + "state_key": self.remote_bad_user_id, + "depth": 1000, + "origin_server_ts": 1, + "type": EventTypes.Member, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + + # Send the join, it should return None (which is not an error) + self.assertEqual( + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, self.remote_bad_user_join_event + ) + ), + None, + ) + + # Make sure we actually joined the room + self.assertEqual( + self.get_success(self.store.get_latest_event_ids_in_room(self.room_id)), + {self.remote_bad_user_join_event.event_id}, + ) + + def test_cant_hide_direct_ancestors(self) -> None: + """ + If you send a message, you must be able to provide the direct + prev_events that said event references. + """ + + async def post_json( + destination: str, + path: str, + data: Optional[JsonDict] = None, + long_retries: bool = False, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + args: Optional[QueryParams] = None, + ) -> Union[JsonDict, list]: + # If it asks us for new missing events, give them NOTHING + if path.startswith("/_matrix/federation/v1/get_missing_events/"): + return {"events": []} + return {} + + self.http_client.post_json = post_json + + # Figure out what the forward extremities in the room are (the most recent + # events that aren't tied into the DAG) + forward_extremity_event_ids = self.get_success( + self.hs.get_datastores().main.get_latest_event_ids_in_room(self.room_id) + ) + + # Now lie about an event's prev_events + lying_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_bad_user_id, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": "hewwo?"}, + "auth_events": [], + "prev_events": ["$missing_prev_event"] + + list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + + with LoggingContext("test-context"): + failure = self.get_failure( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, lying_event + ), + FederationError, + ) + + # on_receive_pdu should throw an error + self.assertEqual( + failure.value.args[0], + ( + "ERROR 403: Your server isn't divulging details about prev_events " + "referenced in this event." + ), + ) + + # Make sure the invalid event isn't there + extrem = self.get_success(self.store.get_latest_event_ids_in_room(self.room_id)) + self.assertEqual(extrem, {self.remote_bad_user_join_event.event_id}) + + class ServerACLsTestCase(unittest.TestCase): def test_blocked_server(self) -> None: e = _create_acl_event({"allow": ["*"], "deny": ["evil.com"]}) - logging.info("ACL event: %s", e.content) + logger.info("ACL event: %s", e.content) server_acl_evalutor = server_acl_evaluator_from_event(e) @@ -102,7 +268,7 @@ class ServerACLsTestCase(unittest.TestCase): def test_block_ip_literals(self) -> None: e = _create_acl_event({"allow_ip_literals": False, "allow": ["*"]}) - logging.info("ACL event: %s", e.content) + logger.info("ACL event: %s", e.content) server_acl_evalutor = server_acl_evaluator_from_event(e) @@ -355,13 +521,73 @@ class SendJoinFederationTests(unittest.FederatingHomeserverTestCase): # is probably sufficient to reassure that the bucket is updated. -def _create_acl_event(content: JsonDict) -> EventBase: - return make_event_from_dict( - { - "room_id": "!a:b", - "event_id": "$a:b", - "type": "m.room.server_acls", - "sender": "@a:b", - "content": content, +class StripUnsignedFromEventsTestCase(unittest.TestCase): + """ + Test to make sure that we handle the raw JSON events from federation carefully and + strip anything that shouldn't be there. + """ + + def test_strip_unauthorized_unsigned_values(self) -> None: + event1 = { + "sender": "@baduser:test.serv", + "state_key": "@baduser:test.serv", + "event_id": "$event1:test.serv", + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.member", + "content": {"membership": "join"}, + "auth_events": [], + "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"}, } - ) + filtered_event = event_from_pdu_json(event1, RoomVersions.V1) + # Make sure unauthorized fields are stripped from unsigned + self.assertNotIn("more warez", filtered_event.unsigned) + + def test_strip_event_maintains_allowed_fields(self) -> None: + event2 = { + "sender": "@baduser:test.serv", + "state_key": "@baduser:test.serv", + "event_id": "$event2:test.serv", + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.member", + "auth_events": [], + "content": {"membership": "join"}, + "unsigned": { + "malicious garbage": "hackz", + "more warez": "more hackz", + "age": 14, + "invite_room_state": [], + }, + } + + filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1) + self.assertIn("age", filtered_event2.unsigned) + self.assertEqual(14, filtered_event2.unsigned["age"]) + self.assertNotIn("more warez", filtered_event2.unsigned) + # Invite_room_state is allowed in events of type m.room.member + self.assertIn("invite_room_state", filtered_event2.unsigned) + self.assertEqual([], filtered_event2.unsigned["invite_room_state"]) + + def test_strip_event_removes_fields_based_on_event_type(self) -> None: + event3 = { + "sender": "@baduser:test.serv", + "state_key": "@baduser:test.serv", + "event_id": "$event3:test.serv", + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.power_levels", + "content": {}, + "auth_events": [], + "unsigned": { + "malicious garbage": "hackz", + "more warez": "more hackz", + "age": 14, + "invite_room_state": [], + }, + } + filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1) + self.assertIn("age", filtered_event3.unsigned) + # Invite_room_state field is only permitted in event type m.room.member + self.assertNotIn("invite_room_state", filtered_event3.unsigned) + self.assertNotIn("more warez", filtered_event3.unsigned) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 166a01c1a2..14345be0f3 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -21,7 +21,7 @@ from collections import OrderedDict from typing import Any, Dict, List, Optional -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersion, RoomVersions diff --git a/tests/handlers/test_admin.py b/tests/handlers/test_admin.py index 9ff853a83d..906d241f1a 100644 --- a/tests/handlers/test_admin.py +++ b/tests/handlers/test_admin.py @@ -22,7 +22,7 @@ from collections import Counter from unittest.mock import Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin import synapse.storage diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 1eec0d43b7..a47b03b143 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -25,7 +25,7 @@ from unittest.mock import AsyncMock, Mock from parameterized import parameterized from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin import synapse.storage @@ -43,6 +43,7 @@ from synapse.types import ( MultiWriterStreamToken, RoomStreamToken, StreamKeyType, + UserID, ) from synapse.util import Clock from synapse.util.stringutils import random_string @@ -1009,7 +1010,7 @@ class ApplicationServicesHandlerSendEventsTestCase(unittest.HomeserverTestCase): appservice = ApplicationService( token=random_string(10), id=random_string(10), - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), rate_limited=False, namespaces=namespaces, supports_ephemeral=True, @@ -1087,7 +1088,7 @@ class ApplicationServicesHandlerDeviceListsTestCase(unittest.HomeserverTestCase) appservice = ApplicationService( token=random_string(10), id=random_string(10), - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), rate_limited=False, namespaces={ ApplicationService.NS_USERS: [ @@ -1151,9 +1152,9 @@ class ApplicationServicesHandlerOtkCountsTestCase(unittest.HomeserverTestCase): # Define an application service for the tests self._service_token = "VERYSECRET" self._service = ApplicationService( - self._service_token, - "as1", - "@as.sender:test", + token=self._service_token, + id="as1", + sender=UserID.from_string("@as.sender:test"), namespaces={ "users": [ {"regex": "@_as_.*:test", "exclusive": True}, @@ -1165,12 +1166,23 @@ class ApplicationServicesHandlerOtkCountsTestCase(unittest.HomeserverTestCase): self.hs.get_datastores().main.services_cache = [self._service] # Register some appservice users - self._sender_user, self._sender_device = self.register_appservice_user( + user_id, device_id = self.register_appservice_user( "as.sender", self._service_token ) - self._namespaced_user, self._namespaced_device = self.register_appservice_user( + # With MSC4190 enabled, there will not be a device created + # during AS registration. However MSC4190 is not enabled + # in this test. It may become the default behaviour in the + # future, in which case this test will need to be updated. + assert device_id is not None + self._sender_user = user_id + self._sender_device = device_id + + user_id, device_id = self.register_appservice_user( "_as_user1", self._service_token ) + assert device_id is not None + self._namespaced_user = user_id + self._namespaced_device = device_id # Register a real user as well. self._real_user = self.register_user("real.user", "meow") diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py index c417431e85..0d9940c63e 100644 --- a/tests/handlers/test_auth.py +++ b/tests/handlers/test_auth.py @@ -23,7 +23,7 @@ from unittest.mock import AsyncMock import pymacaroons -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import AuthError, ResourceLimitError from synapse.rest import admin diff --git a/tests/handlers/test_cas.py b/tests/handlers/test_cas.py index f41f7d36ad..9de5e67863 100644 --- a/tests/handlers/test_cas.py +++ b/tests/handlers/test_cas.py @@ -21,7 +21,7 @@ from typing import Any, Dict from unittest.mock import AsyncMock, Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.handlers.cas import CasResponse from synapse.server import HomeServer diff --git a/tests/handlers/test_deactivate_account.py b/tests/handlers/test_deactivate_account.py index d7b54383db..b7b8387780 100644 --- a/tests/handlers/test_deactivate_account.py +++ b/tests/handlers/test_deactivate_account.py @@ -19,7 +19,7 @@ # # -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules, Membership from synapse.push.rulekinds import PRIORITY_CLASS_MAP diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 080e6a7028..195cdfeaef 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -24,17 +24,17 @@ from typing import Optional from unittest import mock from twisted.internet.defer import ensureDeferred -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import RoomEncryptionAlgorithms from synapse.api.errors import NotFoundError, SynapseError from synapse.appservice import ApplicationService -from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceHandler +from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceWriterHandler from synapse.rest import admin from synapse.rest.client import devices, login, register from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex -from synapse.types import JsonDict, create_requester +from synapse.types import JsonDict, UserID, create_requester from synapse.util import Clock from synapse.util.task_scheduler import TaskScheduler @@ -53,7 +53,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): application_service_api=self.appservice_api, ) handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) + assert isinstance(handler, DeviceWriterHandler) self.handler = handler self.store = hs.get_datastores().main self.device_message_handler = hs.get_device_message_handler() @@ -229,7 +229,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): # queue a bunch of messages in the inbox requester = create_requester(sender, device_id=DEVICE_ID) - for i in range(DeviceHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT + 10): + for i in range(DeviceWriterHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT + 10): self.get_success( self.device_message_handler.send_device_message( requester, "message_type", {receiver: {"*": {"val": i}}} @@ -419,7 +419,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( @@ -462,7 +462,7 @@ class DehydrationTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: hs = self.setup_test_homeserver("server") handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) + assert isinstance(handler, DeviceWriterHandler) self.handler = handler self.message_handler = hs.get_device_message_handler() self.registration = hs.get_registration_handler() diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 4a3e36ffde..4d6243ef74 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -22,7 +22,7 @@ from typing import Any, Awaitable, Callable, Dict from unittest.mock import AsyncMock, Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.api.errors import synapse.rest.admin @@ -587,6 +587,7 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase): self.room_list_handler = hs.get_room_list_handler() self.directory_handler = hs.get_directory_handler() + @unittest.override_config({"room_list_publication_rules": [{"action": "allow"}]}) def test_disabling_room_list(self) -> None: self.room_list_handler.enable_room_list_search = True self.directory_handler.enable_room_list_search = True diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index e67efcc17f..fda485d413 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -26,12 +26,12 @@ from unittest import mock from parameterized import parameterized from signedjson import key as key, sign as sign -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import RoomEncryptionAlgorithms from synapse.api.errors import Codes, SynapseError from synapse.appservice import ApplicationService -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex from synapse.types import JsonDict, UserID @@ -856,7 +856,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): self.get_success(self.handler.upload_signing_keys_for_user(local_user, keys1)) device_handler = self.hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) e = self.get_failure( device_handler.check_device_registered( user_id=local_user, @@ -1457,7 +1457,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( @@ -1525,7 +1525,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( @@ -1751,7 +1751,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( @@ -1896,3 +1896,153 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): self.assertEqual( remaining_key_ids, {"AAAAAAAAAA", "BAAAAA", "BAAAAB", "BAAAAAAAAA"} ) + + @override_config( + { + "experimental_features": { + "msc4263_limit_key_queries_to_users_who_share_rooms": True + } + } + ) + def test_query_devices_remote_restricted_not_in_shared_room(self) -> None: + """Tests that querying keys for a remote user that we don't share a room + with returns nothing. + """ + + remote_user_id = "@test:other" + local_user_id = "@test:test" + + # Do *not* pretend we're sharing a room with the user we're querying. + + remote_master_key = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" + remote_self_signing_key = "QeIiFEjluPBtI7WQdG365QKZcFs9kqmHir6RBD0//nQ" + + self.hs.get_federation_client().query_client_keys = mock.AsyncMock( # type: ignore[method-assign] + return_value={ + "device_keys": {remote_user_id: {}}, + "master_keys": { + remote_user_id: { + "user_id": remote_user_id, + "usage": ["master"], + "keys": {"ed25519:" + remote_master_key: remote_master_key}, + }, + }, + "self_signing_keys": { + remote_user_id: { + "user_id": remote_user_id, + "usage": ["self_signing"], + "keys": { + "ed25519:" + + remote_self_signing_key: remote_self_signing_key + }, + } + }, + } + ) + + e2e_handler = self.hs.get_e2e_keys_handler() + + query_result = self.get_success( + e2e_handler.query_devices( + { + "device_keys": {remote_user_id: []}, + }, + timeout=10, + from_user_id=local_user_id, + from_device_id="some_device_id", + ) + ) + + self.assertEqual( + query_result, + { + "device_keys": {}, + "failures": {}, + "master_keys": {}, + "self_signing_keys": {}, + "user_signing_keys": {}, + }, + ) + + @override_config( + { + "experimental_features": { + "msc4263_limit_key_queries_to_users_who_share_rooms": True + } + } + ) + def test_query_devices_remote_restricted_in_shared_room(self) -> None: + """Tests that querying keys for a remote user that we share a room + with returns the cross signing keys correctly. + """ + + remote_user_id = "@test:other" + local_user_id = "@test:test" + + # Pretend we're sharing a room with the user we're querying. If not, + # `query_devices` will filter out the user ID and `_query_devices_for_destination` + # will return early. + self.store.do_users_share_a_room_joined_or_invited = mock.AsyncMock( # type: ignore[method-assign] + return_value=[remote_user_id] + ) + self.store.get_rooms_for_user = mock.AsyncMock(return_value={"some_room_id"}) + + remote_master_key = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" + remote_self_signing_key = "QeIiFEjluPBtI7WQdG365QKZcFs9kqmHir6RBD0//nQ" + + self.hs.get_federation_client().query_user_devices = mock.AsyncMock( # type: ignore[method-assign] + return_value={ + "user_id": remote_user_id, + "stream_id": 1, + "devices": [], + "master_key": { + "user_id": remote_user_id, + "usage": ["master"], + "keys": {"ed25519:" + remote_master_key: remote_master_key}, + }, + "self_signing_key": { + "user_id": remote_user_id, + "usage": ["self_signing"], + "keys": { + "ed25519:" + remote_self_signing_key: remote_self_signing_key + }, + }, + } + ) + + e2e_handler = self.hs.get_e2e_keys_handler() + + query_result = self.get_success( + e2e_handler.query_devices( + { + "device_keys": {remote_user_id: []}, + }, + timeout=10, + from_user_id=local_user_id, + from_device_id="some_device_id", + ) + ) + + self.assertEqual(query_result["failures"], {}) + self.assertEqual( + query_result["master_keys"], + { + remote_user_id: { + "user_id": remote_user_id, + "usage": ["master"], + "keys": {"ed25519:" + remote_master_key: remote_master_key}, + } + }, + ) + self.assertEqual( + query_result["self_signing_keys"], + { + remote_user_id: { + "user_id": remote_user_id, + "usage": ["self_signing"], + "keys": { + "ed25519:" + remote_self_signing_key: remote_self_signing_key + }, + } + }, + ) diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 3ec46402b7..9b280659ab 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -23,7 +23,7 @@ import copy from unittest import mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import SynapseError from synapse.server import HomeServer diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index b64a8a86a2..4de90e6578 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -24,7 +24,7 @@ from unittest import TestCase from unittest.mock import AsyncMock, Mock, patch from twisted.internet.defer import Deferred -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes from synapse.api.errors import ( diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 5db10fa74c..02dd60e76d 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -21,7 +21,7 @@ from typing import Optional from unittest import mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import AuthError, StoreError from synapse.api.room_versions import RoomVersion @@ -375,7 +375,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): In this test, we pretend we are processing a "pulled" event via backfill. The pulled event succesfully processes and the backward - extremeties are updated along with clearing out any failed pull attempts + extremities are updated along with clearing out any failed pull attempts for those old extremities. We check that we correctly cleared failed pull attempts of the @@ -807,6 +807,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): OTHER_USER = f"@user:{self.OTHER_SERVER_NAME}" main_store = self.hs.get_datastores().main + state_deletion_store = self.hs.get_datastores().state_deletion # Create the room. kermit_user_id = self.register_user("kermit", "test") @@ -958,7 +959,9 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): bert_member_event.event_id: bert_member_event, rejected_kick_event.event_id: rejected_kick_event, }, - state_res_store=StateResolutionStore(main_store), + state_res_store=StateResolutionStore( + main_store, state_deletion_store + ), ) ), [bert_member_event.event_id, rejected_kick_event.event_id], @@ -1003,7 +1006,9 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): rejected_power_levels_event.event_id, ], event_map={}, - state_res_store=StateResolutionStore(main_store), + state_res_store=StateResolutionStore( + main_store, state_deletion_store + ), full_conflicted_set=set(), ) ), diff --git a/tests/handlers/test_message.py b/tests/handlers/test_message.py index 76ab83d1f7..0a1092eae4 100644 --- a/tests/handlers/test_message.py +++ b/tests/handlers/test_message.py @@ -21,7 +21,7 @@ import logging from typing import Tuple -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError @@ -204,46 +204,19 @@ class EventCreationTestCase(unittest.HomeserverTestCase): self.assertEqual(len(events), 2) self.assertEqual(events[0].event_id, events[1].event_id) - def test_when_empty_prev_events_allowed_create_event_with_empty_prev_events( + def test_reject_event_with_empty_prev_events( self, ) -> None: - """When we set allow_no_prev_events=True, should be able to create a - event without any prev_events (only auth_events). """ - # Create a member event we can use as an auth_event - memberEvent, _ = self._create_and_persist_member_event() - - # Try to create the event with empty prev_events bit with some auth_events - event, _ = self.get_success( - self.handler.create_event( - self.requester, - { - "type": EventTypes.Message, - "room_id": self.room_id, - "sender": self.requester.user.to_string(), - "content": {"msgtype": "m.text", "body": random_string(5)}, - }, - # Empty prev_events is the key thing we're testing here - prev_event_ids=[], - # But with some auth_events - auth_event_ids=[memberEvent.event_id], - # Allow no prev_events! - allow_no_prev_events=True, - ) - ) - self.assertIsNotNone(event) - - def test_when_empty_prev_events_not_allowed_reject_event_with_empty_prev_events( - self, - ) -> None: - """When we set allow_no_prev_events=False, shouldn't be able to create a - event without any prev_events even if it has auth_events. Expect an - exception to be raised. + Shouldn't be able to create an event without any `prev_events` even if it has + `auth_events`. Expect an exception to be raised. """ # Create a member event we can use as an auth_event memberEvent, _ = self._create_and_persist_member_event() # Try to create the event with empty prev_events but with some auth_events + # + # We expect the test to fail because empty prev_events are not allowed self.get_failure( self.handler.create_event( self.requester, @@ -257,35 +230,6 @@ class EventCreationTestCase(unittest.HomeserverTestCase): prev_event_ids=[], # But with some auth_events auth_event_ids=[memberEvent.event_id], - # We expect the test to fail because empty prev_events are not - # allowed here! - allow_no_prev_events=False, - ), - AssertionError, - ) - - def test_when_empty_prev_events_allowed_reject_event_with_empty_prev_events_and_auth_events( - self, - ) -> None: - """When we set allow_no_prev_events=True, should be able to create a - event without any prev_events or auth_events. Expect an exception to be - raised. - """ - # Try to create the event with empty prev_events and empty auth_events - self.get_failure( - self.handler.create_event( - self.requester, - { - "type": EventTypes.Message, - "room_id": self.room_id, - "sender": self.requester.user.to_string(), - "content": {"msgtype": "m.text", "body": random_string(5)}, - }, - prev_event_ids=[], - # The event should be rejected when there are no auth_events - auth_event_ids=[], - # Allow no prev_events! - allow_no_prev_events=True, ), AssertionError, ) diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 5b5dc713d1..d24614f6a3 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -19,12 +19,17 @@ # # +import json +import threading +import time from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, HTTPServer from io import BytesIO -from typing import Any, Dict, Optional, Union +from typing import Any, ClassVar, Coroutine, Dict, Generator, Optional, TypeVar, Union from unittest.mock import ANY, AsyncMock, Mock from urllib.parse import parse_qs +from parameterized.parameterized import parameterized_class from signedjson.key import ( encode_verify_key_base64, generate_signing_key, @@ -32,26 +37,27 @@ from signedjson.key import ( ) from signedjson.sign import sign_json -from twisted.test.proto_helpers import MemoryReactor -from twisted.web.http_headers import Headers -from twisted.web.iweb import IResponse +from twisted.internet.defer import Deferred, ensureDeferred +from twisted.internet.testing import MemoryReactor +from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.errors import ( AuthError, Codes, + HttpResponseException, InvalidClientTokenError, - OAuthInsufficientScopeError, SynapseError, ) +from synapse.appservice import ApplicationService from synapse.http.site import SynapseRequest from synapse.rest import admin from synapse.rest.client import account, devices, keys, login, logout, register from synapse.server import HomeServer -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, UserID, create_requester from synapse.util import Clock from tests.server import FakeChannel -from tests.test_utils import FakeResponse, get_awaitable_result +from tests.test_utils import get_awaitable_result from tests.unittest import HomeserverTestCase, override_config, skip_unless from tests.utils import HAS_AUTHLIB, checked_cast, mock_getRawHeaders @@ -71,11 +77,7 @@ JWKS_URI = ISSUER + ".well-known/jwks.json" INTROSPECTION_ENDPOINT = ISSUER + "introspect" SYNAPSE_ADMIN_SCOPE = "urn:synapse:admin:*" -MATRIX_USER_SCOPE = "urn:matrix:org.matrix.msc2967.client:api:*" -MATRIX_GUEST_SCOPE = "urn:matrix:org.matrix.msc2967.client:api:guest" -MATRIX_DEVICE_SCOPE_PREFIX = "urn:matrix:org.matrix.msc2967.client:device:" DEVICE = "AABBCCDD" -MATRIX_DEVICE_SCOPE = MATRIX_DEVICE_SCOPE_PREFIX + DEVICE SUBJECT = "abc-def-ghi" USERNAME = "test-user" USER_ID = "@" + USERNAME + ":" + SERVER_NAME @@ -105,15 +107,27 @@ async def get_json(url: str) -> JsonDict: @skip_unless(HAS_AUTHLIB, "requires authlib") +@parameterized_class( + ("device_scope_prefix", "api_scope"), + [ + ("urn:matrix:client:device:", "urn:matrix:client:api:*"), + ( + "urn:matrix:org.matrix.msc2967.client:device:", + "urn:matrix:org.matrix.msc2967.client:api:*", + ), + ], +) class MSC3861OAuthDelegation(HomeserverTestCase): + device_scope_prefix: ClassVar[str] + api_scope: ClassVar[str] + + @property + def device_scope(self) -> str: + return self.device_scope_prefix + DEVICE + servlets = [ account.register_servlets, - devices.register_servlets, keys.register_servlets, - register.register_servlets, - login.register_servlets, - logout.register_servlets, - admin.register_servlets, ] def default_config(self) -> Dict[str, Any]: @@ -144,11 +158,30 @@ class MSC3861OAuthDelegation(HomeserverTestCase): self.auth = checked_cast(MSC3861DelegatedAuth, hs.get_auth()) + self._rust_client = Mock(spec=["post"]) + self.auth._rust_http_client = self._rust_client + return hs + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Provision the user and the device we use in the tests. + store = homeserver.get_datastores().main + self.get_success(store.register_user(USER_ID)) + self.get_success( + store.store_device(USER_ID, DEVICE, initial_device_display_name=None) + ) + + def _set_introspection_returnvalue(self, response_value: Any) -> AsyncMock: + self._rust_client.post = mock = AsyncMock( + return_value=json.dumps(response_value).encode("utf-8") + ) + return mock + def _assertParams(self) -> None: """Assert that the request parameters are correct.""" - params = parse_qs(self.http_client.request.call_args[1]["data"].decode("utf-8")) + params = parse_qs(self._rust_client.post.call_args[1]["request_body"]) self.assertEqual(params["token"], ["mockAccessToken"]) self.assertEqual(params["client_id"], [CLIENT_ID]) self.assertEqual(params["client_secret"], [CLIENT_SECRET]) @@ -156,128 +189,125 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_inactive_token(self) -> None: """The handler should return a 403 where the token is inactive.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={"active": False}, - ) - ) + self._set_introspection_returnvalue({"active": False}) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_no_scope(self) -> None: """The handler should return a 403 where no scope is given.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={"active": True}, - ) - ) + self._set_introspection_returnvalue({"active": True}) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_user_no_subject(self) -> None: """The handler should return a 500 when no subject is present.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={"active": True, "scope": " ".join([MATRIX_USER_SCOPE])}, - ) + self._set_introspection_returnvalue( + {"active": True, "scope": " ".join([self.api_scope])}, ) + request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_no_user_scope(self) -> None: """The handler should return a 500 when no subject is present.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_DEVICE_SCOPE]), - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.device_scope]), + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_admin_not_user(self) -> None: """The handler should raise when the scope has admin right but not user.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([SYNAPSE_ADMIN_SCOPE]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_admin(self) -> None: """The handler should return a requester with admin rights.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE, MATRIX_USER_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([SYNAPSE_ADMIN_SCOPE, self.api_scope]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -290,26 +320,24 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_active_admin_highest_privilege(self) -> None: """The handler should resolve to the most permissive scope.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join( - [SYNAPSE_ADMIN_SCOPE, MATRIX_USER_SCOPE, MATRIX_GUEST_SCOPE] - ), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([SYNAPSE_ADMIN_SCOPE, self.api_scope]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -322,24 +350,24 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_active_user(self) -> None: """The handler should return a requester with normal user rights.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_USER_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.api_scope]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -352,24 +380,62 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_active_user_with_device(self) -> None: """The handler should return a requester with normal user rights and a device ID.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_USER_SCOPE, MATRIX_DEVICE_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.api_scope, self.device_scope]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, + ) + self._assertParams() + self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) + self.assertEqual(requester.is_guest, False) + self.assertEqual( + get_awaitable_result(self.auth.is_server_admin(requester)), False + ) + self.assertEqual(requester.device_id, DEVICE) + + def test_active_user_with_device_explicit_device_id(self) -> None: + """The handler should return a requester with normal user rights and a device ID, given explicitly, as supported by MAS 0.15+""" + + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.api_scope]), + "device_id": DEVICE, + "username": USERNAME, + } + ) + request = Mock(args={}) + request.args[b"access_token"] = [b"mockAccessToken"] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + requester = self.get_success(self.auth.get_user_by_req(request)) + self.http_client.get_json.assert_called_once_with(WELL_KNOWN) + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, + ) + # It should have called with the 'X-MAS-Supports-Device-Id: 1' header + self.assertEqual( + self._rust_client.post.call_args[1]["headers"].get( + "X-MAS-Supports-Device-Id", + ), + "1", ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -382,90 +448,25 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_multiple_devices(self) -> None: """The handler should raise an error if multiple devices are found in the scope.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join( - [ - MATRIX_USER_SCOPE, - f"{MATRIX_DEVICE_SCOPE_PREFIX}AABBCC", - f"{MATRIX_DEVICE_SCOPE_PREFIX}DDEEFF", - ] - ), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join( + [ + self.api_scope, + f"{self.device_scope_prefix}AABBCC", + f"{self.device_scope_prefix}DDEEFF", + ] + ), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), AuthError) - def test_active_guest_not_allowed(self) -> None: - """The handler should return an insufficient scope error.""" - - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_GUEST_SCOPE, MATRIX_DEVICE_SCOPE]), - "username": USERNAME, - }, - ) - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - error = self.get_failure( - self.auth.get_user_by_req(request), OAuthInsufficientScopeError - ) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY - ) - self._assertParams() - self.assertEqual( - getattr(error.value, "headers", {})["WWW-Authenticate"], - 'Bearer error="insufficient_scope", scope="urn:matrix:org.matrix.msc2967.client:api:*"', - ) - - def test_active_guest_allowed(self) -> None: - """The handler should return a requester with guest user rights and a device ID.""" - - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_GUEST_SCOPE, MATRIX_DEVICE_SCOPE]), - "username": USERNAME, - }, - ) - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success( - self.auth.get_user_by_req(request, allow_guest=True) - ) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, True) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), False - ) - self.assertEqual(requester.device_id, DEVICE) - def test_unavailable_introspection_endpoint(self) -> None: """The handler should return an internal server error.""" request = Mock(args={}) @@ -473,33 +474,67 @@ class MSC3861OAuthDelegation(HomeserverTestCase): request.requestHeaders.getRawHeaders = mock_getRawHeaders() # The introspection endpoint is returning an error. - self.http_client.request = AsyncMock( - return_value=FakeResponse(code=500, body=b"Internal Server Error") - ) - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint request fails. - self.http_client.request = AsyncMock(side_effect=Exception()) - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint does not return a JSON object. - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, payload=["this is an array", "not an object"] + self._rust_client.post = AsyncMock( + side_effect=HttpResponseException( + code=500, msg="Internal Server Error", response=b"{}" ) ) error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) self.assertEqual(error.value.code, 503) - # The introspection endpoint does not return valid JSON. - self.http_client.request = AsyncMock( - return_value=FakeResponse(code=200, body=b"this is not valid JSON") - ) + # The introspection endpoint request fails. + self._rust_client.post = AsyncMock(side_effect=Exception()) error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) self.assertEqual(error.value.code, 503) + # The introspection endpoint does not return a JSON object. + self._set_introspection_returnvalue(["this is an array", "not an object"]) + + error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) + self.assertEqual(error.value.code, 503) + + # The introspection endpoint does not return valid JSON. + self._set_introspection_returnvalue("this is not valid JSON") + + error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) + self.assertEqual(error.value.code, 503) + + def test_cached_expired_introspection(self) -> None: + """The handler should raise an error if the introspection response gives + an expiry time, the introspection response is cached and then the entry is + re-requested after it has expired.""" + + introspection_mock = self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join( + [ + self.api_scope, + f"{self.device_scope_prefix}AABBCC", + ] + ), + "username": USERNAME, + "expires_in": 60, + } + ) + + request = Mock(args={}) + request.args[b"access_token"] = [b"mockAccessToken"] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + + # The first CS-API request causes a successful introspection + self.get_success(self.auth.get_user_by_req(request)) + self.assertEqual(introspection_mock.call_count, 1) + + # Sleep for 60 seconds so the token expires. + self.reactor.advance(60.0) + + # Now the CS-API request fails because the token expired + self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) + # Ensure another introspection request was not sent + self.assertEqual(introspection_mock.call_count, 1) + def make_device_keys(self, user_id: str, device_id: str) -> JsonDict: # We only generate a master key to simplify the test. master_signing_key = generate_signing_key(device_id) @@ -520,16 +555,13 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_cross_signing(self) -> None: """Try uploading device keys with OAuth delegation enabled.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_USER_SCOPE, MATRIX_DEVICE_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.api_scope, self.device_scope]), + "username": USERNAME, + } ) keys_upload_body = self.make_device_keys(USER_ID, DEVICE) channel = self.make_request( @@ -552,6 +584,531 @@ class MSC3861OAuthDelegation(HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) + def test_admin_token(self) -> None: + """The handler should return a requester with admin rights when admin_token is used.""" + self._set_introspection_returnvalue({"active": False}) + + request = Mock(args={}) + request.args[b"access_token"] = [b"admin_token_value"] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + requester = self.get_success(self.auth.get_user_by_req(request)) + self.assertEqual( + requester.user.to_string(), + OIDC_ADMIN_USERID, + ) + self.assertEqual(requester.is_guest, False) + self.assertEqual(requester.device_id, None) + self.assertEqual( + get_awaitable_result(self.auth.is_server_admin(requester)), True + ) + + # There should be no call to the introspection endpoint + self._rust_client.post.assert_not_called() + + @override_config({"mau_stats_only": True}) + def test_request_tracking(self) -> None: + """Using an access token should update the client_ips and MAU tables.""" + # To start, there are no MAU users. + store = self.hs.get_datastores().main + mau = self.get_success(store.get_monthly_active_count()) + self.assertEqual(mau, 0) + + known_token = "token-token-GOOD-:)" + + async def mock_http_client_request( + url: str, request_body: str, **kwargs: Any + ) -> bytes: + """Mocked auth provider response.""" + token = parse_qs(request_body)["token"][0] + if token == known_token: + return json.dumps( + { + "active": True, + "scope": self.api_scope, + "sub": SUBJECT, + "username": USERNAME, + }, + ).encode("utf-8") + + return json.dumps({"active": False}).encode("utf-8") + + self._rust_client.post = mock_http_client_request + + EXAMPLE_IPV4_ADDR = "123.123.123.123" + EXAMPLE_USER_AGENT = "httprettygood" + + # First test a known access token + channel = FakeChannel(self.site, self.reactor) + # type-ignore: FakeChannel is a mock of an HTTPChannel, not a proper HTTPChannel + req = SynapseRequest(channel, self.site, self.hs.hostname) # type: ignore[arg-type] + req.client.host = EXAMPLE_IPV4_ADDR + req.requestHeaders.addRawHeader("Authorization", f"Bearer {known_token}") + req.requestHeaders.addRawHeader("User-Agent", EXAMPLE_USER_AGENT) + req.content = BytesIO(b"") + req.requestReceived( + b"GET", + b"/_matrix/client/v3/account/whoami", + b"1.1", + ) + channel.await_result() + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + self.assertEqual(channel.json_body["user_id"], USER_ID, channel.json_body) + + # Expect to see one MAU entry, from the first request + mau = self.get_success(store.get_monthly_active_count()) + self.assertEqual(mau, 1) + + conn_infos = self.get_success( + store.get_user_ip_and_agents(UserID.from_string(USER_ID)) + ) + self.assertEqual(len(conn_infos), 1, conn_infos) + conn_info = conn_infos[0] + self.assertEqual(conn_info["access_token"], known_token) + self.assertEqual(conn_info["ip"], EXAMPLE_IPV4_ADDR) + self.assertEqual(conn_info["user_agent"], EXAMPLE_USER_AGENT) + + # Now test MAS making a request using the special __oidc_admin token + MAS_IPV4_ADDR = "127.0.0.1" + MAS_USER_AGENT = "masmasmas" + + channel = FakeChannel(self.site, self.reactor) + req = SynapseRequest(channel, self.site, self.hs.hostname) # type: ignore[arg-type] + req.client.host = MAS_IPV4_ADDR + req.requestHeaders.addRawHeader( + "Authorization", f"Bearer {self.auth._admin_token()}" + ) + req.requestHeaders.addRawHeader("User-Agent", MAS_USER_AGENT) + req.content = BytesIO(b"") + req.requestReceived( + b"GET", + b"/_matrix/client/v3/account/whoami", + b"1.1", + ) + channel.await_result() + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + self.assertEqual( + channel.json_body["user_id"], OIDC_ADMIN_USERID, channel.json_body + ) + + # Still expect to see one MAU entry, from the first request + mau = self.get_success(store.get_monthly_active_count()) + self.assertEqual(mau, 1) + + conn_infos = self.get_success( + store.get_user_ip_and_agents(UserID.from_string(OIDC_ADMIN_USERID)) + ) + self.assertEqual(conn_infos, []) + + +class FakeMasHandler(BaseHTTPRequestHandler): + server: "FakeMasServer" + + def do_POST(self) -> None: + self.server.calls += 1 + + if self.path != "/oauth2/introspect": + self.send_response(404) + self.end_headers() + self.wfile.close() + return + + auth = self.headers.get("Authorization") + if auth is None or auth != f"Bearer {self.server.secret}": + self.send_response(401) + self.end_headers() + self.wfile.close() + return + + content_length = self.headers.get("Content-Length") + if content_length is None: + self.send_response(400) + self.end_headers() + self.wfile.close() + return + + raw_body = self.rfile.read(int(content_length)) + body = parse_qs(raw_body) + param = body.get(b"token") + if param is None: + self.send_response(400) + self.end_headers() + self.wfile.close() + return + + self.server.last_token_seen = param[0].decode("utf-8") + + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(self.server.introspection_response).encode("utf-8")) + + def log_message(self, format: str, *args: Any) -> None: + # Don't log anything; by default, the server logs to stderr + pass + + +class FakeMasServer(HTTPServer): + """A fake MAS server for testing. + + This opens a real HTTP server on a random port, on a separate thread. + """ + + introspection_response: JsonDict = {} + """Determines what the response to the introspection endpoint will be.""" + + secret: str = "verysecret" + """The shared secret used to authenticate the introspection endpoint.""" + + last_token_seen: Optional[str] = None + """What is the last access token seen by the introspection endpoint.""" + + calls: int = 0 + """How many times has the introspection endpoint been called.""" + + _thread: threading.Thread + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), FakeMasHandler) + + self._thread = threading.Thread( + target=self.serve_forever, + name="FakeMasServer", + kwargs={"poll_interval": 0.01}, + daemon=True, + ) + self._thread.start() + + def shutdown(self) -> None: + super().shutdown() + self._thread.join() + + @property + def endpoint(self) -> str: + return f"http://127.0.0.1:{self.server_port}/" + + +T = TypeVar("T") + + +@parameterized_class( + ("device_scope_prefix", "api_scope"), + [ + ("urn:matrix:client:device:", "urn:matrix:client:api:*"), + ( + "urn:matrix:org.matrix.msc2967.client:device:", + "urn:matrix:org.matrix.msc2967.client:api:*", + ), + ], +) +class MasAuthDelegation(HomeserverTestCase): + server: FakeMasServer + device_scope_prefix: ClassVar[str] + api_scope: ClassVar[str] + + @property + def device_scope(self) -> str: + return self.device_scope_prefix + DEVICE + + def till_deferred_has_result( + self, + awaitable: Union[ + "Coroutine[Deferred[Any], Any, T]", + "Generator[Deferred[Any], Any, T]", + "Deferred[T]", + ], + ) -> "Deferred[T]": + """Wait until a deferred has a result. + + This is useful because the Rust HTTP client will resolve the deferred + using reactor.callFromThread, which are only run when we call + reactor.advance. + """ + deferred = ensureDeferred(awaitable) + tries = 0 + while not deferred.called: + time.sleep(0.1) + self.reactor.advance(0) + tries += 1 + if tries > 100: + raise Exception("Timed out waiting for deferred to resolve") + + return deferred + + def default_config(self) -> Dict[str, Any]: + config = super().default_config() + config["public_baseurl"] = BASE_URL + config["disable_registration"] = True + config["matrix_authentication_service"] = { + "enabled": True, + "endpoint": self.server.endpoint, + "secret": self.server.secret, + } + return config + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.server = FakeMasServer() + hs = self.setup_test_homeserver() + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + self._auth = checked_cast(MasDelegatedAuth, hs.get_auth()) + return hs + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Provision the user and the device we use in the tests. + store = homeserver.get_datastores().main + self.get_success(store.register_user(USER_ID)) + self.get_success( + store.store_device(USER_ID, DEVICE, initial_device_display_name=None) + ) + + def tearDown(self) -> None: + self.server.shutdown() + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + + def test_simple_introspection(self) -> None: + self.server.introspection_response = { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.api_scope, self.device_scope]), + "username": USERNAME, + "expires_in": 60, + } + + requester = self.get_success( + self.till_deferred_has_result( + self._auth.get_user_by_access_token("some_token") + ) + ) + + self.assertEquals(requester.user.to_string(), USER_ID) + self.assertEquals(requester.device_id, DEVICE) + self.assertFalse(self.get_success(self._auth.is_server_admin(requester))) + + self.assertEquals( + self.server.last_token_seen, + "some_token", + ) + + def test_unexpiring_token(self) -> None: + self.server.introspection_response = { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.api_scope, self.device_scope]), + "username": USERNAME, + } + + requester = self.get_success( + self.till_deferred_has_result( + self._auth.get_user_by_access_token("some_token") + ) + ) + + self.assertEquals(requester.user.to_string(), USER_ID) + self.assertEquals(requester.device_id, DEVICE) + self.assertFalse(self.get_success(self._auth.is_server_admin(requester))) + + self.assertEquals( + self.server.last_token_seen, + "some_token", + ) + + def test_inexistent_device(self) -> None: + self.server.introspection_response = { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.api_scope, f"{self.device_scope_prefix}ABCDEF"]), + "username": USERNAME, + "expires_in": 60, + } + + failure = self.get_failure( + self.till_deferred_has_result( + self._auth.get_user_by_access_token("some_token") + ), + InvalidClientTokenError, + ) + self.assertEqual(failure.value.code, 401) + + def test_inexistent_user(self) -> None: + self.server.introspection_response = { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.api_scope]), + "username": "inexistent_user", + "expires_in": 60, + } + + failure = self.get_failure( + self.till_deferred_has_result( + self._auth.get_user_by_access_token("some_token") + ), + AuthError, + ) + # This is a 500, it should never happen really + self.assertEqual(failure.value.code, 500) + + def test_missing_scope(self) -> None: + self.server.introspection_response = { + "active": True, + "sub": SUBJECT, + "scope": "openid", + "username": USERNAME, + "expires_in": 60, + } + + failure = self.get_failure( + self.till_deferred_has_result( + self._auth.get_user_by_access_token("some_token") + ), + InvalidClientTokenError, + ) + self.assertEqual(failure.value.code, 401) + + def test_invalid_response(self) -> None: + self.server.introspection_response = {} + + failure = self.get_failure( + self.till_deferred_has_result( + self._auth.get_user_by_access_token("some_token") + ), + SynapseError, + ) + self.assertEqual(failure.value.code, 503) + + def test_device_id_in_body(self) -> None: + self.server.introspection_response = { + "active": True, + "sub": SUBJECT, + "scope": self.api_scope, + "username": USERNAME, + "expires_in": 60, + "device_id": DEVICE, + } + + requester = self.get_success( + self.till_deferred_has_result( + self._auth.get_user_by_access_token("some_token") + ) + ) + + self.assertEqual(requester.device_id, DEVICE) + + def test_admin_scope(self) -> None: + self.server.introspection_response = { + "active": True, + "sub": SUBJECT, + "scope": " ".join([SYNAPSE_ADMIN_SCOPE, self.api_scope]), + "username": USERNAME, + "expires_in": 60, + } + + requester = self.get_success( + self.till_deferred_has_result( + self._auth.get_user_by_access_token("some_token") + ) + ) + + self.assertEqual(requester.user.to_string(), USER_ID) + self.assertTrue(self.get_success(self._auth.is_server_admin(requester))) + + def test_cached_expired_introspection(self) -> None: + """The handler should raise an error if the introspection response gives + an expiry time, the introspection response is cached and then the entry is + re-requested after it has expired.""" + + self.server.introspection_response = { + "active": True, + "sub": SUBJECT, + "scope": " ".join([self.api_scope, self.device_scope]), + "username": USERNAME, + "expires_in": 60, + } + + self.assertEqual(self.server.calls, 0) + + request = Mock(args={}) + request.args[b"access_token"] = [b"some_token"] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + + # The first CS-API request causes a successful introspection + self.get_success( + self.till_deferred_has_result(self._auth.get_user_by_req(request)) + ) + self.assertEqual(self.server.calls, 1) + + # Sleep for 60 seconds so the token expires. + self.reactor.advance(60.0) + + # Now the CS-API request fails because the token expired + self.assertFailure( + self.till_deferred_has_result(self._auth.get_user_by_req(request)), + InvalidClientTokenError, + ) + # Ensure another introspection request was not sent + self.assertEqual(self.server.calls, 1) + + +@parameterized_class( + ("config",), + [ + ( + { + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:1234/", + "secret": "secret", + }, + }, + ), + ] + # Run the tests with experimental delegation only if authlib is available + + [ + ( + { + "experimental_features": { + "msc3861": { + "enabled": True, + "issuer": ISSUER, + "client_id": CLIENT_ID, + "client_auth_method": "client_secret_post", + "client_secret": CLIENT_SECRET, + "admin_token": "admin_token_value", + } + } + }, + ), + ] + * HAS_AUTHLIB, +) +class DisabledEndpointsTestCase(HomeserverTestCase): + servlets = [ + account.register_servlets, + devices.register_servlets, + keys.register_servlets, + register.register_servlets, + login.register_servlets, + logout.register_servlets, + admin.register_servlets, + ] + + config: Dict[str, Any] + + def default_config(self) -> Dict[str, Any]: + config = super().default_config() + config["public_baseurl"] = BASE_URL + config["disable_registration"] = True + config.update(self.config) + return config + def expect_unauthorized( self, method: str, path: str, content: Union[bytes, str, JsonDict] = "" ) -> None: @@ -560,15 +1117,31 @@ class MSC3861OAuthDelegation(HomeserverTestCase): self.assertEqual(channel.code, 401, channel.json_body) def expect_unrecognized( - self, method: str, path: str, content: Union[bytes, str, JsonDict] = "" + self, + method: str, + path: str, + content: Union[bytes, str, JsonDict] = "", + auth: bool = False, ) -> None: - channel = self.make_request(method, path, content) + channel = self.make_request( + method, path, content, access_token="token" if auth else None + ) self.assertEqual(channel.code, 404, channel.json_body) self.assertEqual( channel.json_body["errcode"], Codes.UNRECOGNIZED, channel.json_body ) + def expect_forbidden( + self, method: str, path: str, content: Union[bytes, str, JsonDict] = "" + ) -> None: + channel = self.make_request(method, path, content) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["errcode"], Codes.FORBIDDEN, channel.json_body + ) + def test_uia_endpoints(self) -> None: """Test that endpoints that were removed in MSC2964 are no longer available.""" @@ -623,11 +1196,35 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_registration_endpoints_removed(self) -> None: """Test that registration endpoints that were removed in MSC2964 are no longer available.""" + appservice = ApplicationService( + token="i_am_an_app_service", + id="1234", + namespaces={"users": [{"regex": r"@alice:.+", "exclusive": True}]}, + sender=UserID.from_string("@as_main:test"), + ) + + self.hs.get_datastores().main.services_cache = [appservice] self.expect_unrecognized( "GET", "/_matrix/client/v1/register/m.login.registration_token/validity" ) + + # Registration is disabled + self.expect_forbidden( + "POST", + "/_matrix/client/v3/register", + {"username": "alice", "password": "hunter2"}, + ) + # This is still available for AS registrations - # self.expect_unrecognized("POST", "/_matrix/client/v3/register") + channel = self.make_request( + "POST", + "/_matrix/client/v3/register", + {"username": "alice", "type": "m.login.application_service"}, + shorthand=False, + access_token="i_am_an_app_service", + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.expect_unrecognized("GET", "/_matrix/client/v3/register/available") self.expect_unrecognized( "POST", "/_matrix/client/v3/register/email/requestToken" @@ -648,8 +1245,20 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_device_management_endpoints_removed(self) -> None: """Test that device management endpoints that were removed in MSC2964 are no longer available.""" - self.expect_unrecognized("POST", "/_matrix/client/v3/delete_devices") - self.expect_unrecognized("DELETE", "/_matrix/client/v3/devices/{DEVICE}") + + # Because we still support those endpoints with ASes, it checks the + # access token before returning 404 + self.hs.get_auth().get_user_by_req = AsyncMock( # type: ignore[method-assign] + return_value=create_requester( + user_id=USER_ID, + device_id=DEVICE, + ) + ) + + self.expect_unrecognized("POST", "/_matrix/client/v3/delete_devices", auth=True) + self.expect_unrecognized( + "DELETE", "/_matrix/client/v3/devices/{DEVICE}", auth=True + ) def test_openid_endpoints_removed(self) -> None: """Test that OpenID id_token endpoints that were removed in MSC2964 are no longer available.""" @@ -673,125 +1282,3 @@ class MSC3861OAuthDelegation(HomeserverTestCase): self.expect_unrecognized("GET", "/_synapse/admin/v1/users/foo/admin") self.expect_unrecognized("PUT", "/_synapse/admin/v1/users/foo/admin") self.expect_unrecognized("POST", "/_synapse/admin/v1/account_validity/validity") - - def test_admin_token(self) -> None: - """The handler should return a requester with admin rights when admin_token is used.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json(code=200, payload={"active": False}), - ) - - request = Mock(args={}) - request.args[b"access_token"] = [b"admin_token_value"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual( - requester.user.to_string(), - OIDC_ADMIN_USERID, - ) - self.assertEqual(requester.is_guest, False) - self.assertEqual(requester.device_id, None) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), True - ) - - # There should be no call to the introspection endpoint - self.http_client.request.assert_not_called() - - @override_config({"mau_stats_only": True}) - def test_request_tracking(self) -> None: - """Using an access token should update the client_ips and MAU tables.""" - # To start, there are no MAU users. - store = self.hs.get_datastores().main - mau = self.get_success(store.get_monthly_active_count()) - self.assertEqual(mau, 0) - - known_token = "token-token-GOOD-:)" - - async def mock_http_client_request( - method: str, - uri: str, - data: Optional[bytes] = None, - headers: Optional[Headers] = None, - ) -> IResponse: - """Mocked auth provider response.""" - assert method == "POST" - token = parse_qs(data)[b"token"][0].decode("utf-8") - if token == known_token: - return FakeResponse.json( - code=200, - payload={ - "active": True, - "scope": MATRIX_USER_SCOPE, - "sub": SUBJECT, - "username": USERNAME, - }, - ) - - return FakeResponse.json(code=200, payload={"active": False}) - - self.http_client.request = mock_http_client_request - - EXAMPLE_IPV4_ADDR = "123.123.123.123" - EXAMPLE_USER_AGENT = "httprettygood" - - # First test a known access token - channel = FakeChannel(self.site, self.reactor) - # type-ignore: FakeChannel is a mock of an HTTPChannel, not a proper HTTPChannel - req = SynapseRequest(channel, self.site) # type: ignore[arg-type] - req.client.host = EXAMPLE_IPV4_ADDR - req.requestHeaders.addRawHeader("Authorization", f"Bearer {known_token}") - req.requestHeaders.addRawHeader("User-Agent", EXAMPLE_USER_AGENT) - req.content = BytesIO(b"") - req.requestReceived( - b"GET", - b"/_matrix/client/v3/account/whoami", - b"1.1", - ) - channel.await_result() - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - self.assertEqual(channel.json_body["user_id"], USER_ID, channel.json_body) - - # Expect to see one MAU entry, from the first request - mau = self.get_success(store.get_monthly_active_count()) - self.assertEqual(mau, 1) - - conn_infos = self.get_success( - store.get_user_ip_and_agents(UserID.from_string(USER_ID)) - ) - self.assertEqual(len(conn_infos), 1, conn_infos) - conn_info = conn_infos[0] - self.assertEqual(conn_info["access_token"], known_token) - self.assertEqual(conn_info["ip"], EXAMPLE_IPV4_ADDR) - self.assertEqual(conn_info["user_agent"], EXAMPLE_USER_AGENT) - - # Now test MAS making a request using the special __oidc_admin token - MAS_IPV4_ADDR = "127.0.0.1" - MAS_USER_AGENT = "masmasmas" - - channel = FakeChannel(self.site, self.reactor) - req = SynapseRequest(channel, self.site) # type: ignore[arg-type] - req.client.host = MAS_IPV4_ADDR - req.requestHeaders.addRawHeader( - "Authorization", f"Bearer {self.auth._admin_token}" - ) - req.requestHeaders.addRawHeader("User-Agent", MAS_USER_AGENT) - req.content = BytesIO(b"") - req.requestReceived( - b"GET", - b"/_matrix/client/v3/account/whoami", - b"1.1", - ) - channel.await_result() - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - self.assertEqual( - channel.json_body["user_id"], OIDC_ADMIN_USERID, channel.json_body - ) - - # Still expect to see one MAU entry, from the first request - mau = self.get_success(store.get_monthly_active_count()) - self.assertEqual(mau, 1) - - conn_infos = self.get_success( - store.get_user_ip_and_agents(UserID.from_string(OIDC_ADMIN_USERID)) - ) - self.assertEqual(conn_infos, []) diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index a81501979d..db37e7d185 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -25,7 +25,7 @@ from urllib.parse import parse_qs, urlparse import pymacaroons -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.handlers.sso import MappingException from synapse.http.site import SynapseRequest @@ -57,6 +57,7 @@ CLIENT_ID = "test-client-id" CLIENT_SECRET = "test-client-secret" BASE_URL = "https://synapse/" CALLBACK_URL = BASE_URL + "_synapse/client/oidc/callback" +TEST_REDIRECT_URI = "https://test/oidc/callback" SCOPES = ["openid"] # config for common cases @@ -70,12 +71,16 @@ DEFAULT_CONFIG = { } # extends the default config with explicit OAuth2 endpoints instead of using discovery +# +# We add "explicit" to things to make them different from the discovered values to make +# sure that the explicit values override the discovered ones. EXPLICIT_ENDPOINT_CONFIG = { **DEFAULT_CONFIG, "discover": False, - "authorization_endpoint": ISSUER + "authorize", - "token_endpoint": ISSUER + "token", - "jwks_uri": ISSUER + "jwks", + "authorization_endpoint": ISSUER + "authorize-explicit", + "token_endpoint": ISSUER + "token-explicit", + "jwks_uri": ISSUER + "jwks-explicit", + "id_token_signing_alg_values_supported": ["RS256", ""], } @@ -259,12 +264,64 @@ class OidcHandlerTestCase(HomeserverTestCase): self.get_success(self.provider.load_metadata()) self.fake_server.get_metadata_handler.assert_not_called() + @override_config({"oidc_config": {**EXPLICIT_ENDPOINT_CONFIG, "discover": True}}) + def test_discovery_with_explicit_config(self) -> None: + """ + The handler should discover the endpoints from OIDC discovery document but + values are overriden by the explicit config. + """ + # This would throw if some metadata were invalid + metadata = self.get_success(self.provider.load_metadata()) + self.fake_server.get_metadata_handler.assert_called_once() + + self.assertEqual(metadata.issuer, self.fake_server.issuer) + # It seems like authlib does not have that defined in its metadata models + self.assertEqual( + metadata.get("userinfo_endpoint"), + self.fake_server.userinfo_endpoint, + ) + + # Ensure the values are overridden correctly since these were configured + # explicitly + self.assertEqual( + metadata.authorization_endpoint, + EXPLICIT_ENDPOINT_CONFIG["authorization_endpoint"], + ) + self.assertEqual( + metadata.token_endpoint, EXPLICIT_ENDPOINT_CONFIG["token_endpoint"] + ) + self.assertEqual(metadata.jwks_uri, EXPLICIT_ENDPOINT_CONFIG["jwks_uri"]) + self.assertEqual( + metadata.id_token_signing_alg_values_supported, + EXPLICIT_ENDPOINT_CONFIG["id_token_signing_alg_values_supported"], + ) + + # subsequent calls should be cached + self.reset_mocks() + self.get_success(self.provider.load_metadata()) + self.fake_server.get_metadata_handler.assert_not_called() + @override_config({"oidc_config": EXPLICIT_ENDPOINT_CONFIG}) def test_no_discovery(self) -> None: """When discovery is disabled, it should not try to load from discovery document.""" - self.get_success(self.provider.load_metadata()) + metadata = self.get_success(self.provider.load_metadata()) self.fake_server.get_metadata_handler.assert_not_called() + # Ensure the values are overridden correctly since these were configured + # explicitly + self.assertEqual( + metadata.authorization_endpoint, + EXPLICIT_ENDPOINT_CONFIG["authorization_endpoint"], + ) + self.assertEqual( + metadata.token_endpoint, EXPLICIT_ENDPOINT_CONFIG["token_endpoint"] + ) + self.assertEqual(metadata.jwks_uri, EXPLICIT_ENDPOINT_CONFIG["jwks_uri"]) + self.assertEqual( + metadata.id_token_signing_alg_values_supported, + EXPLICIT_ENDPOINT_CONFIG["id_token_signing_alg_values_supported"], + ) + @override_config({"oidc_config": DEFAULT_CONFIG}) def test_load_jwks(self) -> None: """JWKS loading is done once (then cached) if used.""" @@ -427,6 +484,32 @@ class OidcHandlerTestCase(HomeserverTestCase): self.assertEqual(code_verifier, "") self.assertEqual(redirect, "http://client/redirect") + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "passthrough_authorization_parameters": ["additional_parameter"], + } + } + ) + def test_passthrough_parameters(self) -> None: + """The redirect request has additional parameters, one is authorized, one is not""" + req = Mock(spec=["cookies", "args"]) + req.cookies = [] + req.args = {} + req.args[b"additional_parameter"] = ["a_value".encode("utf-8")] + req.args[b"not_authorized_parameter"] = ["any".encode("utf-8")] + + url = urlparse( + self.get_success( + self.provider.handle_redirect_request(req, b"http://client/redirect") + ) + ) + + params = parse_qs(url.query) + self.assertEqual(params["additional_parameter"], ["a_value"]) + self.assertNotIn("not_authorized_parameters", params) + @override_config({"oidc_config": DEFAULT_CONFIG}) def test_redirect_request_with_code_challenge(self) -> None: """The redirect request has the right arguments & generates a valid session cookie.""" @@ -530,6 +613,24 @@ class OidcHandlerTestCase(HomeserverTestCase): code_verifier = get_value_from_macaroon(macaroon, "code_verifier") self.assertEqual(code_verifier, "") + @override_config( + {"oidc_config": {**DEFAULT_CONFIG, "redirect_uri": TEST_REDIRECT_URI}} + ) + def test_redirect_request_with_overridden_redirect_uri(self) -> None: + """The authorization endpoint redirect has the overridden `redirect_uri` value.""" + req = Mock(spec=["cookies"]) + req.cookies = [] + + url = urlparse( + self.get_success( + self.provider.handle_redirect_request(req, b"http://client/redirect") + ) + ) + + # Ensure that the redirect_uri in the returned url has been overridden. + params = parse_qs(url.query) + self.assertEqual(params["redirect_uri"], [TEST_REDIRECT_URI]) + @override_config({"oidc_config": DEFAULT_CONFIG}) def test_callback_error(self) -> None: """Errors from the provider returned in the callback are displayed.""" @@ -897,6 +998,81 @@ class OidcHandlerTestCase(HomeserverTestCase): self.assertEqual(args["client_id"], [CLIENT_ID]) self.assertEqual(args["redirect_uri"], [CALLBACK_URL]) + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "redirect_uri": TEST_REDIRECT_URI, + } + } + ) + def test_code_exchange_with_overridden_redirect_uri(self) -> None: + """Code exchange behaves correctly and handles various error scenarios.""" + # Set up a fake IdP with a token endpoint handler. + token = { + "type": "Bearer", + "access_token": "aabbcc", + } + + self.fake_server.post_token_handler.side_effect = None + self.fake_server.post_token_handler.return_value = FakeResponse.json( + payload=token + ) + code = "code" + + # Exchange the code against the fake IdP. + self.get_success(self.provider._exchange_code(code, code_verifier="")) + + # Check that the `redirect_uri` parameter provided matches our + # overridden config value. + kwargs = self.fake_server.request.call_args[1] + args = parse_qs(kwargs["data"].decode("utf-8")) + self.assertEqual(args["redirect_uri"], [TEST_REDIRECT_URI]) + + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "redirect_uri": TEST_REDIRECT_URI, + } + } + ) + def test_code_exchange_ignores_access_token(self) -> None: + """ + Code exchange completes successfully and doesn't validate the `at_hash` + (access token hash) field of an ID token when the access token isn't + going to be used. + + The access token won't be used in this test because Synapse (currently) + only needs it to fetch a user's metadata if it isn't included in the ID + token itself. + + Because we have included "openid" in the requested scopes for this IdP + (see `SCOPES`), user metadata is be included in the ID token. Thus the + access token isn't needed, and it's unnecessary for Synapse to validate + the access token. + + This is a regression test for a situation where an upstream identity + provider was providing an invalid `at_hash` value, which Synapse errored + on, yet Synapse wasn't using the access token for anything. + """ + # Exchange the code against the fake IdP. + userinfo = { + "sub": "foo", + "username": "foo", + "phone": "1234567", + } + with self.fake_server.id_token_override( + { + "at_hash": "invalid-hash", + } + ): + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + + # If no error was rendered, then we have success. + self.render_error.assert_not_called() + @override_config( { "oidc_config": { @@ -1267,6 +1443,113 @@ class OidcHandlerTestCase(HomeserverTestCase): auth_provider_session_id=None, ) + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "attribute_requirements": [ + {"attribute": "test", "one_of": ["foo", "bar"]} + ], + } + } + ) + def test_attribute_requirements_one_of_succeeds(self) -> None: + """Test that auth succeeds if userinfo attribute has multiple values and CONTAINS required value""" + # userinfo with "test": ["bar"] attribute should succeed. + userinfo = { + "sub": "tester", + "username": "tester", + "test": ["bar"], + } + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + + # check that the auth handler got called as expected + self.complete_sso_login.assert_called_once_with( + "@tester:test", + self.provider.idp_id, + request, + ANY, + None, + new_user=True, + auth_provider_session_id=None, + ) + + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "attribute_requirements": [ + {"attribute": "test", "one_of": ["foo", "bar"]} + ], + } + } + ) + def test_attribute_requirements_one_of_fails(self) -> None: + """Test that auth fails if userinfo attribute has multiple values yet + DOES NOT CONTAIN a required value + """ + # userinfo with "test": ["something else"] attribute should fail. + userinfo = { + "sub": "tester", + "username": "tester", + "test": ["something else"], + } + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + self.complete_sso_login.assert_not_called() + + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "attribute_requirements": [{"attribute": "test"}], + } + } + ) + def test_attribute_requirements_does_not_exist(self) -> None: + """OIDC login fails if the required attribute does not exist in the OIDC userinfo response.""" + # userinfo lacking "test" attribute should fail. + userinfo = { + "sub": "tester", + "username": "tester", + } + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + self.complete_sso_login.assert_not_called() + + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "attribute_requirements": [{"attribute": "test"}], + } + } + ) + def test_attribute_requirements_exist(self) -> None: + """OIDC login succeeds if the required attribute exist (regardless of value) + in the OIDC userinfo response. + """ + # userinfo with "test" attribute and random value should succeed. + userinfo = { + "sub": "tester", + "username": "tester", + "test": random_string(5), # value does not matter + } + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + + # check that the auth handler got called as expected + self.complete_sso_login.assert_called_once_with( + "@tester:test", + self.provider.idp_id, + request, + ANY, + None, + new_user=True, + auth_provider_session_id=None, + ) + @override_config( { "oidc_config": { diff --git a/tests/handlers/test_password_providers.py b/tests/handlers/test_password_providers.py index ed203eb299..0a78fe0304 100644 --- a/tests/handlers/test_password_providers.py +++ b/tests/handlers/test_password_providers.py @@ -25,7 +25,7 @@ from http import HTTPStatus from typing import Any, Dict, List, Optional, Type, Union from unittest.mock import AsyncMock, Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse from synapse.api.constants import LoginType diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 598d6c13cd..51b6c60531 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -23,14 +23,21 @@ from typing import Optional, cast from unittest.mock import Mock, call from parameterized import parameterized -from signedjson.key import generate_signing_key +from signedjson.key import ( + encode_verify_key_base64, + generate_signing_key, + get_verify_key, +) -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, Membership, PresenceState from synapse.api.presence import UserDevicePresenceState, UserPresenceState -from synapse.api.room_versions import KNOWN_ROOM_VERSIONS -from synapse.events.builder import EventBuilder +from synapse.api.room_versions import ( + RoomVersion, +) +from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.events import EventBase, make_event_from_dict from synapse.federation.sender import FederationSender from synapse.handlers.presence import ( BUSY_ONLINE_TIMEOUT, @@ -45,18 +52,24 @@ from synapse.handlers.presence import ( handle_update, ) from synapse.rest import admin -from synapse.rest.client import room +from synapse.rest.client import login, room, sync from synapse.server import HomeServer from synapse.storage.database import LoggingDatabaseConnection +from synapse.storage.keys import FetchKeyResult from synapse.types import JsonDict, UserID, get_domain_from_id from synapse.util import Clock from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase +from tests.unittest import override_config class PresenceUpdateTestCase(unittest.HomeserverTestCase): - servlets = [admin.register_servlets] + servlets = [ + admin.register_servlets, + login.register_servlets, + sync.register_servlets, + ] def prepare( self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer @@ -77,6 +90,7 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): prev_state, new_state, is_mine=True, + our_server_name=self.hs.hostname, wheel_timer=wheel_timer, now=now, persist=False, @@ -124,6 +138,7 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): prev_state, new_state, is_mine=True, + our_server_name=self.hs.hostname, wheel_timer=wheel_timer, now=now, persist=False, @@ -174,6 +189,7 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): prev_state, new_state, is_mine=True, + our_server_name=self.hs.hostname, wheel_timer=wheel_timer, now=now, persist=False, @@ -222,6 +238,7 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): prev_state, new_state, is_mine=True, + our_server_name=self.hs.hostname, wheel_timer=wheel_timer, now=now, persist=False, @@ -262,6 +279,7 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): prev_state, new_state, is_mine=False, + our_server_name=self.hs.hostname, wheel_timer=wheel_timer, now=now, persist=False, @@ -301,6 +319,7 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): prev_state, new_state, is_mine=True, + our_server_name=self.hs.hostname, wheel_timer=wheel_timer, now=now, persist=False, @@ -328,6 +347,7 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): prev_state, new_state, is_mine=True, + our_server_name=self.hs.hostname, wheel_timer=wheel_timer, now=now, persist=False, @@ -418,6 +438,7 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): prev_state, new_state, is_mine=True, + our_server_name=self.hs.hostname, wheel_timer=wheel_timer, now=now, persist=True, @@ -425,6 +446,103 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): wheel_timer.insert.assert_not_called() + # `rc_presence` is set very high during unit tests to avoid ratelimiting + # subtly impacting unrelated tests. We set the ratelimiting back to a + # reasonable value for the tests specific to presence ratelimiting. + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def test_over_ratelimit_offline_to_online_to_unavailable(self) -> None: + """ + Send a presence update, check that it went through, immediately send another one and + check that it was ignored. + """ + self._test_ratelimit_offline_to_online_to_unavailable(ratelimited=True) + + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def test_within_ratelimit_offline_to_online_to_unavailable(self) -> None: + """ + Send a presence update, check that it went through, advancing time a sufficient amount, + send another presence update and check that it also worked. + """ + self._test_ratelimit_offline_to_online_to_unavailable(ratelimited=False) + + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def _test_ratelimit_offline_to_online_to_unavailable( + self, ratelimited: bool + ) -> None: + """Test rate limit for presence updates sent with sync requests. + + Args: + ratelimited: Test rate limited case. + """ + wheel_timer = Mock() + user_id = "@user:pass" + now = 5000000 + sync_url = "/sync?access_token=%s&set_presence=%s" + + # Register the user who syncs presence + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Get the handler (which kicks off a bunch of timers). + presence_handler = self.hs.get_presence_handler() + + # Ensure the user is initially offline. + prev_state = UserPresenceState.default(user_id) + new_state = prev_state.copy_and_replace( + state=PresenceState.OFFLINE, last_active_ts=now + ) + + state, persist_and_notify, federation_ping = handle_update( + prev_state, + new_state, + is_mine=True, + our_server_name=self.hs.hostname, + wheel_timer=wheel_timer, + now=now, + persist=False, + ) + + # Check that the user is offline. + state = self.get_success( + presence_handler.get_state(UserID.from_string(user_id)) + ) + self.assertEqual(state.state, PresenceState.OFFLINE) + + # Send sync request with set_presence=online. + channel = self.make_request("GET", sync_url % (access_token, "online")) + self.assertEqual(200, channel.code) + + # Assert the user is now online. + state = self.get_success( + presence_handler.get_state(UserID.from_string(user_id)) + ) + self.assertEqual(state.state, PresenceState.ONLINE) + + if not ratelimited: + # Advance time a sufficient amount to avoid rate limiting. + self.reactor.advance(30) + + # Send another sync request with set_presence=unavailable. + channel = self.make_request("GET", sync_url % (access_token, "unavailable")) + self.assertEqual(200, channel.code) + + state = self.get_success( + presence_handler.get_state(UserID.from_string(user_id)) + ) + + if ratelimited: + # Assert the user is still online and presence update was ignored. + self.assertEqual(state.state, PresenceState.ONLINE) + else: + # Assert the user is now unavailable. + self.assertEqual(state.state, PresenceState.UNAVAILABLE) + class PresenceTimeoutTestCase(unittest.TestCase): """Tests different timers and that the timer does not change `status_msg` of user.""" @@ -1825,6 +1943,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): # self.event_builder_for_2.hostname = "test2" self.store = hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() self.state = hs.get_state_handler() self._event_auth_handler = hs.get_event_auth_handler() @@ -1940,29 +2059,35 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): hostname = get_domain_from_id(user_id) - room_version = self.get_success(self.store.get_room_version_id(room_id)) + room_version = self.get_success(self.store.get_room_version(room_id)) - builder = EventBuilder( - state=self.state, - event_auth_handler=self._event_auth_handler, - store=self.store, - clock=self.clock, - hostname=hostname, - signing_key=self.random_signing_key, - room_version=KNOWN_ROOM_VERSIONS[room_version], - room_id=room_id, - type=EventTypes.Member, - sender=user_id, - state_key=user_id, - content={"membership": Membership.JOIN}, + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) ) - prev_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(room_id) + # Figure out what the forward extremities in the room are (the most recent + # events that aren't tied into the DAG) + forward_extremity_event_ids = self.get_success( + self.hs.get_datastores().main.get_latest_event_ids_in_room(room_id) ) - event = self.get_success( - builder.build(prev_event_ids=list(prev_event_ids), auth_event_ids=None) + event = self.create_fake_event_from_remote_server( + remote_server_name=hostname, + event_dict={ + "room_id": room_id, + "sender": user_id, + "type": EventTypes.Member, + "state_key": user_id, + "depth": 1000, + "origin_server_ts": 1, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + }, + room_version=room_version, ) self.get_success(self.federation_event_handler.on_receive_pdu(hostname, event)) @@ -1970,3 +2095,50 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): # Check that it was successfully persisted. self.get_success(self.store.get_event(event.event_id)) self.get_success(self.store.get_event(event.event_id)) + + def create_fake_event_from_remote_server( + self, remote_server_name: str, event_dict: JsonDict, room_version: RoomVersion + ) -> EventBase: + """ + This is similar to what `FederatingHomeserverTestCase` is doing but we don't + need all of the extra baggage and we want to be able to create an event from + many remote servers. + """ + + # poke the other server's signing key into the key store, so that we don't + # make requests for it + other_server_signature_key = generate_signing_key("test") + verify_key = get_verify_key(other_server_signature_key) + verify_key_id = "%s:%s" % (verify_key.alg, verify_key.version) + + self.get_success( + self.hs.get_datastores().main.store_server_keys_response( + remote_server_name, + from_server=remote_server_name, + ts_added_ms=self.clock.time_msec(), + verify_keys={ + verify_key_id: FetchKeyResult( + verify_key=verify_key, + valid_until_ts=self.clock.time_msec() + 10000, + ), + }, + response_json={ + "verify_keys": { + verify_key_id: {"key": encode_verify_key_base64(verify_key)} + } + }, + ) + ) + + add_hashes_and_signatures( + room_version=room_version, + event_dict=event_dict, + signature_name=remote_server_name, + signing_key=other_server_signature_key, + ) + event = make_event_from_dict( + event_dict, + room_version=room_version, + ) + + return event diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index cb1c6fbb80..93934e9ff7 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -23,7 +23,7 @@ from unittest.mock import AsyncMock, Mock from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.types from synapse.api.errors import AuthError, SynapseError @@ -369,6 +369,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): time_now_ms=self.clock.time_msec(), upload_name=None, filesystem_id="xyz", + sha256="abcdefg12345", ) ) diff --git a/tests/handlers/test_receipts.py b/tests/handlers/test_receipts.py index 7c5bec2b76..cf04ac6e00 100644 --- a/tests/handlers/test_receipts.py +++ b/tests/handlers/test_receipts.py @@ -22,7 +22,7 @@ from copy import deepcopy from typing import List -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EduTypes, ReceiptTypes from synapse.server import HomeServer diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 92487692db..43ded2fc10 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -22,7 +22,7 @@ from typing import Any, Collection, List, Optional, Tuple from unittest.mock import AsyncMock, Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.auth.internal import InternalAuth from synapse.api.constants import UserTypes @@ -588,6 +588,29 @@ class RegistrationTestCase(unittest.HomeserverTestCase): d = self.store.is_support_user(user_id) self.assertFalse(self.get_success(d)) + def test_underscore_localpart_rejected_by_default(self) -> None: + for invalid_user_id in ("_", "_prefixed"): + with self.subTest(invalid_user_id=invalid_user_id): + self.get_failure( + self.handler.register_user(localpart=invalid_user_id), + SynapseError, + ) + + @override_config( + { + "allow_underscore_prefixed_localpart": True, + } + ) + def test_underscore_localpart_allowed_if_configured(self) -> None: + for valid_user_id in ("_", "_prefixed"): + with self.subTest(valid_user_id=valid_user_id): + user_id = self.get_success( + self.handler.register_user( + localpart=valid_user_id, + ), + ) + self.assertEqual(user_id, f"@{valid_user_id}:test") + def test_invalid_user_id(self) -> None: invalid_user_id = "^abcd" self.get_failure( @@ -715,6 +738,41 @@ class RegistrationTestCase(unittest.HomeserverTestCase): self.handler.register_user(localpart="bobflimflob", auth_provider_id="saml") ) + def test_register_default_user_type(self) -> None: + """Test that the default user type is none when registering a user.""" + user_id = self.get_success(self.handler.register_user(localpart="user")) + user_info = self.get_success(self.store.get_user_by_id(user_id)) + assert user_info is not None + self.assertEqual(user_info.user_type, None) + + def test_register_extra_user_types_valid(self) -> None: + """ + Test that the specified user type is set correctly when registering a user. + n.b. No validation is done on the user type, so this test + is only to ensure that the user type can be set to any value. + """ + user_id = self.get_success( + self.handler.register_user(localpart="user", user_type="anyvalue") + ) + user_info = self.get_success(self.store.get_user_by_id(user_id)) + assert user_info is not None + self.assertEqual(user_info.user_type, "anyvalue") + + @override_config( + { + "user_types": { + "extra_user_types": ["extra1", "extra2"], + "default_user_type": "extra1", + } + } + ) + def test_register_extra_user_types_with_default(self) -> None: + """Test that the default_user_type in config is set correctly when registering a user.""" + user_id = self.get_success(self.handler.register_user(localpart="user")) + user_info = self.get_success(self.store.get_user_by_id(user_id)) + assert user_info is not None + self.assertEqual(user_info.user_type, "extra1") + async def get_or_create_user( self, requester: Requester, diff --git a/tests/handlers/test_room_list.py b/tests/handlers/test_room_list.py index 4d22ef98c2..45cef09b22 100644 --- a/tests/handlers/test_room_list.py +++ b/tests/handlers/test_room_list.py @@ -6,6 +6,7 @@ from synapse.rest.client import directory, login, room from synapse.types import JsonDict from tests import unittest +from tests.utils import default_config class RoomListHandlerTestCase(unittest.HomeserverTestCase): @@ -30,6 +31,11 @@ class RoomListHandlerTestCase(unittest.HomeserverTestCase): assert channel.code == HTTPStatus.OK, f"couldn't publish room: {channel.result}" return room_id + def default_config(self) -> JsonDict: + config = default_config("test") + config["room_list_publication_rules"] = [{"action": "allow"}] + return config + def test_acls_applied_to_room_directory_results(self) -> None: """ Creates 3 rooms. Room 2 has an ACL that only permits the homeservers diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index f43ce66483..3084f180f5 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -1,14 +1,17 @@ from unittest.mock import AsyncMock, patch -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin import synapse.rest.client.login import synapse.rest.client.room -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import AccountDataTypes, EventTypes, Membership from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import FrozenEventV3 +from synapse.federation.federation_base import ( + event_from_pdu_json, +) from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer from synapse.types import UserID, create_requester @@ -453,3 +456,165 @@ class RoomMemberMasterHandlerTestCase(HomeserverTestCase): new_count = rows[0][0] self.assertEqual(initial_count, new_count) + + +class TestInviteFiltering(FederatingHomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + synapse.rest.client.login.register_servlets, + synapse.rest.client.room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.handler = hs.get_room_member_handler() + self.fed_handler = hs.get_federation_handler() + self.store = hs.get_datastores().main + + # Create three users. + self.alice = self.register_user("alice", "pass") + self.alice_token = self.login("alice", "pass") + self.bob = self.register_user("bob", "pass") + self.bob_token = self.login("bob", "pass") + + @override_config({"experimental_features": {"msc4155_enabled": True}}) + def test_misc4155_block_invite_local(self) -> None: + """Test that MSC4155 will block a user from being invited to a room""" + room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG, + { + "blocked_users": [self.alice], + }, + ) + ) + + f = self.get_failure( + self.handler.update_membership( + requester=create_requester(self.alice), + target=UserID.from_string(self.bob), + room_id=room_id, + action=Membership.INVITE, + ), + SynapseError, + ).value + self.assertEqual(f.code, 403) + self.assertEqual(f.errcode, "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED") + + @override_config({"experimental_features": {"msc4155_enabled": False}}) + def test_msc4155_disabled_allow_invite_local(self) -> None: + """Test that MSC4155 will block a user from being invited to a room""" + room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG, + { + "blocked_users": [self.alice], + }, + ) + ) + + self.get_success( + self.handler.update_membership( + requester=create_requester(self.alice), + target=UserID.from_string(self.bob), + room_id=room_id, + action=Membership.INVITE, + ), + ) + + @override_config({"experimental_features": {"msc4155_enabled": True}}) + def test_msc4155_block_invite_remote(self) -> None: + """Test that MSC4155 will block a remote user from being invited to a room""" + # A remote user who sends the invite + remote_server = "otherserver" + remote_user = "@otheruser:" + remote_server + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG, + {"blocked_users": [remote_user]}, + ) + ) + + room_id = self.helper.create_room_as( + room_creator=self.alice, tok=self.alice_token + ) + room_version = self.get_success(self.store.get_room_version(room_id)) + + invite_event = event_from_pdu_json( + { + "type": EventTypes.Member, + "content": {"membership": "invite"}, + "room_id": room_id, + "sender": remote_user, + "state_key": self.bob, + "depth": 32, + "prev_events": [], + "auth_events": [], + "origin_server_ts": self.clock.time_msec(), + }, + room_version, + ) + + f = self.get_failure( + self.fed_handler.on_invite_request( + remote_server, + invite_event, + invite_event.room_version, + ), + SynapseError, + ).value + self.assertEqual(f.code, 403) + self.assertEqual(f.errcode, "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED") + + @override_config({"experimental_features": {"msc4155_enabled": True}}) + def test_msc4155_block_invite_remote_server(self) -> None: + """Test that MSC4155 will block a remote server's user from being invited to a room""" + # A remote user who sends the invite + remote_server = "otherserver" + remote_user = "@otheruser:" + remote_server + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG, + {"blocked_servers": [remote_server]}, + ) + ) + + room_id = self.helper.create_room_as( + room_creator=self.alice, tok=self.alice_token + ) + room_version = self.get_success(self.store.get_room_version(room_id)) + + invite_event = event_from_pdu_json( + { + "type": EventTypes.Member, + "content": {"membership": "invite"}, + "room_id": room_id, + "sender": remote_user, + "state_key": self.bob, + "depth": 32, + "prev_events": [], + "auth_events": [], + "origin_server_ts": self.clock.time_msec(), + }, + room_version, + ) + + f = self.get_failure( + self.fed_handler.on_invite_request( + remote_server, + invite_event, + invite_event.room_version, + ), + SynapseError, + ).value + self.assertEqual(f.code, 403) + self.assertEqual(f.errcode, "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED") diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py new file mode 100644 index 0000000000..3ea6f13cce --- /dev/null +++ b/tests/handlers/test_room_policy.py @@ -0,0 +1,226 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +from typing import Optional +from unittest import mock + +from twisted.internet.testing import MemoryReactor + +from synapse.events import EventBase, make_event_from_dict +from synapse.rest import admin +from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.types import JsonDict, UserID +from synapse.types.handlers.policy_server import RECOMMENDATION_OK, RECOMMENDATION_SPAM +from synapse.util import Clock + +from tests import unittest +from tests.test_utils import event_injection + + +class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): + """Tests room policy handler.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + # mock out the federation transport client + self.mock_federation_transport_client = mock.Mock( + spec=["get_policy_recommendation_for_pdu"] + ) + self.mock_federation_transport_client.get_policy_recommendation_for_pdu = ( + mock.AsyncMock() + ) + return super().setup_test_homeserver( + federation_transport_client=self.mock_federation_transport_client + ) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.hs = hs + self.handler = hs.get_room_policy_handler() + main_store = self.hs.get_datastores().main + + # Create a room + self.creator = self.register_user("creator", "test1234") + self.creator_token = self.login("creator", "test1234") + self.room_id = self.helper.create_room_as( + room_creator=self.creator, tok=self.creator_token + ) + room_version = self.get_success(main_store.get_room_version(self.room_id)) + + # Create some sample events + self.spammy_event = make_event_from_dict( + room_version=room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is a spammy event.", + }, + }, + ) + self.not_spammy_event = make_event_from_dict( + room_version=room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@not_spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is a NOT spammy event.", + }, + }, + ) + + # Prepare the policy server mock to decide spam vs not spam on those events + self.call_count = 0 + + async def get_policy_recommendation_for_pdu( + destination: str, + pdu: EventBase, + timeout: Optional[int] = None, + ) -> JsonDict: + self.call_count += 1 + self.assertEqual(destination, self.OTHER_SERVER_NAME) + if pdu.event_id == self.spammy_event.event_id: + return {"recommendation": RECOMMENDATION_SPAM} + elif pdu.event_id == self.not_spammy_event.event_id: + return {"recommendation": RECOMMENDATION_OK} + else: + self.fail("Unexpected event ID") + + self.mock_federation_transport_client.get_policy_recommendation_for_pdu.side_effect = get_policy_recommendation_for_pdu + + def _add_policy_server_to_room(self) -> None: + # Inject a member event into the room + policy_user_id = f"@policy:{self.OTHER_SERVER_NAME}" + self.get_success( + event_injection.inject_member_event( + self.hs, self.room_id, policy_user_id, "join" + ) + ) + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + "via": self.OTHER_SERVER_NAME, + }, + tok=self.creator_token, + state_key="", + ) + + def test_no_policy_event_set(self) -> None: + # We don't need to modify the room state at all - we're testing the default + # case where a room doesn't use a policy server. + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_empty_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + # empty content (no `via`) + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_nonstring_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + "via": 42, # should be a server name + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_self_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + # We ignore events when the policy server is ourselves (for now?) + "via": (UserID.from_string(self.creator)).domain, + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_invalid_server_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + "via": "|this| is *not* a (valid) server name.com", + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_not_in_room_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + "via": f"x.{self.OTHER_SERVER_NAME}", + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_spammy_event_is_spam(self) -> None: + self._add_policy_server_to_room() + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, False) + self.assertEqual(self.call_count, 1) + + def test_not_spammy_event_is_not_spam(self) -> None: + self._add_policy_server_to_room() + + ok = self.get_success(self.handler.is_event_allowed(self.not_spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 1) diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index b55fa1a8fd..27646d7365 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -22,7 +22,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Tuple from unittest import mock from twisted.internet.defer import ensureDeferred -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import ( EventContentFields, @@ -45,6 +45,7 @@ from synapse.types import JsonDict, UserID, create_requester from synapse.util import Clock from tests import unittest +from tests.unittest import override_config def _create_event( @@ -245,6 +246,7 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): ) self._assert_hierarchy(result, expected) + @override_config({"rc_room_creation": {"burst_count": 1000, "per_second": 1}}) def test_large_space(self) -> None: """Test a space with a large number of rooms.""" rooms = [self.room] @@ -527,6 +529,7 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): ) self._assert_hierarchy(result, expected) + @override_config({"rc_room_creation": {"burst_count": 1000, "per_second": 1}}) def test_pagination(self) -> None: """Test simple pagination works.""" room_ids = [] @@ -564,6 +567,7 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): self._assert_hierarchy(result, expected) self.assertNotIn("next_batch", result) + @override_config({"rc_room_creation": {"burst_count": 1000, "per_second": 1}}) def test_invalid_pagination_token(self) -> None: """An invalid pagination token, or changing other parameters, shoudl be rejected.""" room_ids = [] @@ -615,6 +619,7 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): SynapseError, ) + @override_config({"rc_room_creation": {"burst_count": 1000, "per_second": 1}}) def test_max_depth(self) -> None: """Create a deep tree to test the max depth against.""" spaces = [self.space] @@ -1080,6 +1085,62 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): self.assertEqual(federation_requests, 2) self._assert_hierarchy(result, expected) + def test_fed_remote_room_hosts(self) -> None: + """ + Test if requested room is available over federation using via's. + """ + fed_hostname = self.hs.hostname + "2" + fed_space = "#fed_space:" + fed_hostname + fed_subroom = "#fed_sub_room:" + fed_hostname + + remote_room_hosts = tuple(fed_hostname) + + requested_room_entry = _RoomEntry( + fed_space, + { + "room_id": fed_space, + "world_readable": True, + "join_rule": "public", + "room_type": RoomTypes.SPACE, + }, + [ + { + "type": EventTypes.SpaceChild, + "room_id": fed_space, + "state_key": fed_subroom, + "content": {"via": [fed_hostname]}, + } + ], + ) + child_room = { + "room_id": fed_subroom, + "world_readable": True, + "join_rule": "public", + } + + async def summarize_remote_room_hierarchy( + _self: Any, room: Any, suggested_only: bool + ) -> Tuple[Optional[_RoomEntry], Dict[str, JsonDict], Set[str]]: + return requested_room_entry, {fed_subroom: child_room}, set() + + expected = [ + (fed_space, [fed_subroom]), + (fed_subroom, ()), + ] + + with mock.patch( + "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room_hierarchy", + new=summarize_remote_room_hierarchy, + ): + result = self.get_success( + self.handler.get_room_hierarchy( + create_requester(self.user), + fed_space, + remote_room_hosts=remote_room_hosts, + ) + ) + self._assert_hierarchy(result, expected) + class RoomSummaryTestCase(unittest.HomeserverTestCase): servlets = [ diff --git a/tests/handlers/test_saml.py b/tests/handlers/test_saml.py index 6ab8fda6e7..98a4276a3a 100644 --- a/tests/handlers/test_saml.py +++ b/tests/handlers/test_saml.py @@ -24,7 +24,7 @@ from unittest.mock import AsyncMock, Mock import attr -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import RedirectException from synapse.module_api import ModuleApi @@ -363,6 +363,52 @@ class SamlHandlerTestCase(HomeserverTestCase): auth_provider_session_id=None, ) + @override_config( + { + "saml2_config": { + "attribute_requirements": [ + {"attribute": "userGroup", "one_of": ["staff", "admin"]}, + ], + }, + } + ) + def test_attribute_requirements_one_of(self) -> None: + """The required attributes can be comma-separated.""" + + # stub out the auth handler + auth_handler = self.hs.get_auth_handler() + auth_handler.complete_sso_login = AsyncMock() # type: ignore[method-assign] + + # The response doesn't have the proper department. + saml_response = FakeAuthnResponse( + {"uid": "test_user", "username": "test_user", "userGroup": ["nogroup"]} + ) + request = _mock_request() + self.get_success( + self.handler._handle_authn_response(request, saml_response, "redirect_uri") + ) + auth_handler.complete_sso_login.assert_not_called() + + # Add the proper attributes and it should succeed. + saml_response = FakeAuthnResponse( + {"uid": "test_user", "username": "test_user", "userGroup": ["admin"]} + ) + request.reset_mock() + self.get_success( + self.handler._handle_authn_response(request, saml_response, "redirect_uri") + ) + + # check that the auth handler got called as expected + auth_handler.complete_sso_login.assert_called_once_with( + "@test_user:test", + "saml", + request, + "redirect_uri", + None, + new_user=True, + auth_provider_session_id=None, + ) + def _mock_request() -> Mock: """Returns a mock which will stand in as a SynapseRequest""" diff --git a/tests/handlers/test_send_email.py b/tests/handlers/test_send_email.py index cedcea27d9..5f7839c82c 100644 --- a/tests/handlers/test_send_email.py +++ b/tests/handlers/test_send_email.py @@ -163,6 +163,7 @@ class SendEmailHandlerTestCaseIPv4(HomeserverTestCase): "email": { "notif_from": "noreply@test", "force_tls": True, + "tlsname": "example.org", }, } ) @@ -186,10 +187,9 @@ class SendEmailHandlerTestCaseIPv4(HomeserverTestCase): self.assertEqual(host, self.reactor.lookups["localhost"]) self.assertEqual(port, 465) # We need to make sure that TLS is happenning - self.assertIsInstance( - client_factory._wrappedFactory._testingContextFactory, - ClientTLSOptions, - ) + context_factory = client_factory._wrappedFactory._testingContextFactory + self.assertIsInstance(context_factory, ClientTLSOptions) + self.assertEqual(context_factory._hostname, "example.org") # tlsname # And since we use endpoints, they go through reactor.connectTCP # which works differently to connectSSL on the testing reactor diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 5b7e2937f8..8c390f0c57 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -22,9 +22,9 @@ from typing import AbstractSet, Dict, Mapping, Optional, Set, Tuple from unittest.mock import patch import attr -from parameterized import parameterized +from parameterized import parameterized, parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import ( EventTypes, @@ -43,13 +43,15 @@ from synapse.rest import admin from synapse.rest.client import knock, login, room from synapse.server import HomeServer from synapse.storage.util.id_generators import MultiWriterIdGenerator -from synapse.types import JsonDict, StateMap, StreamToken, UserID -from synapse.types.handlers.sliding_sync import SlidingSyncConfig +from synapse.types import JsonDict, StateMap, StreamToken, UserID, create_requester +from synapse.types.handlers.sliding_sync import PerConnectionState, SlidingSyncConfig from synapse.types.state import StateFilter from synapse.util import Clock from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.test_utils.event_injection import create_event from tests.unittest import HomeserverTestCase, TestCase logger = logging.getLogger(__name__) @@ -572,12 +574,32 @@ class RoomSyncConfigTestCase(TestCase): self._assert_room_config_equal(combined_config, expected, "A into B") -class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): +# FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the +# foreground update for +# `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` (tracked by +# https://github.com/element-hq/synapse/issues/17623) +@parameterized_class( + ("use_new_tables",), + [ + (True,), + (False,), + ], + class_name_func=lambda cls, + num, + params_dict: f"{cls.__name__}_{'new' if params_dict['use_new_tables'] else 'fallback'}", +) +class ComputeInterestedRoomsTestCase(SlidingSyncBase): """ - Tests Sliding Sync handler `get_room_membership_for_user_at_to_token()` to make sure it returns + Tests Sliding Sync handler `compute_interested_rooms()` to make sure it returns the correct list of rooms IDs. """ + # FIXME: We should refactor these tests to run against `compute_interested_rooms(...)` + # instead of just `get_room_membership_for_user_at_to_token(...)` which is only used + # in the fallback path (`_compute_interested_rooms_fallback(...)`). These scenarios do + # well to stress that logic and we shouldn't remove them just because we're removing + # the fallback path (tracked by https://github.com/element-hq/synapse/issues/17623). + servlets = [ admin.register_servlets, knock.register_servlets, @@ -596,6 +618,11 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): self.store = self.hs.get_datastores().main self.event_sources = hs.get_event_sources() self.storage_controllers = hs.get_storage_controllers() + persistence = self.hs.get_storage_controllers().persistence + assert persistence is not None + self.persistence = persistence + + super().prepare(reactor, clock, hs) def test_no_rooms(self) -> None: """ @@ -606,15 +633,28 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): now_token = self.event_sources.get_current_token() - room_id_results, _, _ = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=now_token, to_token=now_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) - self.assertEqual(room_id_results.keys(), set()) + self.assertIncludes(room_id_results, set(), exact=True) def test_get_newly_joined_room(self) -> None: """ @@ -633,22 +673,44 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room_token, to_token=after_room_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual(room_id_results.keys(), {room_id}) + self.assertIncludes( + room_id_results, + {room_id}, + exact=True, + ) # It should be pointing to the join event (latest membership event in the # from/to range) self.assertEqual( - room_id_results[room_id].event_id, + interested_rooms.room_membership_for_user_map[room_id].event_id, join_response["event_id"], ) - self.assertEqual(room_id_results[room_id].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id].membership, + Membership.JOIN, + ) # We should be considered `newly_joined` because we joined during the token # range self.assertTrue(room_id in newly_joined) @@ -668,22 +730,40 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room_token, to_token=after_room_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual(room_id_results.keys(), {room_id}) + self.assertIncludes(room_id_results, {room_id}, exact=True) # It should be pointing to the join event (latest membership event in the # from/to range) self.assertEqual( - room_id_results[room_id].event_id, + interested_rooms.room_membership_for_user_map[room_id].event_id, join_response["event_id"], ) - self.assertEqual(room_id_results[room_id].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id not in newly_joined) self.assertTrue(room_id not in newly_left) @@ -742,46 +822,71 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room_token, to_token=after_room_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Ensure that the invited, ban, and knock rooms show up - self.assertEqual( - room_id_results.keys(), + self.assertIncludes( + room_id_results, { invited_room_id, ban_room_id, knock_room_id, }, + exact=True, ) # It should be pointing to the the respective membership event (latest # membership event in the from/to range) self.assertEqual( - room_id_results[invited_room_id].event_id, + interested_rooms.room_membership_for_user_map[invited_room_id].event_id, invite_response["event_id"], ) - self.assertEqual(room_id_results[invited_room_id].membership, Membership.INVITE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[invited_room_id].membership, + Membership.INVITE, + ) self.assertTrue(invited_room_id not in newly_joined) self.assertTrue(invited_room_id not in newly_left) self.assertEqual( - room_id_results[ban_room_id].event_id, + interested_rooms.room_membership_for_user_map[ban_room_id].event_id, ban_response["event_id"], ) - self.assertEqual(room_id_results[ban_room_id].membership, Membership.BAN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[ban_room_id].membership, + Membership.BAN, + ) self.assertTrue(ban_room_id not in newly_joined) self.assertTrue(ban_room_id not in newly_left) self.assertEqual( - room_id_results[knock_room_id].event_id, + interested_rooms.room_membership_for_user_map[knock_room_id].event_id, knock_room_membership_state_event.event_id, ) - self.assertEqual(room_id_results[knock_room_id].membership, Membership.KNOCK) + self.assertEqual( + interested_rooms.room_membership_for_user_map[knock_room_id].membership, + Membership.KNOCK, + ) self.assertTrue(knock_room_id not in newly_joined) self.assertTrue(knock_room_id not in newly_left) @@ -814,23 +919,43 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_kick_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_kick_token, to_token=after_kick_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # The kicked room should show up - self.assertEqual(room_id_results.keys(), {kick_room_id}) + self.assertIncludes(room_id_results, {kick_room_id}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[kick_room_id].event_id, + interested_rooms.room_membership_for_user_map[kick_room_id].event_id, kick_response["event_id"], ) - self.assertEqual(room_id_results[kick_room_id].membership, Membership.LEAVE) - self.assertNotEqual(room_id_results[kick_room_id].sender, user1_id) + self.assertEqual( + interested_rooms.room_membership_for_user_map[kick_room_id].membership, + Membership.LEAVE, + ) + self.assertNotEqual( + interested_rooms.room_membership_for_user_map[kick_room_id].sender, user1_id + ) # We should *NOT* be `newly_joined` because we were not joined at the the time # of the `to_token`. self.assertTrue(kick_room_id not in newly_joined) @@ -907,16 +1032,29 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): ) self.assertEqual(channel.code, 200, channel.result) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room_forgets, to_token=before_room_forgets, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) # We shouldn't see the room because it was forgotten - self.assertEqual(room_id_results.keys(), set()) + self.assertIncludes(room_id_results, set(), exact=True) def test_newly_left_rooms(self) -> None: """ @@ -927,7 +1065,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave before we calculate the `from_token` room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) - leave_response1 = self.helper.leave(room_id1, user1_id, tok=user1_tok) + _leave_response1 = self.helper.leave(room_id1, user1_id, tok=user1_tok) after_room1_token = self.event_sources.get_current_token() @@ -937,31 +1075,52 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room2_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room2_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual(room_id_results.keys(), {room_id1, room_id2}) - - self.assertEqual( - room_id_results[room_id1].event_id, - leave_response1["event_id"], + # `room_id1` should not show up because it was left before the token range. + # `room_id2` should show up because it is `newly_left` within the token range. + self.assertIncludes( + room_id_results, + {room_id2}, + exact=True, + message="Corresponding map to disambiguate the opaque room IDs: " + + str( + { + "room_id1": room_id1, + "room_id2": room_id2, + } + ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined` or `newly_left` because that happened before - # the from/to range - self.assertTrue(room_id1 not in newly_joined) - self.assertTrue(room_id1 not in newly_left) self.assertEqual( - room_id_results[room_id2].event_id, + interested_rooms.room_membership_for_user_map[room_id2].event_id, leave_response2["event_id"], ) - self.assertEqual(room_id_results[room_id2].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id2].membership, + Membership.LEAVE, + ) # We should *NOT* be `newly_joined` because we are instead `newly_left` self.assertTrue(room_id2 not in newly_joined) self.assertTrue(room_id2 in newly_left) @@ -987,21 +1146,39 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): room_id2 = self.helper.create_room_as(user2_id, tok=user2_tok) self.helper.join(room_id2, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response1["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1027,20 +1204,35 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave the room after we already have our tokens leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # We should still see the room because we were joined during the # from_token/to_token time period. - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1050,7 +1242,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1074,19 +1269,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave the room after we already have our tokens leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # We should still see the room because we were joined before the `from_token` - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1096,7 +1306,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1138,19 +1351,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_response2 = self.helper.join(kick_room_id, user1_id, tok=user1_tok) leave_response = self.helper.leave(kick_room_id, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_kick_token, to_token=after_kick_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # We shouldn't see the room because it was forgotten - self.assertEqual(room_id_results.keys(), {kick_room_id}) + self.assertIncludes(room_id_results, {kick_room_id}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[kick_room_id].event_id, + interested_rooms.room_membership_for_user_map[kick_room_id].event_id, kick_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1162,8 +1390,13 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[kick_room_id].membership, Membership.LEAVE) - self.assertNotEqual(room_id_results[kick_room_id].sender, user1_id) + self.assertEqual( + interested_rooms.room_membership_for_user_map[kick_room_id].membership, + Membership.LEAVE, + ) + self.assertNotEqual( + interested_rooms.room_membership_for_user_map[kick_room_id].sender, user1_id + ) # We should *NOT* be `newly_joined` because we were kicked self.assertTrue(kick_room_id not in newly_joined) self.assertTrue(kick_room_id not in newly_left) @@ -1194,19 +1427,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_response2 = self.helper.join(room_id1, user1_id, tok=user1_tok) leave_response2 = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should still show up because it's newly_left during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, leave_response1["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1218,7 +1466,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.LEAVE, + ) # We should *NOT* be `newly_joined` because we are actually `newly_left` during # the token range self.assertTrue(room_id1 not in newly_joined) @@ -1249,19 +1500,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Join the room after we already have our tokens join_response2 = self.helper.join(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should still show up because it's newly_left during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, leave_response1["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1272,7 +1538,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.LEAVE, + ) # We should *NOT* be `newly_joined` because we are actually `newly_left` during # the token range self.assertTrue(room_id1 not in newly_joined) @@ -1301,48 +1570,54 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Join and leave the room2 before the `to_token` self.helper.join(room_id2, user1_id, tok=user1_tok) - leave_response2 = self.helper.leave(room_id2, user1_id, tok=user1_tok) + _leave_response2 = self.helper.leave(room_id2, user1_id, tok=user1_tok) after_room1_token = self.event_sources.get_current_token() # Join the room2 after we already have our tokens self.helper.join(room_id2, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=None, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Only rooms we were joined to before the `to_token` should show up - self.assertEqual(room_id_results.keys(), {room_id1, room_id2}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # Room1 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response1["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined`/`newly_left` because there is no # `from_token` to define a "live" range to compare against self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) - # Room2 - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id2].event_id, - leave_response2["event_id"], - ) - self.assertEqual(room_id_results[room_id2].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because there is no - # `from_token` to define a "live" range to compare against - self.assertTrue(room_id2 not in newly_joined) - self.assertTrue(room_id2 not in newly_left) - def test_from_token_ahead_of_to_token(self) -> None: """ Test when the provided `from_token` comes after the `to_token`. We should @@ -1365,7 +1640,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Join and leave the room2 before `to_token` _join_room2_response1 = self.helper.join(room_id2, user1_id, tok=user1_tok) - leave_room2_response1 = self.helper.leave(room_id2, user1_id, tok=user1_tok) + _leave_room2_response1 = self.helper.leave(room_id2, user1_id, tok=user1_tok) # Note: These are purposely swapped. The `from_token` should come after # the `to_token` in this test @@ -1390,55 +1665,70 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Join the room4 after we already have our tokens self.helper.join(room_id4, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=from_token, to_token=to_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # In the "current" state snapshot, we're joined to all of the rooms but in the # from/to token range... self.assertIncludes( - room_id_results.keys(), + room_id_results, { # Included because we were joined before both tokens room_id1, - # Included because we had membership before the to_token - room_id2, + # Excluded because we left before the `from_token` and `to_token` + # room_id2, # Excluded because we joined after the `to_token` # room_id3, # Excluded because we joined after the `to_token` # room_id4, }, exact=True, + message="Corresponding map to disambiguate the opaque room IDs: " + + str( + { + "room_id1": room_id1, + "room_id2": room_id2, + "room_id3": room_id3, + "room_id4": room_id4, + } + ), ) # Room1 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_room1_response1["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined`/`newly_left` because we joined `room1` # before either of the tokens self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) - # Room2 - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id2].event_id, - leave_room2_response1["event_id"], - ) - self.assertEqual(room_id_results[room_id2].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because we joined and left - # `room1` before either of the tokens - self.assertTrue(room_id2 not in newly_joined) - self.assertTrue(room_id2 not in newly_left) - def test_leave_before_range_and_join_leave_after_to_token(self) -> None: """ Test old left rooms. But we're also testing that joining and leaving after the @@ -1455,7 +1745,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) # Join and leave the room before the from/to range self.helper.join(room_id1, user1_id, tok=user1_tok) - leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) after_room1_token = self.event_sources.get_current_token() @@ -1463,25 +1753,28 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): self.helper.join(room_id1, user1_id, tok=user1_tok) self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) - self.assertEqual(room_id_results.keys(), {room_id1}) - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id1].event_id, - leave_response["event_id"], - ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because we joined and left - # `room1` before either of the tokens - self.assertTrue(room_id1 not in newly_joined) - self.assertTrue(room_id1 not in newly_left) + self.assertIncludes(room_id_results, set(), exact=True) def test_leave_before_range_and_join_after_to_token(self) -> None: """ @@ -1499,32 +1792,35 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) # Join and leave the room before the from/to range self.helper.join(room_id1, user1_id, tok=user1_tok) - leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) after_room1_token = self.event_sources.get_current_token() # Join the room after we already have our tokens self.helper.join(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) - self.assertEqual(room_id_results.keys(), {room_id1}) - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id1].event_id, - leave_response["event_id"], - ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because we joined and left - # `room1` before either of the tokens - self.assertTrue(room_id1 not in newly_joined) - self.assertTrue(room_id1 not in newly_left) + self.assertIncludes(room_id_results, set(), exact=True) def test_join_leave_multiple_times_during_range_and_after_to_token( self, @@ -1556,19 +1852,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_response3 = self.helper.join(room_id1, user1_id, tok=user1_tok) leave_response3 = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because it was newly_left and joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response2["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1582,7 +1893,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) # We should *NOT* be `newly_left` because we joined during the token range and @@ -1618,19 +1932,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_response3 = self.helper.join(room_id1, user1_id, tok=user1_tok) leave_response3 = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined before the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response2["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1644,7 +1973,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1677,19 +2009,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_respsonse = self.helper.join(room_id1, user1_id, tok=user1_tok) leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were invited before the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, invite_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1700,7 +2047,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.INVITE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.INVITE, + ) # We should *NOT* be `newly_joined` because we were only invited before the # token range self.assertTrue(room_id1 not in newly_joined) @@ -1751,19 +2101,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): tok=user1_tok, ) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_during_token_range_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1778,7 +2143,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1816,19 +2184,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_change1_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_change1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_during_token_range_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1840,7 +2223,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1888,19 +2274,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): tok=user1_tok, ) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined before the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_before_token_range_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1915,18 +2316,22 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) - def test_display_name_changes_leave_after_token_range( + def test_newly_joined_display_name_changes_leave_after_token_range( self, ) -> None: """ Test that we point to the correct membership event within the from/to range even - if there are multiple `join` membership events in a row indicating - `displayname`/`avatar_url` updates and we leave after the `to_token`. + if we are `newly_joined` and there are multiple `join` membership events in a + row indicating `displayname`/`avatar_url` updates and we leave after the + `to_token`. See condition "1a)" comments in the `get_room_membership_for_user_at_to_token()` method. """ @@ -1941,6 +2346,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # leave and can still re-join. room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) join_response = self.helper.join(room_id1, user1_id, tok=user1_tok) + # Update the displayname during the token range displayname_change_during_token_range_response = self.helper.send_state( room_id1, @@ -1970,19 +2376,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave after the token self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_during_token_range_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1997,11 +2418,118 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) + def test_display_name_changes_leave_after_token_range( + self, + ) -> None: + """ + Test that we point to the correct membership event within the from/to range even + if there are multiple `join` membership events in a row indicating + `displayname`/`avatar_url` updates and we leave after the `to_token`. + + See condition "1a)" comments in the `get_room_membership_for_user_at_to_token()` method. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + _before_room1_token = self.event_sources.get_current_token() + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + join_response = self.helper.join(room_id1, user1_id, tok=user1_tok) + + after_join_token = self.event_sources.get_current_token() + + # Update the displayname during the token range + displayname_change_during_token_range_response = self.helper.send_state( + room_id1, + event_type=EventTypes.Member, + state_key=user1_id, + body={ + "membership": Membership.JOIN, + "displayname": "displayname during token range", + }, + tok=user1_tok, + ) + + after_display_name_change_token = self.event_sources.get_current_token() + + # Update the displayname after the token range + displayname_change_after_token_range_response = self.helper.send_state( + room_id1, + event_type=EventTypes.Member, + state_key=user1_id, + body={ + "membership": Membership.JOIN, + "displayname": "displayname after token range", + }, + tok=user1_tok, + ) + + # Leave after the token + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), + from_token=after_join_token, + to_token=after_display_name_change_token, + ) + ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms + + # Room should show up because we were joined during the from/to range + self.assertIncludes(room_id_results, {room_id1}, exact=True) + # It should be pointing to the latest membership event in the from/to range + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].event_id, + displayname_change_during_token_range_response["event_id"], + "Corresponding map to disambiguate the opaque event IDs: " + + str( + { + "join_response": join_response["event_id"], + "displayname_change_during_token_range_response": displayname_change_during_token_range_response[ + "event_id" + ], + "displayname_change_after_token_range_response": displayname_change_after_token_range_response[ + "event_id" + ], + } + ), + ) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) + # We only changed our display name during the token range so we shouldn't be + # considered `newly_joined` or `newly_left` + self.assertTrue(room_id1 not in newly_joined) + self.assertTrue(room_id1 not in newly_left) + def test_display_name_changes_join_after_token_range( self, ) -> None: @@ -2038,16 +2566,29 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): tok=user1_tok, ) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) # Room shouldn't show up because we joined after the from/to range - self.assertEqual(room_id_results.keys(), set()) + self.assertIncludes(room_id_results, set(), exact=True) def test_newly_joined_with_leave_join_in_token_range( self, @@ -2074,22 +2615,40 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_more_changes_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_more_changes_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response2["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be considered `newly_joined` because there is some non-join event in # between our latest join event. self.assertTrue(room_id1 in newly_joined) @@ -2139,19 +2698,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room1_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because it was newly_left and joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_during_token_range_response2["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -2166,7 +2740,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we first joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -2192,7 +2769,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Invited and left the room before the token self.helper.invite(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) - leave_room1_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) + _leave_room1_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) # Invited to room2 invite_room2_response = self.helper.invite( room_id2, src=user2_id, targ=user1_id, tok=user2_tok @@ -2215,45 +2792,52 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave room3 self.helper.leave(room_id3, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room3_token, to_token=after_room3_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual( - room_id_results.keys(), + self.assertIncludes( + room_id_results, { - # Left before the from/to range - room_id1, + # Excluded because we left before the from/to range + # room_id1, # Invited before the from/to range room_id2, # `newly_left` during the from/to range room_id3, }, + exact=True, ) - # Room1 - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id1].event_id, - leave_room1_response["event_id"], - ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because we were invited and left - # before the token range - self.assertTrue(room_id1 not in newly_joined) - self.assertTrue(room_id1 not in newly_left) - # Room2 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id2].event_id, + interested_rooms.room_membership_for_user_map[room_id2].event_id, invite_room2_response["event_id"], ) - self.assertEqual(room_id_results[room_id2].membership, Membership.INVITE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id2].membership, + Membership.INVITE, + ) # We should *NOT* be `newly_joined`/`newly_left` because we were invited before # the token range self.assertTrue(room_id2 not in newly_joined) @@ -2262,10 +2846,13 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Room3 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id3].event_id, + interested_rooms.room_membership_for_user_map[room_id3].event_id, leave_room3_response["event_id"], ) - self.assertEqual(room_id_results[room_id3].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id3].membership, + Membership.LEAVE, + ) # We should be `newly_left` because we were invited and left during # the token range self.assertTrue(room_id3 not in newly_joined) @@ -2282,7 +2869,16 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): user2_tok = self.login(user2_id, "pass") # The room where the state reset will happen - room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) + room_id1 = self.helper.create_room_as( + user2_id, + is_public=True, + tok=user2_tok, + ) + # Create a dummy event for us to point back to for the state reset + dummy_event_response = self.helper.send(room_id1, "test", tok=user2_tok) + dummy_event_id = dummy_event_response["event_id"] + + # Join after the dummy event join_response1 = self.helper.join(room_id1, user1_id, tok=user1_tok) # Join another room so we don't hit the short-circuit and return early if they @@ -2292,95 +2888,106 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): before_reset_token = self.event_sources.get_current_token() - # Send another state event to make a position for the state reset to happen at - dummy_state_response = self.helper.send_state( - room_id1, - event_type="foobarbaz", - state_key="", - body={"foo": "bar"}, - tok=user2_tok, + # Trigger a state reset + join_rule_event, join_rule_context = self.get_success( + create_event( + self.hs, + prev_event_ids=[dummy_event_id], + type=EventTypes.JoinRules, + state_key="", + content={"join_rule": JoinRules.INVITE}, + sender=user2_id, + room_id=room_id1, + room_version=self.get_success(self.store.get_room_version_id(room_id1)), + ) ) - dummy_state_pos = self.get_success( - self.store.get_position_for_event(dummy_state_response["event_id"]) + _, join_rule_event_pos, _ = self.get_success( + self.persistence.persist_event(join_rule_event, join_rule_context) ) - # Mock a state reset removing the membership for user1 in the current state - self.get_success( - self.store.db_pool.simple_delete( - table="current_state_events", - keyvalues={ - "room_id": room_id1, - "type": EventTypes.Member, - "state_key": user1_id, - }, - desc="state reset user in current_state_events", - ) - ) - self.get_success( - self.store.db_pool.simple_delete( - table="local_current_membership", - keyvalues={ - "room_id": room_id1, - "user_id": user1_id, - }, - desc="state reset user in local_current_membership", - ) - ) - self.get_success( - self.store.db_pool.simple_insert( - table="current_state_delta_stream", - values={ - "stream_id": dummy_state_pos.stream, - "room_id": room_id1, - "type": EventTypes.Member, - "state_key": user1_id, - "event_id": None, - "prev_event_id": join_response1["event_id"], - "instance_name": dummy_state_pos.instance_name, - }, - desc="state reset user in current_state_delta_stream", - ) - ) - - # Manually bust the cache since we we're just manually messing with the database - # and not causing an actual state reset. - self.store._membership_stream_cache.entity_has_changed( - user1_id, dummy_state_pos.stream - ) + # Ensure that the state reset worked and only user2 is in the room now + users_in_room = self.get_success(self.store.get_users_in_room(room_id1)) + self.assertIncludes(set(users_in_room), {user2_id}, exact=True) after_reset_token = self.event_sources.get_current_token() # The function under test - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_reset_token, to_token=after_reset_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room1 should show up because it was `newly_left` via state reset during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1, room_id2}) + self.assertIncludes(room_id_results, {room_id1, room_id2}, exact=True) # It should be pointing to no event because we were removed from the room # without a corresponding leave event self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, None, + "Corresponding map to disambiguate the opaque event IDs: " + + str( + { + "join_response1": join_response1["event_id"], + } + ), ) # State reset caused us to leave the room and there is no corresponding leave event - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.LEAVE, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) # We should be `newly_left` because we were removed via state reset during the from/to range self.assertTrue(room_id1 in newly_left) -class GetRoomMembershipForUserAtToTokenShardTestCase(BaseMultiWorkerStreamTestCase): +# FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the +# foreground update for +# `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` (tracked by +# https://github.com/element-hq/synapse/issues/17623) +@parameterized_class( + ("use_new_tables",), + [ + (True,), + (False,), + ], + class_name_func=lambda cls, + num, + params_dict: f"{cls.__name__}_{'new' if params_dict['use_new_tables'] else 'fallback'}", +) +class ComputeInterestedRoomsShardTestCase( + BaseMultiWorkerStreamTestCase, SlidingSyncBase +): """ - Tests Sliding Sync handler `get_room_membership_for_user_at_to_token()` to make sure it works with + Tests Sliding Sync handler `compute_interested_rooms()` to make sure it works with sharded event stream_writers enabled """ + # FIXME: We should refactor these tests to run against `compute_interested_rooms(...)` + # instead of just `get_room_membership_for_user_at_to_token(...)` which is only used + # in the fallback path (`_compute_interested_rooms_fallback(...)`). These scenarios do + # well to stress that logic and we shouldn't remove them just because we're removing + # the fallback path (tracked by https://github.com/element-hq/synapse/issues/17623). + servlets = [ admin.register_servlets_for_client_rest_resource, room.register_servlets, @@ -2475,7 +3082,7 @@ class GetRoomMembershipForUserAtToTokenShardTestCase(BaseMultiWorkerStreamTestCa join_response1 = self.helper.join(room_id1, user1_id, tok=user1_tok) join_response2 = self.helper.join(room_id2, user1_id, tok=user1_tok) # Leave room2 - leave_room2_response = self.helper.leave(room_id2, user1_id, tok=user1_tok) + _leave_room2_response = self.helper.leave(room_id2, user1_id, tok=user1_tok) join_response3 = self.helper.join(room_id3, user1_id, tok=user1_tok) # Leave room3 self.helper.leave(room_id3, user1_id, tok=user1_tok) @@ -2565,57 +3172,74 @@ class GetRoomMembershipForUserAtToTokenShardTestCase(BaseMultiWorkerStreamTestCa self.get_success(actx.__aexit__(None, None, None)) # The function under test - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_stuck_activity_token, to_token=stuck_activity_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual( - room_id_results.keys(), + self.assertIncludes( + room_id_results, { room_id1, - room_id2, + # Excluded because we left before the from/to range and the second join + # event happened while worker2 was stuck and technically occurs after + # the `stuck_activity_token`. + # room_id2, room_id3, }, + exact=True, + message="Corresponding map to disambiguate the opaque room IDs: " + + str( + { + "room_id1": room_id1, + "room_id2": room_id2, + "room_id3": room_id3, + } + ), ) # Room1 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_room1_response["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) - # Room2 - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id2].event_id, - leave_room2_response["event_id"], - ) - self.assertEqual(room_id_results[room_id2].membership, Membership.LEAVE) - # room_id2 should *NOT* be considered `newly_left` because we left before the - # from/to range and the join event during the range happened while worker2 was - # stuck. This means that from the perspective of the master, where the - # `stuck_activity_token` is generated, the stream position for worker2 wasn't - # advanced to the join yet. Looking at the `instance_map`, the join technically - # comes after `stuck_activity_token`. - self.assertTrue(room_id2 not in newly_joined) - self.assertTrue(room_id2 not in newly_left) - # Room3 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id3].event_id, + interested_rooms.room_membership_for_user_map[room_id3].event_id, join_on_worker3_response["event_id"], ) - self.assertEqual(room_id_results[room_id3].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id3].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id3 in newly_joined) self.assertTrue(room_id3 not in newly_left) @@ -2645,6 +3269,9 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): self.store = self.hs.get_datastores().main self.event_sources = hs.get_event_sources() self.storage_controllers = hs.get_storage_controllers() + persistence = self.hs.get_storage_controllers().persistence + assert persistence is not None + self.persistence = persistence def _get_sync_room_ids_for_user( self, @@ -2687,7 +3314,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): to_token=now_token, ) - self.assertEqual(room_id_results.keys(), set()) + self.assertIncludes(room_id_results.keys(), set(), exact=True) def test_basic_rooms(self) -> None: """ @@ -2753,7 +3380,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ) # Ensure that the invited, ban, and knock rooms show up - self.assertEqual( + self.assertIncludes( room_id_results.keys(), { join_room_id, @@ -2761,6 +3388,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ban_room_id, knock_room_id, }, + exact=True, ) # It should be pointing to the the respective membership event (latest # membership event in the from/to range) @@ -2824,7 +3452,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ) # Only the `newly_left` room should show up - self.assertEqual(room_id_results.keys(), {room_id2}) + self.assertIncludes(room_id_results.keys(), {room_id2}, exact=True) self.assertEqual( room_id_results[room_id2].event_id, _leave_response2["event_id"], @@ -2869,7 +3497,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ) # The kicked room should show up - self.assertEqual(room_id_results.keys(), {kick_room_id}) + self.assertIncludes(room_id_results.keys(), {kick_room_id}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( room_id_results[kick_room_id].event_id, @@ -2893,8 +3521,17 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): user2_tok = self.login(user2_id, "pass") # The room where the state reset will happen - room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) - join_response1 = self.helper.join(room_id1, user1_id, tok=user1_tok) + room_id1 = self.helper.create_room_as( + user2_id, + is_public=True, + tok=user2_tok, + ) + # Create a dummy event for us to point back to for the state reset + dummy_event_response = self.helper.send(room_id1, "test", tok=user2_tok) + dummy_event_id = dummy_event_response["event_id"] + + # Join after the dummy event + self.helper.join(room_id1, user1_id, tok=user1_tok) # Join another room so we don't hit the short-circuit and return early if they # have no room membership @@ -2903,61 +3540,26 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): before_reset_token = self.event_sources.get_current_token() - # Send another state event to make a position for the state reset to happen at - dummy_state_response = self.helper.send_state( - room_id1, - event_type="foobarbaz", - state_key="", - body={"foo": "bar"}, - tok=user2_tok, + # Trigger a state reset + join_rule_event, join_rule_context = self.get_success( + create_event( + self.hs, + prev_event_ids=[dummy_event_id], + type=EventTypes.JoinRules, + state_key="", + content={"join_rule": JoinRules.INVITE}, + sender=user2_id, + room_id=room_id1, + room_version=self.get_success(self.store.get_room_version_id(room_id1)), + ) ) - dummy_state_pos = self.get_success( - self.store.get_position_for_event(dummy_state_response["event_id"]) + _, join_rule_event_pos, _ = self.get_success( + self.persistence.persist_event(join_rule_event, join_rule_context) ) - # Mock a state reset removing the membership for user1 in the current state - self.get_success( - self.store.db_pool.simple_delete( - table="current_state_events", - keyvalues={ - "room_id": room_id1, - "type": EventTypes.Member, - "state_key": user1_id, - }, - desc="state reset user in current_state_events", - ) - ) - self.get_success( - self.store.db_pool.simple_delete( - table="local_current_membership", - keyvalues={ - "room_id": room_id1, - "user_id": user1_id, - }, - desc="state reset user in local_current_membership", - ) - ) - self.get_success( - self.store.db_pool.simple_insert( - table="current_state_delta_stream", - values={ - "stream_id": dummy_state_pos.stream, - "room_id": room_id1, - "type": EventTypes.Member, - "state_key": user1_id, - "event_id": None, - "prev_event_id": join_response1["event_id"], - "instance_name": dummy_state_pos.instance_name, - }, - desc="state reset user in current_state_delta_stream", - ) - ) - - # Manually bust the cache since we we're just manually messing with the database - # and not causing an actual state reset. - self.store._membership_stream_cache.entity_has_changed( - user1_id, dummy_state_pos.stream - ) + # Ensure that the state reset worked and only user2 is in the room now + users_in_room = self.get_success(self.store.get_users_in_room(room_id1)) + self.assertIncludes(set(users_in_room), {user2_id}, exact=True) after_reset_token = self.event_sources.get_current_token() @@ -2969,7 +3571,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ) # Room1 should show up because it was `newly_left` via state reset during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1, room_id2}) + self.assertIncludes(room_id_results.keys(), {room_id1, room_id2}, exact=True) # It should be pointing to no event because we were removed from the room # without a corresponding leave event self.assertEqual( diff --git a/tests/handlers/test_sso.py b/tests/handlers/test_sso.py index 25e9130aaf..896e4fac9a 100644 --- a/tests/handlers/test_sso.py +++ b/tests/handlers/test_sso.py @@ -21,7 +21,7 @@ from http import HTTPStatus from typing import BinaryIO, Callable, Dict, List, Optional, Tuple from unittest.mock import Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.http_headers import Headers from synapse.api.errors import Codes, SynapseError diff --git a/tests/handlers/test_stats.py b/tests/handlers/test_stats.py index bdb6fdb120..cd17cd86e0 100644 --- a/tests/handlers/test_stats.py +++ b/tests/handlers/test_stats.py @@ -20,7 +20,7 @@ from typing import Any, Dict, List, Optional, Tuple, cast -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, room diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 9dd0e98971..9d3e88c126 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -17,13 +17,14 @@ # [This file includes modifications made by New Vector Limited] # # +from http import HTTPStatus from typing import Collection, ContextManager, List, Optional from unittest.mock import AsyncMock, Mock, patch from parameterized import parameterized, parameterized_class from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules from synapse.api.errors import Codes, ResourceLimitError @@ -36,7 +37,6 @@ from synapse.handlers.sync import ( SyncConfig, SyncRequestKey, SyncResult, - SyncVersion, TimelineBatch, ) from synapse.rest import admin @@ -112,7 +112,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( requester, sync_config, - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -123,7 +122,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( requester, sync_config, - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ), ResourceLimitError, @@ -141,7 +139,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( requester, sync_config, - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ), ResourceLimitError, @@ -166,7 +163,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): sync_config=generate_sync_config( user, device_id="dev", use_state_after=self.use_state_after ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -202,7 +198,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): sync_config=generate_sync_config( user, use_state_after=self.use_state_after ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -217,7 +212,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): sync_config=generate_sync_config( user, device_id="dev", use_state_after=self.use_state_after ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=initial_result.next_batch, ) @@ -251,7 +245,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): sync_config=generate_sync_config( user, use_state_after=self.use_state_after ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -266,7 +259,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): sync_config=generate_sync_config( user, device_id="dev", use_state_after=self.use_state_after ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=initial_result.next_batch, ) @@ -309,7 +301,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( create_requester(owner), generate_sync_config(owner, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -335,7 +326,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( eve_requester, eve_sync_config, - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -347,14 +337,21 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): # the prev_events used when creating the join event, such that the ban does not # precede the join. with self._patch_get_latest_events([last_room_creation_event_id]): - self.helper.join(room_id, eve, tok=eve_token) + self.helper.join( + room_id, + eve, + tok=eve_token, + # Previously, this join would succeed but now we expect it to fail at + # this point. The rest of the test is for the case when this used to + # succeed. + expect_code=HTTPStatus.FORBIDDEN, + ) # Eve makes a second, incremental sync. eve_incremental_sync_after_join: SyncResult = self.get_success( self.sync_handler.wait_for_sync_for_user( eve_requester, eve_sync_config, - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=eve_sync_after_ban.next_batch, ) @@ -367,7 +364,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( eve_requester, eve_sync_config, - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=None, ) @@ -402,7 +398,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( alice_requester, generate_sync_config(alice, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -432,7 +427,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): ), use_state_after=self.use_state_after, ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=initial_sync_result.next_batch, ) @@ -478,7 +472,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( alice_requester, generate_sync_config(alice, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -518,7 +511,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): ), use_state_after=self.use_state_after, ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=initial_sync_result.next_batch, ) @@ -567,7 +559,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( alice_requester, generate_sync_config(alice, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -594,7 +585,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): ), use_state_after=self.use_state_after, ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=initial_sync_result.next_batch, ) @@ -634,7 +624,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): ), use_state_after=self.use_state_after, ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=incremental_sync.next_batch, ) @@ -708,7 +697,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( alice_requester, generate_sync_config(alice, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -734,7 +722,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): ), use_state_after=self.use_state_after, ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -760,7 +747,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( alice_requester, generate_sync_config(alice, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=initial_sync_result.next_batch, ) @@ -824,7 +810,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( bob_requester, generate_sync_config(bob, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -858,7 +843,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): filter_collection=FilterCollection(self.hs, filter_dict), use_state_after=self.use_state_after, ), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=None if initial_sync else initial_sync_result.next_batch, ) @@ -958,7 +942,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( create_requester(user), generate_sync_config(user, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -1007,7 +990,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( create_requester(user2), generate_sync_config(user2, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -1033,7 +1015,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( create_requester(user), generate_sync_config(user, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), ) ) @@ -1070,7 +1051,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( create_requester(user), generate_sync_config(user, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=since_token, timeout=0, @@ -1125,7 +1105,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.sync_handler.wait_for_sync_for_user( create_requester(user), generate_sync_config(user, use_state_after=self.use_state_after), - sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=since_token, timeout=0, diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 9d8960315f..614b12c62a 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -26,7 +26,7 @@ from unittest.mock import ANY, AsyncMock, Mock, call from netaddr import IPSet -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource from synapse.api.constants import EduTypes @@ -86,11 +86,13 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase): self.mock_federation_client = AsyncMock(spec=["put_json"]) self.mock_federation_client.put_json.return_value = (200, "OK") self.mock_federation_client.agent = MatrixFederationAgent( - reactor, + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, ip_blocklist=IPSet(), + proxy_config=None, ) # the tests assume that we are starting at unix time 1000 diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index 878d9683b6..7458fe0885 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -21,7 +21,7 @@ from typing import Any, Tuple from unittest.mock import AsyncMock, Mock, patch from urllib.parse import quote -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import UserTypes @@ -31,7 +31,7 @@ from synapse.appservice import ApplicationService from synapse.rest.client import login, register, room, user_directory from synapse.server import HomeServer from synapse.storage.roommember import ProfileInfo -from synapse.types import JsonDict, UserProfile, create_requester +from synapse.types import JsonDict, UserID, UserProfile, create_requester from synapse.util import Clock from tests import unittest @@ -78,7 +78,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, # Note: this user does not match the regex above, so that tests # can distinguish the sender from the AS user. - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) mock_load_appservices = Mock(return_value=[self.appservice]) @@ -196,7 +196,9 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): user = self.register_user("user", "pass") token = self.login(user, "pass") room = self.helper.create_room_as(user, is_public=True, tok=token) - self.helper.join(room, self.appservice.sender, tok=self.appservice.token) + self.helper.join( + room, self.appservice.sender.to_string(), tok=self.appservice.token + ) self._check_only_one_user_in_directory(user, room) def test_search_term_with_colon_in_it_does_not_raise(self) -> None: @@ -433,7 +435,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): def test_handle_local_profile_change_with_appservice_sender(self) -> None: # profile is not in directory profile = self.get_success( - self.store._get_user_in_directory(self.appservice.sender) + self.store._get_user_in_directory(self.appservice.sender.to_string()) ) self.assertIsNone(profile) @@ -441,13 +443,13 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): profile_info = ProfileInfo(avatar_url="avatar_url", display_name="4L1c3") self.get_success( self.handler.handle_local_profile_change( - self.appservice.sender, profile_info + self.appservice.sender.to_string(), profile_info ) ) # profile is still not in directory profile = self.get_success( - self.store._get_user_in_directory(self.appservice.sender) + self.store._get_user_in_directory(self.appservice.sender.to_string()) ) self.assertIsNone(profile) @@ -796,6 +798,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): s = self.get_success(self.handler.search_users(u1, "user2", 10)) self.assertEqual(len(s["results"]), 1) + # Kept old spam checker without `requester_id` tests for backwards compatibility. async def allow_all(user_profile: UserProfile) -> bool: # Allow all users. return False @@ -809,6 +812,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): s = self.get_success(self.handler.search_users(u1, "user2", 10)) self.assertEqual(len(s["results"]), 1) + # Kept old spam checker without `requester_id` tests for backwards compatibility. # Configure a spam checker that filters all users. async def block_all(user_profile: UserProfile) -> bool: # All users are spammy. @@ -820,6 +824,40 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): s = self.get_success(self.handler.search_users(u1, "user2", 10)) self.assertEqual(len(s["results"]), 0) + async def allow_all_expects_requester_id( + user_profile: UserProfile, requester_id: str + ) -> bool: + self.assertEqual(requester_id, u1) + # Allow all users. + return False + + # Configure a spam checker that does not filter any users. + spam_checker = self.hs.get_module_api_callbacks().spam_checker + spam_checker._check_username_for_spam_callbacks = [ + allow_all_expects_requester_id + ] + + # The results do not change: + # We get one search result when searching for user2 by user1. + s = self.get_success(self.handler.search_users(u1, "user2", 10)) + self.assertEqual(len(s["results"]), 1) + + # Configure a spam checker that filters all users. + async def block_all_expects_requester_id( + user_profile: UserProfile, requester_id: str + ) -> bool: + self.assertEqual(requester_id, u1) + # All users are spammy. + return True + + spam_checker._check_username_for_spam_callbacks = [ + block_all_expects_requester_id + ] + + # User1 now gets no search results for any of the other users. + s = self.get_success(self.handler.search_users(u1, "user2", 10)) + self.assertEqual(len(s["results"]), 0) + @override_config( { "spam_checker": { @@ -956,6 +994,67 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): [self.assertIn(user, local_users) for user in received_user_id_ordering[:3]] [self.assertIn(user, remote_users) for user in received_user_id_ordering[3:]] + @override_config( + { + "user_directory": { + "enabled": True, + "search_all_users": True, + "exclude_remote_users": True, + } + } + ) + def test_exclude_remote_users(self) -> None: + """Tests that only local users are returned when + user_directory.exclude_remote_users is True. + """ + + # Create a room and few users to test the directory with + searching_user = self.register_user("searcher", "password") + searching_user_tok = self.login("searcher", "password") + + room_id = self.helper.create_room_as( + searching_user, + room_version=RoomVersions.V1.identifier, + tok=searching_user_tok, + ) + + # Create a few local users and join them to the room + local_user_1 = self.register_user("user_xxxxx", "password") + local_user_2 = self.register_user("user_bbbbb", "password") + local_user_3 = self.register_user("user_zzzzz", "password") + + self._add_user_to_room(room_id, RoomVersions.V1, local_user_1) + self._add_user_to_room(room_id, RoomVersions.V1, local_user_2) + self._add_user_to_room(room_id, RoomVersions.V1, local_user_3) + + # Create a few "remote" users and join them to the room + remote_user_1 = "@user_aaaaa:remote_server" + remote_user_2 = "@user_yyyyy:remote_server" + remote_user_3 = "@user_ccccc:remote_server" + self._add_user_to_room(room_id, RoomVersions.V1, remote_user_1) + self._add_user_to_room(room_id, RoomVersions.V1, remote_user_2) + self._add_user_to_room(room_id, RoomVersions.V1, remote_user_3) + + local_users = [local_user_1, local_user_2, local_user_3] + remote_users = [remote_user_1, remote_user_2, remote_user_3] + + # The local searching user searches for the term "user", which other users have + # in their user id + results = self.get_success( + self.handler.search_users(searching_user, "user", 20) + )["results"] + received_user_ids = [result["user_id"] for result in results] + + for user in local_users: + self.assertIn( + user, received_user_ids, f"Local user {user} not found in results" + ) + + for user in remote_users: + self.assertNotIn( + user, received_user_ids, f"Remote user {user} should not be in results" + ) + def _add_user_to_room( self, room_id: str, @@ -1081,10 +1180,10 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): for use_numeric in [False, True]: if use_numeric: prefix1 = f"{i}" - prefix2 = f"{i+1}" + prefix2 = f"{i + 1}" else: prefix1 = f"a{i}" - prefix2 = f"a{i+1}" + prefix2 = f"a{i + 1}" local_user_1 = self.register_user(f"user{char}{prefix1}", "password") local_user_2 = self.register_user(f"user{char}{prefix2}", "password") diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index 6e9a15c8ee..3d3904eac7 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -19,8 +19,11 @@ # # +import logging +import platform + from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer from synapse.util import Clock @@ -29,6 +32,8 @@ from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.utils import test_timeout +logger = logging.getLogger(__name__) + class WorkerLockTestCase(unittest.HomeserverTestCase): def prepare( @@ -53,12 +58,29 @@ class WorkerLockTestCase(unittest.HomeserverTestCase): def test_lock_contention(self) -> None: """Test lock contention when a lot of locks wait on a single worker""" - + nb_locks_to_test = 500 + current_machine = platform.machine().lower() + if current_machine.startswith("riscv"): + # RISC-V specific settings + timeout_seconds = 15 # Increased timeout for RISC-V + # add a print or log statement here for visibility in CI logs + logger.info( # use logger.info + "Detected RISC-V architecture (%s). " + "Adjusting test_lock_contention: timeout=%ss", + current_machine, + timeout_seconds, + ) + else: + # Settings for other architectures + timeout_seconds = 5 # It takes around 0.5s on a 5+ years old laptop - with test_timeout(5): - nb_locks = 500 - d = self._take_locks(nb_locks) - self.assertEqual(self.get_success(d), nb_locks) + with test_timeout(timeout_seconds): # Use the dynamically set timeout + d = self._take_locks( + nb_locks_to_test + ) # Use the (potentially adjusted) number of locks + self.assertEqual( + self.get_success(d), nb_locks_to_test + ) # Assert against the used number of locks async def _take_locks(self, nb_locks: int) -> int: locks = [ diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 0fbb4db2f7..12428e64a9 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -45,6 +45,7 @@ from twisted.web.http_headers import Headers from twisted.web.iweb import IPolicyForHTTPS, IResponse from synapse.config.homeserver import HomeServerConfig +from synapse.config.server import parse_proxy_config from synapse.crypto.context_factory import FederationPolicyForHTTPS from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent from synapse.http.federation.srv_resolver import Server, SrvResolver @@ -85,15 +86,20 @@ class MatrixFederationAgentTests(unittest.TestCase): self.tls_factory = FederationPolicyForHTTPS(config) self.well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache( - "test_cache", timer=self.reactor.seconds + cache_name="test_cache", + server_name="test_server", + timer=self.reactor.seconds, ) self.had_well_known_cache: TTLCache[bytes, bool] = TTLCache( - "test_cache", timer=self.reactor.seconds + cache_name="test_cache", + server_name="test_server", + timer=self.reactor.seconds, ) self.well_known_resolver = WellKnownResolver( - self.reactor, - Agent(self.reactor, contextFactory=self.tls_factory), - b"test-agent", + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=self.reactor, + agent=Agent(self.reactor, contextFactory=self.tls_factory), + user_agent=b"test-agent", well_known_cache=self.well_known_cache, had_well_known_cache=self.had_well_known_cache, ) @@ -269,11 +275,13 @@ class MatrixFederationAgentTests(unittest.TestCase): because it is created too early during setUp """ return MatrixFederationAgent( + server_name="OUR_STUB_HOMESERVER_NAME", reactor=cast(ISynapseReactor, self.reactor), tls_client_options_factory=self.tls_factory, user_agent=b"test-agent", # Note that this is unused since _well_known_resolver is provided. ip_allowlist=IPSet(), ip_blocklist=IPSet(), + proxy_config=parse_proxy_config({}), _srv_resolver=self.mock_resolver, _well_known_resolver=self.well_known_resolver, ) @@ -1011,16 +1019,19 @@ class MatrixFederationAgentTests(unittest.TestCase): # Build a new agent and WellKnownResolver with a different tls factory tls_factory = FederationPolicyForHTTPS(config) agent = MatrixFederationAgent( + server_name="OUR_STUB_HOMESERVER_NAME", reactor=self.reactor, tls_client_options_factory=tls_factory, user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below. ip_allowlist=IPSet(), ip_blocklist=IPSet(), + proxy_config=None, _srv_resolver=self.mock_resolver, _well_known_resolver=WellKnownResolver( - cast(ISynapseReactor, self.reactor), - Agent(self.reactor, contextFactory=tls_factory), - b"test-agent", + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=cast(ISynapseReactor, self.reactor), + agent=Agent(self.reactor, contextFactory=tls_factory), + user_agent=b"test-agent", well_known_cache=self.well_known_cache, had_well_known_cache=self.had_well_known_cache, ), @@ -1822,7 +1833,7 @@ def _get_test_protocol_factory() -> IProtocolFactory: def _log_request(request: str) -> None: """Implements Factory.log, which is expected by Request.finish""" - logger.info(f"Completed request {request}") + logger.info("Completed request %s", request) @implementer(IPolicyForHTTPS) diff --git a/tests/http/server/_base.py b/tests/http/server/_base.py index dff5a5d262..393f3ab0bd 100644 --- a/tests/http/server/_base.py +++ b/tests/http/server/_base.py @@ -40,8 +40,8 @@ from unittest.mock import Mock from twisted.internet.defer import Deferred from twisted.internet.error import ConnectionDone +from twisted.internet.testing import MemoryReactorClock from twisted.python.failure import Failure -from twisted.test.proto_helpers import MemoryReactorClock from twisted.web.server import Site from synapse.http.server import ( diff --git a/tests/http/test_client.py b/tests/http/test_client.py index ac6470ebbd..a02f6fc728 100644 --- a/tests/http/test_client.py +++ b/tests/http/test_client.py @@ -27,8 +27,8 @@ from netaddr import IPSet from twisted.internet.defer import Deferred from twisted.internet.error import DNSLookupError +from twisted.internet.testing import AccumulatingProtocol from twisted.python.failure import Failure -from twisted.test.proto_helpers import AccumulatingProtocol from twisted.web.client import Agent, ResponseDone from twisted.web.iweb import UNKNOWN_LENGTH diff --git a/tests/http/test_matrixfederationclient.py b/tests/http/test_matrixfederationclient.py index e34df54e13..224883b635 100644 --- a/tests/http/test_matrixfederationclient.py +++ b/tests/http/test_matrixfederationclient.py @@ -27,7 +27,7 @@ from parameterized import parameterized from twisted.internet import defer from twisted.internet.defer import Deferred, TimeoutError from twisted.internet.error import ConnectingCancelledError, DNSLookupError -from twisted.test.proto_helpers import MemoryReactor, StringTransport +from twisted.internet.testing import MemoryReactor, StringTransport from twisted.web.client import Agent, ResponseNeverReceived from twisted.web.http import HTTPChannel from twisted.web.http_headers import Headers @@ -436,8 +436,7 @@ class FederationClientTests(HomeserverTestCase): # Send it the HTTP response client.dataReceived( - b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" - b"Server: Fake\r\n\r\n" + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nServer: Fake\r\n\r\n" ) # Push by enough to time it out @@ -691,10 +690,7 @@ class FederationClientTests(HomeserverTestCase): # Send it a huge HTTP response protocol.dataReceived( - b"HTTP/1.1 200 OK\r\n" - b"Server: Fake\r\n" - b"Content-Type: application/json\r\n" - b"\r\n" + b"HTTP/1.1 200 OK\r\nServer: Fake\r\nContent-Type: application/json\r\n\r\n" ) self.pump() diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py index f71e4c2b8f..5bc5d18d81 100644 --- a/tests/http/test_proxyagent.py +++ b/tests/http/test_proxyagent.py @@ -39,6 +39,7 @@ from twisted.internet.protocol import Factory, Protocol from twisted.protocols.tls import TLSMemoryBIOProtocol from twisted.web.http import HTTPChannel +from synapse.config.server import ProxyConfig, parse_proxy_config from synapse.http.client import BlocklistingReactorWrapper from synapse.http.connectproxyclient import BasicProxyCredentials from synapse.http.proxyagent import ProxyAgent, parse_proxy @@ -241,7 +242,7 @@ class TestBasicProxyCredentials(TestCase): ) -class MatrixFederationAgentTests(TestCase): +class ProxyAgentTests(TestCase): def setUp(self) -> None: self.reactor = ThreadedMemoryReactorClock() @@ -379,27 +380,40 @@ class MatrixFederationAgentTests(TestCase): self.assertEqual(body, b"result") def test_http_request(self) -> None: - agent = ProxyAgent(self.reactor) + agent = ProxyAgent(reactor=self.reactor) self._test_request_direct_connection(agent, b"http", b"test.com", b"") def test_https_request(self) -> None: - agent = ProxyAgent(self.reactor, contextFactory=get_test_https_policy()) + agent = ProxyAgent(reactor=self.reactor, contextFactory=get_test_https_policy()) self._test_request_direct_connection(agent, b"https", b"test.com", b"abc") - def test_http_request_use_proxy_empty_environment(self) -> None: - agent = ProxyAgent(self.reactor, use_proxy=True) + def test_http_request_proxy_config_empty_environment(self) -> None: + agent = ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config({}), + ) self._test_request_direct_connection(agent, b"http", b"test.com", b"") @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "NO_PROXY": "test.com"}) def test_http_request_via_uppercase_no_proxy(self) -> None: - agent = ProxyAgent(self.reactor, use_proxy=True) + """ + Ensure hosts listed in the NO_PROXY environment variable are not sent via the + proxy. + """ + agent = ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config({}), + ) self._test_request_direct_connection(agent, b"http", b"test.com", b"") @patch.dict( os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "test.com,unused.com"} ) def test_http_request_via_no_proxy(self) -> None: - agent = ProxyAgent(self.reactor, use_proxy=True) + agent = ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config({}), + ) self._test_request_direct_connection(agent, b"http", b"test.com", b"") @patch.dict( @@ -407,23 +421,26 @@ class MatrixFederationAgentTests(TestCase): ) def test_https_request_via_no_proxy(self) -> None: agent = ProxyAgent( - self.reactor, + reactor=self.reactor, contextFactory=get_test_https_policy(), - use_proxy=True, + proxy_config=parse_proxy_config({}), ) self._test_request_direct_connection(agent, b"https", b"test.com", b"abc") @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "*"}) def test_http_request_via_no_proxy_star(self) -> None: - agent = ProxyAgent(self.reactor, use_proxy=True) + agent = ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config({}), + ) self._test_request_direct_connection(agent, b"http", b"test.com", b"") @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "*"}) def test_https_request_via_no_proxy_star(self) -> None: agent = ProxyAgent( - self.reactor, + reactor=self.reactor, contextFactory=get_test_https_policy(), - use_proxy=True, + proxy_config=parse_proxy_config({}), ) self._test_request_direct_connection(agent, b"https", b"test.com", b"abc") @@ -433,9 +450,72 @@ class MatrixFederationAgentTests(TestCase): Tests that requests can be made through a proxy. """ self._do_http_request_via_proxy( - expect_proxy_ssl=False, expected_auth_credentials=None + proxy_config=parse_proxy_config({}), + expect_proxy_ssl=False, + expected_auth_credentials=None, ) + def test_given_http_proxy_config(self) -> None: + self._do_http_request_via_proxy( + proxy_config=parse_proxy_config({"http_proxy": "proxy.com:8888"}), + expect_proxy_ssl=False, + expected_auth_credentials=None, + ) + + def test_given_https_proxy_config(self) -> None: + self._do_https_request_via_proxy( + proxy_config=parse_proxy_config({"https_proxy": "proxy.com"}), + expect_proxy_ssl=False, + expected_auth_credentials=None, + ) + + def test_given_no_proxy_hosts_config(self) -> None: + agent = ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config( + {"http_proxy": "proxy.com:8888", "no_proxy_hosts": ["test.com"]} + ), + ) + self._test_request_direct_connection(agent, b"http", b"test.com", b"") + + @patch.dict( + os.environ, + {"http_proxy": "unused.com", "no_proxy": "unused.com"}, + ) + def test_given_http_proxy_config_overrides_environment_config(self) -> None: + """Tests that the given `http_proxy` in file config overrides the environment config.""" + self._do_http_request_via_proxy( + proxy_config=parse_proxy_config({"http_proxy": "proxy.com:8888"}), + expect_proxy_ssl=False, + expected_auth_credentials=None, + ) + + @patch.dict( + os.environ, + {"https_proxy": "unused.com", "no_proxy": "unused.com"}, + ) + def test_given_https_proxy_config_overrides_environment_config(self) -> None: + """Tests that the given `https_proxy` in file config overrides the environment config.""" + self._do_https_request_via_proxy( + proxy_config=parse_proxy_config({"https_proxy": "proxy.com"}), + expect_proxy_ssl=False, + expected_auth_credentials=None, + ) + + @patch.dict( + os.environ, + {"https_proxy": "unused.com", "no_proxy": "unused.com"}, + ) + def test_given_no_proxy_config_overrides_environment_config(self) -> None: + """Tests that the given `no_proxy_hosts` in file config overrides the `no_proxy` environment config.""" + agent = ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config( + {"http_proxy": "proxy.com:8888", "no_proxy_hosts": ["test.com"]} + ), + ) + self._test_request_direct_connection(agent, b"http", b"test.com", b"") + @patch.dict( os.environ, {"http_proxy": "bob:pinkponies@proxy.com:8888", "no_proxy": "unused.com"}, @@ -445,7 +525,9 @@ class MatrixFederationAgentTests(TestCase): Tests that authenticated requests can be made through a proxy. """ self._do_http_request_via_proxy( - expect_proxy_ssl=False, expected_auth_credentials=b"bob:pinkponies" + proxy_config=parse_proxy_config({}), + expect_proxy_ssl=False, + expected_auth_credentials=b"bob:pinkponies", ) @patch.dict( @@ -453,7 +535,9 @@ class MatrixFederationAgentTests(TestCase): ) def test_http_request_via_https_proxy(self) -> None: self._do_http_request_via_proxy( - expect_proxy_ssl=True, expected_auth_credentials=None + proxy_config=parse_proxy_config({}), + expect_proxy_ssl=True, + expected_auth_credentials=None, ) @patch.dict( @@ -465,14 +549,18 @@ class MatrixFederationAgentTests(TestCase): ) def test_http_request_via_https_proxy_with_auth(self) -> None: self._do_http_request_via_proxy( - expect_proxy_ssl=True, expected_auth_credentials=b"bob:pinkponies" + proxy_config=parse_proxy_config({}), + expect_proxy_ssl=True, + expected_auth_credentials=b"bob:pinkponies", ) @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"}) def test_https_request_via_proxy(self) -> None: """Tests that TLS-encrypted requests can be made through a proxy""" self._do_https_request_via_proxy( - expect_proxy_ssl=False, expected_auth_credentials=None + proxy_config=parse_proxy_config({}), + expect_proxy_ssl=False, + expected_auth_credentials=None, ) @patch.dict( @@ -482,7 +570,9 @@ class MatrixFederationAgentTests(TestCase): def test_https_request_via_proxy_with_auth(self) -> None: """Tests that authenticated, TLS-encrypted requests can be made through a proxy""" self._do_https_request_via_proxy( - expect_proxy_ssl=False, expected_auth_credentials=b"bob:pinkponies" + proxy_config=parse_proxy_config({}), + expect_proxy_ssl=False, + expected_auth_credentials=b"bob:pinkponies", ) @patch.dict( @@ -491,7 +581,9 @@ class MatrixFederationAgentTests(TestCase): def test_https_request_via_https_proxy(self) -> None: """Tests that TLS-encrypted requests can be made through a proxy""" self._do_https_request_via_proxy( - expect_proxy_ssl=True, expected_auth_credentials=None + proxy_config=parse_proxy_config({}), + expect_proxy_ssl=True, + expected_auth_credentials=None, ) @patch.dict( @@ -501,11 +593,14 @@ class MatrixFederationAgentTests(TestCase): def test_https_request_via_https_proxy_with_auth(self) -> None: """Tests that authenticated, TLS-encrypted requests can be made through a proxy""" self._do_https_request_via_proxy( - expect_proxy_ssl=True, expected_auth_credentials=b"bob:pinkponies" + proxy_config=parse_proxy_config({}), + expect_proxy_ssl=True, + expected_auth_credentials=b"bob:pinkponies", ) def _do_http_request_via_proxy( self, + proxy_config: ProxyConfig, expect_proxy_ssl: bool = False, expected_auth_credentials: Optional[bytes] = None, ) -> None: @@ -517,10 +612,15 @@ class MatrixFederationAgentTests(TestCase): """ if expect_proxy_ssl: agent = ProxyAgent( - self.reactor, use_proxy=True, contextFactory=get_test_https_policy() + reactor=self.reactor, + proxy_config=proxy_config, + contextFactory=get_test_https_policy(), ) else: - agent = ProxyAgent(self.reactor, use_proxy=True) + agent = ProxyAgent( + reactor=self.reactor, + proxy_config=proxy_config, + ) self.reactor.lookups["proxy.com"] = "1.2.3.5" d = agent.request(b"GET", b"http://test.com") @@ -580,6 +680,7 @@ class MatrixFederationAgentTests(TestCase): def _do_https_request_via_proxy( self, + proxy_config: ProxyConfig, expect_proxy_ssl: bool = False, expected_auth_credentials: Optional[bytes] = None, ) -> None: @@ -590,9 +691,9 @@ class MatrixFederationAgentTests(TestCase): expected_auth_credentials: credentials to authenticate at proxy """ agent = ProxyAgent( - self.reactor, + reactor=self.reactor, contextFactory=get_test_https_policy(), - use_proxy=True, + proxy_config=proxy_config, ) self.reactor.lookups["proxy.com"] = "1.2.3.5" @@ -713,11 +814,11 @@ class MatrixFederationAgentTests(TestCase): def test_http_request_via_proxy_with_blocklist(self) -> None: # The blocklist includes the configured proxy IP. agent = ProxyAgent( - BlocklistingReactorWrapper( + reactor=BlocklistingReactorWrapper( self.reactor, ip_allowlist=None, ip_blocklist=IPSet(["1.0.0.0/8"]) ), - self.reactor, - use_proxy=True, + proxy_reactor=self.reactor, + proxy_config=parse_proxy_config({}), ) self.reactor.lookups["proxy.com"] = "1.2.3.5" @@ -759,12 +860,12 @@ class MatrixFederationAgentTests(TestCase): def test_https_request_via_uppercase_proxy_with_blocklist(self) -> None: # The blocklist includes the configured proxy IP. agent = ProxyAgent( - BlocklistingReactorWrapper( + reactor=BlocklistingReactorWrapper( self.reactor, ip_allowlist=None, ip_blocklist=IPSet(["1.0.0.0/8"]) ), - self.reactor, + proxy_reactor=self.reactor, contextFactory=get_test_https_policy(), - use_proxy=True, + proxy_config=parse_proxy_config({}), ) self.reactor.lookups["proxy.com"] = "1.2.3.5" @@ -852,28 +953,40 @@ class MatrixFederationAgentTests(TestCase): @patch.dict(os.environ, {"http_proxy": "proxy.com:8888"}) def test_proxy_with_no_scheme(self) -> None: - http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True) + http_proxy_agent = ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config({}), + ) proxy_ep = checked_cast(HostnameEndpoint, http_proxy_agent.http_proxy_endpoint) - self.assertEqual(proxy_ep._hostStr, "proxy.com") + self.assertEqual(proxy_ep._hostText, "proxy.com") self.assertEqual(proxy_ep._port, 8888) @patch.dict(os.environ, {"http_proxy": "socks://proxy.com:8888"}) def test_proxy_with_unsupported_scheme(self) -> None: with self.assertRaises(ValueError): - ProxyAgent(self.reactor, use_proxy=True) + ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config({}), + ) @patch.dict(os.environ, {"http_proxy": "http://proxy.com:8888"}) def test_proxy_with_http_scheme(self) -> None: - http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True) + http_proxy_agent = ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config({}), + ) proxy_ep = checked_cast(HostnameEndpoint, http_proxy_agent.http_proxy_endpoint) - self.assertEqual(proxy_ep._hostStr, "proxy.com") + self.assertEqual(proxy_ep._hostText, "proxy.com") self.assertEqual(proxy_ep._port, 8888) @patch.dict(os.environ, {"http_proxy": "https://proxy.com:8888"}) def test_proxy_with_https_scheme(self) -> None: - https_proxy_agent = ProxyAgent(self.reactor, use_proxy=True) + https_proxy_agent = ProxyAgent( + reactor=self.reactor, + proxy_config=parse_proxy_config({}), + ) proxy_ep = checked_cast(_WrapperEndpoint, https_proxy_agent.http_proxy_endpoint) - self.assertEqual(proxy_ep._wrappedEndpoint._hostStr, "proxy.com") + self.assertEqual(proxy_ep._wrappedEndpoint._hostText, "proxy.com") self.assertEqual(proxy_ep._wrappedEndpoint._port, 8888) @@ -893,4 +1006,4 @@ def _get_test_protocol_factory() -> IProtocolFactory: def _log_request(request: str) -> None: """Implements Factory.log, which is expected by Request.finish""" - logger.info(f"Completed request {request}") + logger.info("Completed request %s", request) diff --git a/tests/http/test_simple_client.py b/tests/http/test_simple_client.py index b7806fa947..c5ead59988 100644 --- a/tests/http/test_simple_client.py +++ b/tests/http/test_simple_client.py @@ -24,7 +24,7 @@ from netaddr import IPSet from twisted.internet import defer from twisted.internet.error import DNSLookupError -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.http import RequestTimedOutError from synapse.http.client import SimpleHttpClient diff --git a/tests/http/test_site.py b/tests/http/test_site.py index bfa26a329c..2eca4587e7 100644 --- a/tests/http/test_site.py +++ b/tests/http/test_site.py @@ -20,7 +20,7 @@ # from twisted.internet.address import IPv6Address -from twisted.test.proto_helpers import MemoryReactor, StringTransport +from twisted.internet.testing import MemoryReactor, StringTransport from synapse.app.homeserver import SynapseHomeServer from synapse.server import HomeServer @@ -90,3 +90,56 @@ class SynapseRequestTestCase(HomeserverTestCase): # default max upload size is 50M, so it should drop on the next buffer after # that. self.assertEqual(sent, 50 * 1024 * 1024 + 1024) + + def test_content_type_multipart(self) -> None: + """HTTP POST requests with `content-type: multipart/form-data` should be rejected""" + self.hs.start_listening() + + # find the HTTP server which is configured to listen on port 0 + (port, factory, _backlog, interface) = self.reactor.tcpServers[0] + self.assertEqual(interface, "::") + self.assertEqual(port, 0) + + # as a control case, first send a regular request. + + # complete the connection and wire it up to a fake transport + client_address = IPv6Address("TCP", "::1", 2345) + protocol = factory.buildProtocol(client_address) + transport = StringTransport() + protocol.makeConnection(transport) + + protocol.dataReceived( + b"POST / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + b"0\r\n" + b"\r\n" + ) + + while not transport.disconnecting: + self.reactor.advance(1) + + # we should get a 404 + self.assertRegex(transport.value().decode(), r"^HTTP/1\.1 404 ") + + # now send request with content-type header + protocol = factory.buildProtocol(client_address) + transport = StringTransport() + protocol.makeConnection(transport) + + protocol.dataReceived( + b"POST / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Transfer-Encoding: chunked\r\n" + b"Content-Type: multipart/form-data\r\n" + b"\r\n" + b"0\r\n" + b"\r\n" + ) + + while not transport.disconnecting: + self.reactor.advance(1) + + # we should get a 415 + self.assertRegex(transport.value().decode(), r"^HTTP/1\.1 415 ") diff --git a/tests/logging/test_loggers.py b/tests/logging/test_loggers.py new file mode 100644 index 0000000000..9a9bf14376 --- /dev/null +++ b/tests/logging/test_loggers.py @@ -0,0 +1,127 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +# +import logging + +from synapse.logging.loggers import ExplicitlyConfiguredLogger + +from tests.unittest import TestCase + + +class ExplicitlyConfiguredLoggerTestCase(TestCase): + def _create_explicitly_configured_logger(self) -> logging.Logger: + original_logger_class = logging.getLoggerClass() + logging.setLoggerClass(ExplicitlyConfiguredLogger) + logger = logging.getLogger("test") + # Restore the original logger class + logging.setLoggerClass(original_logger_class) + + return logger + + def test_no_logs_when_not_set(self) -> None: + """ + Test to make sure that nothing is logged when the logger is *not* explicitly + configured. + """ + root_logger = logging.getLogger() + root_logger.setLevel(logging.DEBUG) + + logger = self._create_explicitly_configured_logger() + + with self.assertLogs(logger=logger, level=logging.NOTSET) as cm: + # XXX: We have to set this again because of a Python bug: + # https://github.com/python/cpython/issues/136958 (feel free to remove once + # that is resolved and we update to a newer Python version that includes the + # fix) + logger.setLevel(logging.NOTSET) + + logger.debug("debug message") + logger.info("info message") + logger.warning("warning message") + logger.error("error message") + + # Nothing should be logged since the logger is *not* explicitly configured + # + # FIXME: Remove this whole block once we update to Python 3.10 or later and + # have access to `assertNoLogs` (replace `assertLogs` with `assertNoLogs`) + self.assertIncludes( + set(cm.output), + set(), + exact=True, + ) + # Stub log message to avoid `assertLogs` failing since it expects at least + # one log message to be logged. + logger.setLevel(logging.INFO) + logger.info("stub message so `assertLogs` doesn't fail") + + def test_logs_when_explicitly_configured(self) -> None: + """ + Test to make sure that logs are emitted when the logger is explicitly configured. + """ + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + + logger = self._create_explicitly_configured_logger() + + with self.assertLogs(logger=logger, level=logging.DEBUG) as cm: + logger.debug("debug message") + logger.info("info message") + logger.warning("warning message") + logger.error("error message") + + self.assertIncludes( + set(cm.output), + { + "DEBUG:test:debug message", + "INFO:test:info message", + "WARNING:test:warning message", + "ERROR:test:error message", + }, + exact=True, + ) + + def test_is_enabled_for_not_set(self) -> None: + """ + Test to make sure `logger.isEnabledFor(...)` returns False when the logger is + not explicitly configured. + """ + + logger = self._create_explicitly_configured_logger() + + # Unset the logger (not configured) + logger.setLevel(logging.NOTSET) + + # The logger shouldn't be enabled for any level + self.assertFalse(logger.isEnabledFor(logging.DEBUG)) + self.assertFalse(logger.isEnabledFor(logging.INFO)) + self.assertFalse(logger.isEnabledFor(logging.WARNING)) + self.assertFalse(logger.isEnabledFor(logging.ERROR)) + + def test_is_enabled_for_info(self) -> None: + """ + Test to make sure `logger.isEnabledFor(...)` returns True any levels above the + explicitly configured level. + """ + + logger = self._create_explicitly_configured_logger() + + # Explicitly configure the logger to `INFO` level + logger.setLevel(logging.INFO) + + # The logger should be enabled for INFO and above once explicitly configured + self.assertFalse(logger.isEnabledFor(logging.DEBUG)) + self.assertTrue(logger.isEnabledFor(logging.INFO)) + self.assertTrue(logger.isEnabledFor(logging.WARNING)) + self.assertTrue(logger.isEnabledFor(logging.ERROR)) diff --git a/tests/logging/test_opentracing.py b/tests/logging/test_opentracing.py index c7ef2bd7a4..5fe57d100e 100644 --- a/tests/logging/test_opentracing.py +++ b/tests/logging/test_opentracing.py @@ -19,10 +19,10 @@ # # -from typing import Awaitable, cast +from typing import Awaitable, Dict, cast from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactorClock +from twisted.internet.testing import MemoryReactorClock from synapse.logging.context import ( LoggingContext, @@ -38,9 +38,11 @@ from synapse.logging.opentracing import ( from synapse.util import Clock try: - from synapse.logging.scopecontextmanager import LogContextScopeManager + import opentracing + from opentracing.scope_managers.contextvars import ContextVarsScopeManager except ImportError: - LogContextScopeManager = None # type: ignore + opentracing = None # type: ignore + ContextVarsScopeManager = None # type: ignore try: import jaeger_client @@ -54,9 +56,10 @@ from tests.unittest import TestCase logger = logging.getLogger(__name__) -class LogContextScopeManagerTestCase(TestCase): +class TracingScopeTestCase(TestCase): """ - Test logging contexts and active opentracing spans. + Test that our tracing machinery works well in a variety of situations (especially + with Twisted's runtime and deferreds). There's casts throughout this from generic opentracing objects (e.g. opentracing.Span) to the ones specific to Jaeger since they have additional @@ -64,7 +67,7 @@ class LogContextScopeManagerTestCase(TestCase): opentracing backend is Jaeger. """ - if LogContextScopeManager is None: + if opentracing is None: skip = "Requires opentracing" # type: ignore[unreachable] if jaeger_client is None: skip = "Requires jaeger_client" # type: ignore[unreachable] @@ -74,7 +77,7 @@ class LogContextScopeManagerTestCase(TestCase): # global variables that power opentracing. We create our own tracer instance # and test with it. - scope_manager = LogContextScopeManager() + scope_manager = ContextVarsScopeManager() config = jaeger_client.config.Config( config={}, service_name="test", scope_manager=scope_manager ) @@ -208,6 +211,135 @@ class LogContextScopeManagerTestCase(TestCase): [scopes[1].span, scopes[2].span, scopes[0].span], ) + def test_run_in_background_active_scope_still_available(self) -> None: + """ + Test that tasks running via `run_in_background` still have access to the + active tracing scope. + + This is a regression test for a previous Synapse issue where the tracing scope + would `__exit__` and close before the `run_in_background` task completed and our + own previous custom `_LogContextScope.close(...)` would clear + `LoggingContext.scope` preventing further tracing spans from having the correct + parent. + """ + reactor = MemoryReactorClock() + clock = Clock(reactor) + + scope_map: Dict[str, opentracing.Scope] = {} + + async def async_task() -> None: + root_scope = scope_map["root"] + root_context = cast(jaeger_client.SpanContext, root_scope.span.context) + + self.assertEqual( + self._tracer.active_span, + root_scope.span, + "expected to inherit the root tracing scope from where this was run", + ) + + # Return control back to the reactor thread and wait an arbitrary amount + await clock.sleep(4) + + # This is a key part of what we're testing! In a previous version of + # Synapse, we would lose the active span at this point. + self.assertEqual( + self._tracer.active_span, + root_scope.span, + "expected to still have a root tracing scope/span active", + ) + + # For complete-ness sake, let's also trace more sub-tasks here and assert + # they have the correct span parents as well (root) + + # Start tracing some other sub-task. + # + # This is a key part of what we're testing! In a previous version of + # Synapse, it would have the incorrect span parents. + scope = start_active_span( + "task1", + tracer=self._tracer, + ) + scope_map["task1"] = scope + + # Ensure the span parent is pointing to the root scope + context = cast(jaeger_client.SpanContext, scope.span.context) + self.assertEqual( + context.parent_id, + root_context.span_id, + "expected task1 parent to be the root span", + ) + + # Ensure that the active span is our new sub-task now + self.assertEqual(self._tracer.active_span, scope.span) + # Return control back to the reactor thread and wait an arbitrary amount + await clock.sleep(4) + # We should still see the active span as the scope wasn't closed yet + self.assertEqual(self._tracer.active_span, scope.span) + scope.close() + + async def root() -> None: + with start_active_span( + "root span", + tracer=self._tracer, + # We will close this off later. We're basically just mimicking the same + # pattern for how we handle requests. We pass the span off to the + # request for it to finish. + finish_on_close=False, + ) as root_scope: + scope_map["root"] = root_scope + self.assertEqual(self._tracer.active_span, root_scope.span) + + # Fire-and-forget a task + # + # XXX: The root scope context manager will `__exit__` before this task + # completes. + run_in_background(async_task) + + # Because we used `run_in_background`, the active span should still be + # the root. + self.assertEqual(self._tracer.active_span, root_scope.span) + + # We shouldn't see any active spans outside of the scope + self.assertIsNone(self._tracer.active_span) + + with LoggingContext("root context"): + # Start the test off + d_root = defer.ensureDeferred(root()) + + # Let the tasks complete + reactor.pump((2,) * 8) + self.successResultOf(d_root) + + # After we see all of the tasks are done (like a request when it + # `_finished_processing`), let's finish our root span + scope_map["root"].span.finish() + + # Sanity check again: We shouldn't see any active spans leftover in this + # this context. + self.assertIsNone(self._tracer.active_span) + + # The spans should be reported in order of their finishing: task 1, task 2, + # root. + # + # We use `assertIncludes` just as an easier way to see if items are missing or + # added. We assert the order just below + self.assertIncludes( + set(self._reporter.get_spans()), + { + scope_map["task1"].span, + scope_map["root"].span, + }, + exact=True, + ) + # This is where we actually assert the correct order + self.assertEqual( + self._reporter.get_spans(), + [ + scope_map["task1"].span, + scope_map["root"].span, + ], + ) + def test_trace_decorator_sync(self) -> None: """ Test whether we can use `@trace_with_opname` (`@trace`) and `@tag_args` diff --git a/tests/logging/test_remote_handler.py b/tests/logging/test_remote_handler.py index f5412ac6e2..e0fd12ccf7 100644 --- a/tests/logging/test_remote_handler.py +++ b/tests/logging/test_remote_handler.py @@ -21,7 +21,7 @@ from typing import Tuple from twisted.internet.protocol import Protocol -from twisted.test.proto_helpers import AccumulatingProtocol, MemoryReactorClock +from twisted.internet.testing import AccumulatingProtocol, MemoryReactorClock from synapse.logging import RemoteHandler @@ -86,11 +86,11 @@ class RemoteHandlerTestCase(LoggerCleanupMixin, TestCase): # Send some debug messages for i in range(3): - logger.debug("debug %s" % (i,)) + logger.debug("debug %s", i) # Send a bunch of useful messages for i in range(7): - logger.info("info %s" % (i,)) + logger.info("info %s", i) # The last debug message pushes it past the maximum buffer logger.debug("too much debug") @@ -116,15 +116,15 @@ class RemoteHandlerTestCase(LoggerCleanupMixin, TestCase): # Send some debug messages for i in range(3): - logger.debug("debug %s" % (i,)) + logger.debug("debug %s", i) # Send a bunch of useful messages for i in range(10): - logger.warning("warn %s" % (i,)) + logger.warning("warn %s", i) # Send a bunch of info messages for i in range(3): - logger.info("info %s" % (i,)) + logger.info("info %s", i) # The last debug message pushes it past the maximum buffer logger.debug("too much debug") @@ -152,7 +152,7 @@ class RemoteHandlerTestCase(LoggerCleanupMixin, TestCase): # Send a bunch of useful messages for i in range(20): - logger.warning("warn %s" % (i,)) + logger.warning("warn %s", i) # Allow the reconnection client, server = connect_logging_client(self.reactor, 0) diff --git a/tests/logging/test_terse_json.py b/tests/logging/test_terse_json.py index 33b94cf9fa..60de8d786f 100644 --- a/tests/logging/test_terse_json.py +++ b/tests/logging/test_terse_json.py @@ -160,12 +160,20 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): logger = self.get_logger(handler) # A full request isn't needed here. - site = Mock(spec=["site_tag", "server_version_string", "getResourceFor"]) + site = Mock( + spec=[ + "site_tag", + "server_version_string", + "getResourceFor", + "_parsePOSTFormSubmission", + ] + ) site.site_tag = "test-site" site.server_version_string = "Server v1" site.reactor = Mock() + request = SynapseRequest( - cast(HTTPChannel, FakeChannel(site, self.reactor)), site + cast(HTTPChannel, FakeChannel(site, self.reactor)), site, "test_server" ) # Call requestReceived to finish instantiating the object. request.content = BytesIO() diff --git a/tests/media/test_media_retention.py b/tests/media/test_media_retention.py index 417d17ebd2..6e01b9aecb 100644 --- a/tests/media/test_media_retention.py +++ b/tests/media/test_media_retention.py @@ -24,13 +24,16 @@ from typing import Iterable, Optional from matrix_common.types.mxc_uri import MXCUri -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, register, room from synapse.server import HomeServer from synapse.types import UserID from synapse.util import Clock +from synapse.util.stringutils import ( + random_string, +) from tests import unittest from tests.unittest import override_config @@ -65,7 +68,6 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase): # quarantined media) into both the local store and the remote cache, plus # one additional local media that is marked as protected from quarantine. media_repository = hs.get_media_repository() - test_media_content = b"example string" def _create_media_and_set_attributes( last_accessed_ms: Optional[int], @@ -73,12 +75,14 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase): is_protected: Optional[bool] = False, ) -> MXCUri: # "Upload" some media to the local media store + # If the meda + random_content = bytes(random_string(24), "utf-8") mxc_uri: MXCUri = self.get_success( - media_repository.create_content( + media_repository.create_or_update_content( media_type="text/plain", upload_name=None, - content=io.BytesIO(test_media_content), - content_length=len(test_media_content), + content=io.BytesIO(random_content), + content_length=len(random_content), auth_user=UserID.from_string(test_user_id), ) ) @@ -129,6 +133,7 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase): time_now_ms=clock.time_msec(), upload_name="testfile.txt", filesystem_id="abcdefg12345", + sha256=random_string(24), ) ) diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index f4fbc0544a..bf334c0371 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -23,19 +23,18 @@ import shutil import tempfile from binascii import unhexlify from io import BytesIO -from typing import Any, BinaryIO, ClassVar, Dict, List, Optional, Tuple, Union +from typing import Any, BinaryIO, ClassVar, Dict, List, Literal, Optional, Tuple, Union from unittest.mock import MagicMock, Mock, patch from urllib import parse import attr from parameterized import parameterized, parameterized_class from PIL import Image as Image -from typing_extensions import Literal from twisted.internet import defer from twisted.internet.defer import Deferred +from twisted.internet.testing import MemoryReactor from twisted.python.failure import Failure -from twisted.test.proto_helpers import MemoryReactor from twisted.web.http_headers import Headers from twisted.web.iweb import UNKNOWN_LENGTH, IResponse from twisted.web.resource import Resource @@ -43,6 +42,7 @@ from twisted.web.resource import Resource from synapse.api.errors import Codes, HttpResponseException from synapse.api.ratelimiting import Ratelimiter from synapse.events import EventBase +from synapse.http.client import ByteWriteable from synapse.http.types import QueryParams from synapse.logging.context import make_deferred_yieldable from synapse.media._base import FileInfo, ThumbnailInfo @@ -60,7 +60,7 @@ from synapse.util import Clock from tests import unittest from tests.server import FakeChannel -from tests.test_utils import SMALL_CMYK_JPEG, SMALL_PNG +from tests.test_utils import SMALL_CMYK_JPEG, SMALL_PNG, SMALL_PNG_SHA256 from tests.unittest import override_config from tests.utils import default_config @@ -250,9 +250,7 @@ small_cmyk_jpeg = TestImage( ) small_lossless_webp = TestImage( - unhexlify( - b"524946461a000000574542505650384c0d0000002f0000001007" b"1011118888fe0700" - ), + unhexlify(b"524946461a000000574542505650384c0d0000002f00000010071011118888fe0700"), b"image/webp", b".webp", ) @@ -1258,3 +1256,146 @@ class RemoteDownloadLimiterTestCase(unittest.HomeserverTestCase): ) assert channel.code == 502 assert channel.json_body["errcode"] == "M_TOO_LARGE" + + +def read_body( + response: IResponse, stream: ByteWriteable, max_size: Optional[int] +) -> Deferred: + d: Deferred = defer.Deferred() + stream.write(SMALL_PNG) + d.callback(len(SMALL_PNG)) + return d + + +class MediaHashesTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + media.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + self.store = hs.get_datastores().main + self.client = hs.get_federation_http_client() + + def create_resource_dict(self) -> Dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + def test_ensure_correct_sha256(self) -> None: + """Check that the hash does not change""" + media = self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=200) + mxc = media.get("content_uri") + assert mxc + store_media = self.get_success(self.store.get_local_media(mxc[11:])) + assert store_media + self.assertEqual( + store_media.sha256, + SMALL_PNG_SHA256, + ) + + def test_ensure_multiple_correct_sha256(self) -> None: + """Check that two media items have the same hash.""" + media_a = self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=200) + mxc_a = media_a.get("content_uri") + assert mxc_a + store_media_a = self.get_success(self.store.get_local_media(mxc_a[11:])) + assert store_media_a + + media_b = self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=200) + mxc_b = media_b.get("content_uri") + assert mxc_b + store_media_b = self.get_success(self.store.get_local_media(mxc_b[11:])) + assert store_media_b + + self.assertNotEqual( + store_media_a.media_id, + store_media_b.media_id, + ) + self.assertEqual( + store_media_a.sha256, + store_media_b.sha256, + ) + + @override_config( + { + "enable_authenticated_media": False, + } + ) + # mock actually reading file body + @patch( + "synapse.http.matrixfederationclient.read_body_with_max_size", + read_body, + ) + def test_ensure_correct_sha256_federated(self) -> None: + """Check that federated media have the same hash.""" + + # Mock getting a file over federation + async def _send_request(*args: Any, **kwargs: Any) -> IResponse: + resp = MagicMock(spec=IResponse) + resp.code = 200 + resp.length = 500 + resp.headers = Headers({"Content-Type": ["application/octet-stream"]}) + resp.phrase = b"OK" + return resp + + self.client._send_request = _send_request # type: ignore + + # first request should go through + channel = self.make_request( + "GET", + "/_matrix/media/v3/download/remote.org/abc", + shorthand=False, + access_token=self.tok, + ) + assert channel.code == 200 + store_media = self.get_success( + self.store.get_cached_remote_media("remote.org", "abc") + ) + assert store_media + self.assertEqual( + store_media.sha256, + SMALL_PNG_SHA256, + ) + + +class MediaRepoSizeModuleCallbackTestCase(unittest.HomeserverTestCase): + servlets = [ + login.register_servlets, + admin.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + self.mock_result = True # Allow all uploads by default + + hs.get_module_api().register_media_repository_callbacks( + is_user_allowed_to_upload_media_of_size=self.is_user_allowed_to_upload_media_of_size, + ) + + def create_resource_dict(self) -> Dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + async def is_user_allowed_to_upload_media_of_size( + self, user_id: str, size: int + ) -> bool: + self.last_user_id = user_id + self.last_size = size + return self.mock_result + + def test_upload_allowed(self) -> None: + self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=200) + assert self.last_user_id == self.user + assert self.last_size == len(SMALL_PNG) + + def test_upload_not_allowed(self) -> None: + self.mock_result = False + self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=413) + assert self.last_user_id == self.user + assert self.last_size == len(SMALL_PNG) diff --git a/tests/media/test_oembed.py b/tests/media/test_oembed.py index b8265ff9ca..afae7e048c 100644 --- a/tests/media/test_oembed.py +++ b/tests/media/test_oembed.py @@ -24,7 +24,7 @@ from typing import Any from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.media.oembed import OEmbedProvider, OEmbedResult from synapse.server import HomeServer diff --git a/tests/media/test_url_previewer.py b/tests/media/test_url_previewer.py index 0ae414d408..bd7190e3e9 100644 --- a/tests/media/test_url_previewer.py +++ b/tests/media/test_url_previewer.py @@ -20,7 +20,7 @@ # import os -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer from synapse.util import Clock diff --git a/tests/metrics/test_background_process_metrics.py b/tests/metrics/test_background_process_metrics.py index f0f6cb2912..1f47601b95 100644 --- a/tests/metrics/test_background_process_metrics.py +++ b/tests/metrics/test_background_process_metrics.py @@ -14,6 +14,8 @@ class TestBackgroundProcessMetrics(StdlibTestCase): mock_logging_context = Mock(spec=LoggingContext) mock_logging_context.get_resource_usage.return_value = usage - process = _BackgroundProcess("test process", mock_logging_context) + process = _BackgroundProcess( + desc="test process", server_name="test_server", ctx=mock_logging_context + ) # Should not raise process.update_metrics() diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index 80f24814e8..832e991730 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -18,16 +18,18 @@ # [This file includes modifications made by New Vector Limited] # # -from importlib import metadata -from typing import Dict, Tuple -from unittest.mock import patch +from typing import Dict, NoReturn, Protocol, Tuple -from pkg_resources import parse_version from prometheus_client.core import Sample -from typing_extensions import Protocol -from synapse.app._base import _set_prometheus_client_use_created_metrics -from synapse.metrics import REGISTRY, InFlightGauge, generate_latest +from synapse.metrics import ( + REGISTRY, + SERVER_NAME_LABEL, + InFlightGauge, + LaterGauge, + all_later_gauges_to_clean_up_on_shutdown, + generate_latest, +) from synapse.util.caches.deferred_cache import DeferredCache from tests import unittest @@ -64,7 +66,8 @@ class TestMauLimit(unittest.TestCase): foo: int bar: int - gauge: InFlightGauge[MetricEntry] = InFlightGauge( + # This is a test and does not matter if it uses `SERVER_NAME_LABEL`. + gauge: InFlightGauge[MetricEntry] = InFlightGauge( # type: ignore[missing-server-name-label] "test1", "", labels=["test_label"], sub_metrics=["foo", "bar"] ) @@ -160,55 +163,242 @@ class CacheMetricsTests(unittest.HomeserverTestCase): Caches produce metrics reflecting their state when scraped. """ CACHE_NAME = "cache_metrics_test_fgjkbdfg" - cache: DeferredCache[str, str] = DeferredCache(CACHE_NAME, max_entries=777) + cache: DeferredCache[str, str] = DeferredCache( + name=CACHE_NAME, server_name=self.hs.hostname, max_entries=777 + ) - items = { - x.split(b"{")[0].decode("ascii"): x.split(b" ")[1].decode("ascii") - for x in filter( - lambda x: b"cache_metrics_test_fgjkbdfg" in x, - generate_latest(REGISTRY).split(b"\n"), - ) - } + metrics_map = get_latest_metrics() - self.assertEqual(items["synapse_util_caches_cache_size"], "0.0") - self.assertEqual(items["synapse_util_caches_cache_max_size"], "777.0") + cache_size_metric = f'synapse_util_caches_cache_size{{name="{CACHE_NAME}",server_name="{self.hs.hostname}"}}' + cache_max_size_metric = f'synapse_util_caches_cache_max_size{{name="{CACHE_NAME}",server_name="{self.hs.hostname}"}}' + + cache_size_metric_value = metrics_map.get(cache_size_metric) + self.assertIsNotNone( + cache_size_metric_value, + f"Missing metric {cache_size_metric} in cache metrics {metrics_map}", + ) + cache_max_size_metric_value = metrics_map.get(cache_max_size_metric) + self.assertIsNotNone( + cache_max_size_metric_value, + f"Missing metric {cache_max_size_metric} in cache metrics {metrics_map}", + ) + + self.assertEqual(cache_size_metric_value, "0.0") + self.assertEqual(cache_max_size_metric_value, "777.0") cache.prefill("1", "hi") - items = { - x.split(b"{")[0].decode("ascii"): x.split(b" ")[1].decode("ascii") - for x in filter( - lambda x: b"cache_metrics_test_fgjkbdfg" in x, - generate_latest(REGISTRY).split(b"\n"), - ) - } + metrics_map = get_latest_metrics() - self.assertEqual(items["synapse_util_caches_cache_size"], "1.0") - self.assertEqual(items["synapse_util_caches_cache_max_size"], "777.0") + cache_size_metric_value = metrics_map.get(cache_size_metric) + self.assertIsNotNone( + cache_size_metric_value, + f"Missing metric {cache_size_metric} in cache metrics {metrics_map}", + ) + cache_max_size_metric_value = metrics_map.get(cache_max_size_metric) + self.assertIsNotNone( + cache_max_size_metric_value, + f"Missing metric {cache_max_size_metric} in cache metrics {metrics_map}", + ) + self.assertEqual(cache_size_metric_value, "1.0") + self.assertEqual(cache_max_size_metric_value, "777.0") -class PrometheusMetricsHackTestCase(unittest.HomeserverTestCase): - if parse_version(metadata.version("prometheus_client")) < parse_version("0.14.0"): - skip = "prometheus-client too old" - - def test_created_metrics_disabled(self) -> None: + def test_cache_metric_multiple_servers(self) -> None: """ - Tests that a brittle hack, to disable `_created` metrics, works. - This involves poking at the internals of prometheus-client. - It's not the end of the world if this doesn't work. - - This test gives us a way to notice if prometheus-client changes - their internals. + Test that cache metrics are reported correctly across multiple servers. We will + have an metrics entry for each homeserver that is labeled with the `server_name` + label. """ - import prometheus_client.metrics + CACHE_NAME = "cache_metric_multiple_servers_test" + cache1: DeferredCache[str, str] = DeferredCache( + name=CACHE_NAME, server_name="hs1", max_entries=777 + ) + cache2: DeferredCache[str, str] = DeferredCache( + name=CACHE_NAME, server_name="hs2", max_entries=777 + ) - PRIVATE_FLAG_NAME = "_use_created" + metrics_map = get_latest_metrics() - # By default, the pesky `_created` metrics are enabled. - # Check this assumption is still valid. - self.assertTrue(getattr(prometheus_client.metrics, PRIVATE_FLAG_NAME)) + hs1_cache_size_metric = ( + f'synapse_util_caches_cache_size{{name="{CACHE_NAME}",server_name="hs1"}}' + ) + hs2_cache_size_metric = ( + f'synapse_util_caches_cache_size{{name="{CACHE_NAME}",server_name="hs2"}}' + ) + hs1_cache_max_size_metric = f'synapse_util_caches_cache_max_size{{name="{CACHE_NAME}",server_name="hs1"}}' + hs2_cache_max_size_metric = f'synapse_util_caches_cache_max_size{{name="{CACHE_NAME}",server_name="hs2"}}' - with patch("prometheus_client.metrics") as mock: - setattr(mock, PRIVATE_FLAG_NAME, True) - _set_prometheus_client_use_created_metrics(False) - self.assertFalse(getattr(mock, PRIVATE_FLAG_NAME, False)) + # Find the metrics for the caches from both homeservers + hs1_cache_size_metric_value = metrics_map.get(hs1_cache_size_metric) + self.assertIsNotNone( + hs1_cache_size_metric_value, + f"Missing metric {hs1_cache_size_metric} in cache metrics {metrics_map}", + ) + hs2_cache_size_metric_value = metrics_map.get(hs2_cache_size_metric) + self.assertIsNotNone( + hs2_cache_size_metric_value, + f"Missing metric {hs2_cache_size_metric} in cache metrics {metrics_map}", + ) + hs1_cache_max_size_metric_value = metrics_map.get(hs1_cache_max_size_metric) + self.assertIsNotNone( + hs1_cache_max_size_metric_value, + f"Missing metric {hs1_cache_max_size_metric} in cache metrics {metrics_map}", + ) + hs2_cache_max_size_metric_value = metrics_map.get(hs2_cache_max_size_metric) + self.assertIsNotNone( + hs2_cache_max_size_metric_value, + f"Missing metric {hs2_cache_max_size_metric} in cache metrics {metrics_map}", + ) + + # Sanity check the metric values + self.assertEqual(hs1_cache_size_metric_value, "0.0") + self.assertEqual(hs2_cache_size_metric_value, "0.0") + self.assertEqual(hs1_cache_max_size_metric_value, "777.0") + self.assertEqual(hs2_cache_max_size_metric_value, "777.0") + + # Add something to both caches to change the numbers + cache1.prefill("1", "hi") + cache2.prefill("2", "ho") + + metrics_map = get_latest_metrics() + + # Find the metrics for the caches from both homeservers + hs1_cache_size_metric_value = metrics_map.get(hs1_cache_size_metric) + self.assertIsNotNone( + hs1_cache_size_metric_value, + f"Missing metric {hs1_cache_size_metric} in cache metrics {metrics_map}", + ) + hs2_cache_size_metric_value = metrics_map.get(hs2_cache_size_metric) + self.assertIsNotNone( + hs2_cache_size_metric_value, + f"Missing metric {hs2_cache_size_metric} in cache metrics {metrics_map}", + ) + hs1_cache_max_size_metric_value = metrics_map.get(hs1_cache_max_size_metric) + self.assertIsNotNone( + hs1_cache_max_size_metric_value, + f"Missing metric {hs1_cache_max_size_metric} in cache metrics {metrics_map}", + ) + hs2_cache_max_size_metric_value = metrics_map.get(hs2_cache_max_size_metric) + self.assertIsNotNone( + hs2_cache_max_size_metric_value, + f"Missing metric {hs2_cache_max_size_metric} in cache metrics {metrics_map}", + ) + + # Sanity check the metric values + self.assertEqual(hs1_cache_size_metric_value, "1.0") + self.assertEqual(hs2_cache_size_metric_value, "1.0") + self.assertEqual(hs1_cache_max_size_metric_value, "777.0") + self.assertEqual(hs2_cache_max_size_metric_value, "777.0") + + +class LaterGaugeTests(unittest.HomeserverTestCase): + def setUp(self) -> None: + super().setUp() + self.later_gauge = LaterGauge( + name="foo", + desc="", + labelnames=[SERVER_NAME_LABEL], + ) + + def tearDown(self) -> None: + super().tearDown() + + REGISTRY.unregister(self.later_gauge) + all_later_gauges_to_clean_up_on_shutdown.pop(self.later_gauge.name, None) + + def test_later_gauge_multiple_servers(self) -> None: + """ + Test that LaterGauge metrics are reported correctly across multiple servers. We + will have an metrics entry for each homeserver that is labeled with the + `server_name` label. + """ + self.later_gauge.register_hook( + homeserver_instance_id="123", hook=lambda: {("hs1",): 1} + ) + self.later_gauge.register_hook( + homeserver_instance_id="456", hook=lambda: {("hs2",): 2} + ) + + metrics_map = get_latest_metrics() + + # Find the metrics from both homeservers + hs1_metric = 'foo{server_name="hs1"}' + hs1_metric_value = metrics_map.get(hs1_metric) + self.assertIsNotNone( + hs1_metric_value, + f"Missing metric {hs1_metric} in metrics {metrics_map}", + ) + self.assertEqual(hs1_metric_value, "1.0") + + hs2_metric = 'foo{server_name="hs2"}' + hs2_metric_value = metrics_map.get(hs2_metric) + self.assertIsNotNone( + hs2_metric_value, + f"Missing metric {hs2_metric} in metrics {metrics_map}", + ) + self.assertEqual(hs2_metric_value, "2.0") + + def test_later_gauge_hook_exception(self) -> None: + """ + Test that LaterGauge metrics are collected across multiple servers even if one + hooks is throwing an exception. + """ + + def raise_exception() -> NoReturn: + raise Exception("fake error generating data") + + # Make the hook for hs1 throw an exception + self.later_gauge.register_hook( + homeserver_instance_id="123", hook=raise_exception + ) + # Metrics from hs2 still work fine + self.later_gauge.register_hook( + homeserver_instance_id="456", hook=lambda: {("hs2",): 2} + ) + + metrics_map = get_latest_metrics() + + # Since we encountered an exception while trying to collect metrics from hs1, we + # don't expect to see it here. + hs1_metric = 'foo{server_name="hs1"}' + hs1_metric_value = metrics_map.get(hs1_metric) + self.assertIsNone( + hs1_metric_value, + ( + "Since we encountered an exception while trying to collect metrics from hs1" + f"we don't expect to see it the metrics_map {metrics_map}" + ), + ) + + # We should still see metrics from hs2 though + hs2_metric = 'foo{server_name="hs2"}' + hs2_metric_value = metrics_map.get(hs2_metric) + self.assertIsNotNone( + hs2_metric_value, + f"Missing metric {hs2_metric} in cache metrics {metrics_map}", + ) + self.assertEqual(hs2_metric_value, "2.0") + + +def get_latest_metrics() -> Dict[str, str]: + """ + Collect the latest metrics from the registry and parse them into an easy to use map. + The key includes the metric name and labels. + + Example output: + { + "synapse_util_caches_cache_size": "0.0", + "synapse_util_caches_cache_max_size{name="some_cache",server_name="hs1"}": "777.0", + ... + } + """ + metric_map = { + x.split(b" ")[0].decode("ascii"): x.split(b" ")[1].decode("ascii") + for x in filter( + lambda x: len(x) > 0 and not x.startswith(b"#"), + generate_latest(REGISTRY).split(b"\n"), + ) + } + + return metric_map diff --git a/tests/metrics/test_phone_home_stats.py b/tests/metrics/test_phone_home_stats.py new file mode 100644 index 0000000000..cf18d8635d --- /dev/null +++ b/tests/metrics/test_phone_home_stats.py @@ -0,0 +1,263 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import logging +from unittest.mock import AsyncMock + +from twisted.internet.testing import MemoryReactor + +from synapse.app.phone_stats_home import ( + PHONE_HOME_INTERVAL_SECONDS, + start_phone_stats_home, +) +from synapse.rest import admin, login, register, room +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock + +from tests import unittest +from tests.server import ThreadedMemoryReactorClock + +TEST_REPORT_STATS_ENDPOINT = "https://fake.endpoint/stats" +TEST_SERVER_CONTEXT = "test-server-context" + + +class PhoneHomeStatsTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + room.register_servlets, + register.register_servlets, + login.register_servlets, + ] + + def make_homeserver( + self, reactor: ThreadedMemoryReactorClock, clock: Clock + ) -> HomeServer: + # Configure the homeserver to enable stats reporting. + config = self.default_config() + config["report_stats"] = True + config["report_stats_endpoint"] = TEST_REPORT_STATS_ENDPOINT + + # Configure the server context so we can check it ends up being reported + config["server_context"] = TEST_SERVER_CONTEXT + + # Allow guests to be registered + config["allow_guest_access"] = True + + hs = self.setup_test_homeserver(config=config) + + # Replace the proxied http client with a mock, so we can inspect outbound requests to + # the configured stats endpoint. + self.put_json_mock = AsyncMock(return_value={}) + hs.get_proxied_http_client().put_json = self.put_json_mock # type: ignore[method-assign] + return hs + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.store = homeserver.get_datastores().main + + # Wait for the background updates to add the database triggers that keep the + # `event_stats` table up-to-date. + self.wait_for_background_updates() + + # Force stats reporting to occur + start_phone_stats_home(hs=homeserver) + + super().prepare(reactor, clock, homeserver) + + def _get_latest_phone_home_stats(self) -> JsonDict: + # Wait for `phone_stats_home` to be called again + a healthy margin (50s). + self.reactor.advance(2 * PHONE_HOME_INTERVAL_SECONDS + 50) + + # Extract the reported stats from our http client mock + mock_calls = self.put_json_mock.call_args_list + report_stats_calls = [] + for call in mock_calls: + if call.args[0] == TEST_REPORT_STATS_ENDPOINT: + report_stats_calls.append(call) + + self.assertGreaterEqual( + (len(report_stats_calls)), + 1, + "Expected at-least one call to the report_stats endpoint", + ) + + # Extract the phone home stats from the call + phone_home_stats = report_stats_calls[0].args[1] + + return phone_home_stats + + def _perform_user_actions(self) -> None: + """ + Perform some actions on the homeserver that would bump the phone home + stats. + + This creates a few users (including a guest), a room, and sends some messages. + Expected number of events: + - 10 unencrypted messages + - 5 encrypted messages + - 24 total events (including room state, etc) + """ + + # Create some users + user_1_mxid = self.register_user( + username="test_user_1", + password="test", + ) + user_2_mxid = self.register_user( + username="test_user_2", + password="test", + ) + # Note: `self.register_user` does not support guest registration, and updating the + # Admin API it calls to add a new parameter would cause the `mac` parameter to fail + # in a backwards-incompatible manner. Hence, we make a manual request here. + _guest_user_mxid = self.make_request( + method="POST", + path="/_matrix/client/v3/register?kind=guest", + content={ + "username": "guest_user", + "password": "test", + }, + shorthand=False, + ) + + # Log in to each user + user_1_token = self.login(username=user_1_mxid, password="test") + user_2_token = self.login(username=user_2_mxid, password="test") + + # Create a room between the two users + room_1_id = self.helper.create_room_as( + is_public=False, + tok=user_1_token, + ) + + # Mark this room as end-to-end encrypted + self.helper.send_state( + room_id=room_1_id, + event_type="m.room.encryption", + body={ + "algorithm": "m.megolm.v1.aes-sha2", + "rotation_period_ms": 604800000, + "rotation_period_msgs": 100, + }, + state_key="", + tok=user_1_token, + ) + + # User 1 invites user 2 + self.helper.invite( + room=room_1_id, + src=user_1_mxid, + targ=user_2_mxid, + tok=user_1_token, + ) + + # User 2 joins + self.helper.join( + room=room_1_id, + user=user_2_mxid, + tok=user_2_token, + ) + + # User 1 sends 10 unencrypted messages + for _ in range(10): + self.helper.send( + room_id=room_1_id, + body="Zoinks Scoob! A message!", + tok=user_1_token, + ) + + # User 2 sends 5 encrypted "messages" + for _ in range(5): + self.helper.send_event( + room_id=room_1_id, + type="m.room.encrypted", + content={ + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "some_key", + "ciphertext": { + "some_key": { + "type": 0, + "body": "encrypted_payload", + }, + }, + }, + tok=user_2_token, + ) + + def test_phone_home_stats(self) -> None: + """ + Test that the phone home stats contain the stats we expect based on + the scenario carried out in `prepare` + """ + # Do things to bump the stats + self._perform_user_actions() + + # Wait for the stats to be reported + phone_home_stats = self._get_latest_phone_home_stats() + + self.assertEqual( + phone_home_stats["homeserver"], self.hs.config.server.server_name + ) + + self.assertTrue(isinstance(phone_home_stats["memory_rss"], int)) + self.assertTrue(isinstance(phone_home_stats["cpu_average"], int)) + + self.assertEqual(phone_home_stats["server_context"], TEST_SERVER_CONTEXT) + + self.assertTrue(isinstance(phone_home_stats["timestamp"], int)) + self.assertTrue(isinstance(phone_home_stats["uptime_seconds"], int)) + self.assertTrue(isinstance(phone_home_stats["python_version"], str)) + + # We expect only our test users to exist on the homeserver + self.assertEqual(phone_home_stats["total_users"], 3) + self.assertEqual(phone_home_stats["total_nonbridged_users"], 3) + self.assertEqual(phone_home_stats["daily_user_type_native"], 2) + self.assertEqual(phone_home_stats["daily_user_type_guest"], 1) + self.assertEqual(phone_home_stats["daily_user_type_bridged"], 0) + self.assertEqual(phone_home_stats["total_room_count"], 1) + self.assertEqual(phone_home_stats["daily_active_users"], 2) + self.assertEqual(phone_home_stats["monthly_active_users"], 2) + self.assertEqual(phone_home_stats["daily_active_rooms"], 1) + self.assertEqual(phone_home_stats["daily_active_e2ee_rooms"], 1) + self.assertEqual(phone_home_stats["daily_messages"], 10) + self.assertEqual(phone_home_stats["daily_e2ee_messages"], 5) + self.assertEqual(phone_home_stats["daily_sent_messages"], 10) + self.assertEqual(phone_home_stats["daily_sent_e2ee_messages"], 5) + + # Our users have not been around for >30 days, hence these are all 0. + self.assertEqual(phone_home_stats["r30v2_users_all"], 0) + self.assertEqual(phone_home_stats["r30v2_users_android"], 0) + self.assertEqual(phone_home_stats["r30v2_users_ios"], 0) + self.assertEqual(phone_home_stats["r30v2_users_electron"], 0) + self.assertEqual(phone_home_stats["r30v2_users_web"], 0) + self.assertEqual( + phone_home_stats["cache_factor"], self.hs.config.caches.global_factor + ) + self.assertEqual( + phone_home_stats["event_cache_size"], + self.hs.config.caches.event_cache_size, + ) + self.assertEqual( + phone_home_stats["database_engine"], + self.hs.config.database.databases[0].config["name"], + ) + self.assertEqual( + phone_home_stats["database_server_version"], + self.hs.get_datastores().main.database_engine.server_version, + ) + + synapse_logger = logging.getLogger("synapse") + log_level = synapse_logger.getEffectiveLevel() + self.assertEqual(phone_home_stats["log_level"], logging.getLevelName(log_level)) diff --git a/tests/module_api/test_account_data_manager.py b/tests/module_api/test_account_data_manager.py index 1a1d5609b2..6539871c11 100644 --- a/tests/module_api/test_account_data_manager.py +++ b/tests/module_api/test_account_data_manager.py @@ -18,7 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import SynapseError from synapse.rest import admin diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index b6ba472d7d..6b761de36d 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -22,13 +22,13 @@ from typing import Any, Dict, Optional from unittest.mock import AsyncMock, Mock from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EduTypes, EventTypes from synapse.api.errors import NotFoundError from synapse.events import EventBase from synapse.federation.units import Transaction -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.handlers.presence import UserPresenceState from synapse.handlers.push_rules import InvalidRuleException from synapse.module_api import ModuleApi @@ -819,7 +819,7 @@ class ModuleApiTestCase(BaseModuleApiTestCase): # Delete the device. device_handler = self.hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) self.get_success(device_handler.delete_devices(user_id, [device_id])) # Check that the callback was called and the pushers still existed. diff --git a/tests/module_api/test_event_unsigned_addition.py b/tests/module_api/test_event_unsigned_addition.py index c429eff4d6..52e3858e6f 100644 --- a/tests/module_api/test_event_unsigned_addition.py +++ b/tests/module_api/test_event_unsigned_addition.py @@ -18,7 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.events import EventBase from synapse.rest import admin, login, room diff --git a/tests/module_api/test_spamchecker.py b/tests/module_api/test_spamchecker.py new file mode 100644 index 0000000000..fa19232ee9 --- /dev/null +++ b/tests/module_api/test_spamchecker.py @@ -0,0 +1,244 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +from typing import Literal, Union + +from twisted.internet.testing import MemoryReactor + +from synapse.config.server import DEFAULT_ROOM_VERSION +from synapse.rest import admin, login, room, room_upgrade_rest_servlet +from synapse.server import HomeServer +from synapse.types import Codes, JsonDict +from synapse.util import Clock + +from tests.server import FakeChannel +from tests.unittest import HomeserverTestCase + + +class SpamCheckerTestCase(HomeserverTestCase): + servlets = [ + room.register_servlets, + admin.register_servlets, + login.register_servlets, + room_upgrade_rest_servlet.register_servlets, + ] + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self._module_api = homeserver.get_module_api() + self.user_id = self.register_user("user", "password") + self.token = self.login("user", "password") + + def create_room(self, content: JsonDict) -> FakeChannel: + channel = self.make_request( + "POST", + "/_matrix/client/r0/createRoom", + content, + access_token=self.token, + ) + + return channel + + def test_may_user_create_room(self) -> None: + """Test that the may_user_create_room callback is called when a user + creates a room, and that it receives the correct parameters. + """ + + async def user_may_create_room( + user_id: str, room_config: JsonDict + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_room_config = room_config + self.last_user_id = user_id + return "NOT_SPAM" + + self._module_api.register_spam_checker_callbacks( + user_may_create_room=user_may_create_room + ) + + channel = self.create_room({"foo": "baa"}) + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_user_id, self.user_id) + self.assertEqual(self.last_room_config["foo"], "baa") + + def test_may_user_create_room_on_upgrade(self) -> None: + """Test that the may_user_create_room callback is called when a room is upgraded.""" + + # First, create a room to upgrade. + channel = self.create_room({"topic": "foo"}) + self.assertEqual(channel.code, 200) + room_id = channel.json_body["room_id"] + + async def user_may_create_room( + user_id: str, room_config: JsonDict + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_room_config = room_config + self.last_user_id = user_id + return "NOT_SPAM" + + # Register the callback for spam checking. + self._module_api.register_spam_checker_callbacks( + user_may_create_room=user_may_create_room + ) + + # Now upgrade the room. + channel = self.make_request( + "POST", + f"/_matrix/client/r0/rooms/{room_id}/upgrade", + # This will upgrade a room to the same version, but that's fine. + content={"new_version": DEFAULT_ROOM_VERSION}, + access_token=self.token, + ) + + # Check that the callback was called and the room was upgraded. + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_user_id, self.user_id) + # Check that the initial state received by callback contains the topic event. + self.assertTrue( + any( + event[0][0] == "m.room.topic" and event[1].get("topic") == "foo" + for event in self.last_room_config["initial_state"] + ) + ) + + def test_may_user_create_room_disallowed(self) -> None: + """Test that the codes response from may_user_create_room callback is respected + and returned via the API. + """ + + async def user_may_create_room( + user_id: str, room_config: JsonDict + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_room_config = room_config + self.last_user_id = user_id + return Codes.UNAUTHORIZED + + self._module_api.register_spam_checker_callbacks( + user_may_create_room=user_may_create_room + ) + + channel = self.create_room({"foo": "baa"}) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], Codes.UNAUTHORIZED) + self.assertEqual(self.last_user_id, self.user_id) + self.assertEqual(self.last_room_config["foo"], "baa") + + def test_may_user_create_room_compatibility(self) -> None: + """Test that the may_user_create_room callback is called when a user + creates a room for a module that uses the old callback signature + (without the `room_config` parameter) + """ + + async def user_may_create_room( + user_id: str, + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_user_id = user_id + return "NOT_SPAM" + + self._module_api.register_spam_checker_callbacks( + user_may_create_room=user_may_create_room + ) + + channel = self.create_room({"foo": "baa"}) + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_user_id, self.user_id) + + def test_user_may_send_state_event(self) -> None: + """Test that the user_may_send_state_event callback is called when a state event + is sent, and that it receives the correct parameters. + """ + + async def user_may_send_state_event( + user_id: str, + room_id: str, + event_type: str, + state_key: str, + content: JsonDict, + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_user_id = user_id + self.last_room_id = room_id + self.last_event_type = event_type + self.last_state_key = state_key + self.last_content = content + return "NOT_SPAM" + + self._module_api.register_spam_checker_callbacks( + user_may_send_state_event=user_may_send_state_event + ) + + channel = self.create_room({}) + self.assertEqual(channel.code, 200) + + room_id = channel.json_body["room_id"] + + event_type = "test.event.type" + state_key = "test.state.key" + channel = self.make_request( + "PUT", + "/_matrix/client/r0/rooms/%s/state/%s/%s" + % ( + room_id, + event_type, + state_key, + ), + content={"foo": "bar"}, + access_token=self.token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_user_id, self.user_id) + self.assertEqual(self.last_room_id, room_id) + self.assertEqual(self.last_event_type, event_type) + self.assertEqual(self.last_state_key, state_key) + self.assertEqual(self.last_content, {"foo": "bar"}) + + def test_user_may_send_state_event_disallows(self) -> None: + """Test that the user_may_send_state_event callback is called when a state event + is sent, and that the response is honoured. + """ + + async def user_may_send_state_event( + user_id: str, + room_id: str, + event_type: str, + state_key: str, + content: JsonDict, + ) -> Union[Literal["NOT_SPAM"], Codes]: + return Codes.FORBIDDEN + + self._module_api.register_spam_checker_callbacks( + user_may_send_state_event=user_may_send_state_event + ) + + channel = self.create_room({}) + self.assertEqual(channel.code, 200) + + room_id = channel.json_body["room_id"] + + event_type = "test.event.type" + state_key = "test.state.key" + channel = self.make_request( + "PUT", + "/_matrix/client/r0/rooms/%s/state/%s/%s" + % ( + room_id, + event_type, + state_key, + ), + content={"foo": "bar"}, + access_token=self.token, + ) + + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) diff --git a/tests/push/test_bulk_push_rule_evaluator.py b/tests/push/test_bulk_push_rule_evaluator.py index 16c1292812..7342a72dff 100644 --- a/tests/push/test_bulk_push_rule_evaluator.py +++ b/tests/push/test_bulk_push_rule_evaluator.py @@ -19,18 +19,19 @@ # # +from http import HTTPStatus from typing import Any, Optional from unittest.mock import AsyncMock, patch from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor -from synapse.api.constants import EventContentFields, RelationTypes +from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.api.room_versions import RoomVersions from synapse.push.bulk_push_rule_evaluator import BulkPushRuleEvaluator from synapse.rest import admin -from synapse.rest.client import login, register, room +from synapse.rest.client import login, push_rule, register, room from synapse.server import HomeServer from synapse.types import JsonDict, create_requester from synapse.util import Clock @@ -44,6 +45,7 @@ class TestBulkPushRuleEvaluator(HomeserverTestCase): room.register_servlets, login.register_servlets, register.register_servlets, + push_rule.register_servlets, ] def prepare( @@ -206,7 +208,10 @@ class TestBulkPushRuleEvaluator(HomeserverTestCase): bulk_evaluator._action_for_event_by_user.assert_not_called() def _create_and_process( - self, bulk_evaluator: BulkPushRuleEvaluator, content: Optional[JsonDict] = None + self, + bulk_evaluator: BulkPushRuleEvaluator, + content: Optional[JsonDict] = None, + type: str = "test", ) -> bool: """Returns true iff the `mentions` trigger an event push action.""" # Create a new message event which should cause a notification. @@ -214,7 +219,7 @@ class TestBulkPushRuleEvaluator(HomeserverTestCase): self.event_creation_handler.create_event( self.requester, { - "type": "test", + "type": type, "room_id": self.room_id, "content": content or {}, "sender": f"@bob:{self.hs.hostname}", @@ -446,3 +451,202 @@ class TestBulkPushRuleEvaluator(HomeserverTestCase): }, ) ) + + @override_config({"experimental_features": {"msc4306_enabled": True}}) + def test_thread_subscriptions(self) -> None: + bulk_evaluator = BulkPushRuleEvaluator(self.hs) + (thread_root_id,) = self.helper.send_messages(self.room_id, 1, tok=self.token) + + self.assertFalse( + self._create_and_process( + bulk_evaluator, + { + "msgtype": "m.text", + "body": "test message before subscription", + "m.relates_to": { + "rel_type": RelationTypes.THREAD, + "event_id": thread_root_id, + }, + }, + type=EventTypes.Message, + ) + ) + + self.get_success( + self.hs.get_datastores().main.subscribe_user_to_thread( + self.alice, + self.room_id, + thread_root_id, + automatic_event_orderings=None, + ) + ) + + self.assertTrue( + self._create_and_process( + bulk_evaluator, + { + "msgtype": "m.text", + "body": "test message after subscription", + "m.relates_to": { + "rel_type": RelationTypes.THREAD, + "event_id": thread_root_id, + }, + }, + type="m.room.message", + ) + ) + + @override_config({"experimental_features": {"msc4306_enabled": True}}) + def test_thread_subscriptions_suppression_after_keyword_mention_overrides( + self, + ) -> None: + """ + Tests one of the purposes of the `postcontent` push rule section: + When a keyword mention is configured (in the `content` section), + it does not get suppressed by the thread being unsubscribed. + """ + # add a keyword mention to alice's push rules + channel = self.make_request( + "PUT", + "/_matrix/client/v3/pushrules/global/content/biscuits", + {"pattern": "biscuits", "actions": ["notify"]}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + bulk_evaluator = BulkPushRuleEvaluator(self.hs) + (thread_root_id,) = self.helper.send_messages(self.room_id, 1, tok=self.token) + + self.assertFalse( + self._create_and_process( + bulk_evaluator, + { + "msgtype": "m.text", + "body": "do you want some cookies?", + "m.relates_to": { + "rel_type": RelationTypes.THREAD, + "event_id": thread_root_id, + }, + }, + type="m.room.message", + ), + "alice is not subscribed to thread and does not have a mention on 'cookies' so should not be notified", + ) + + self.assertTrue( + self._create_and_process( + bulk_evaluator, + { + "msgtype": "m.text", + "body": "biscuits are available in the kitchen", + "m.relates_to": { + "rel_type": RelationTypes.THREAD, + "event_id": thread_root_id, + }, + }, + type="m.room.message", + ), + "alice is not subscribed to thread but DOES have a mention on 'biscuits' so should be notified", + ) + + @override_config({"experimental_features": {"msc4306_enabled": True}}) + def test_thread_subscriptions_notification_before_keywords_and_mentions( + self, + ) -> None: + """ + Tests one of the purposes of the `postcontent` push rule section: + When a room is set to (what is commonly known as) 'keywords & mentions', we still receive notifications + for messages in threads that we are subscribed to. + Effectively making this 'keywords, mentions & subscriptions' + """ + # add a 'keywords & mentions' setting to the room alice's push rules + # In case this rule isn't clear: by adding a rule in the `room` section that does nothing, + # it stops execution of the push rules before we fall through to the `underride` section, + # where intuitively many kinds of messages will ambiently generate notifications. + # Mentions and keywords are triggered before the `room` block, so this doesn't suppress those. + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/pushrules/global/room/{self.room_id}", + {"actions": []}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + bulk_evaluator = BulkPushRuleEvaluator(self.hs) + (thread_root_id,) = self.helper.send_messages(self.room_id, 1, tok=self.token) + + # sanity check that our mentions still work + self.assertFalse( + self._create_and_process( + bulk_evaluator, + { + "msgtype": "m.text", + "body": "this is a plain message with no mention", + }, + type="m.room.message", + ), + "alice should not be notified (mentions & keywords room setting)", + ) + self.assertTrue( + self._create_and_process( + bulk_evaluator, + { + "msgtype": "m.text", + "body": "this is a message that mentions alice", + }, + type="m.room.message", + ), + "alice should be notified (mentioned)", + ) + + # let's have alice subscribe to the thread + self.get_success( + self.hs.get_datastores().main.subscribe_user_to_thread( + self.alice, + self.room_id, + thread_root_id, + automatic_event_orderings=None, + ) + ) + + self.assertTrue( + self._create_and_process( + bulk_evaluator, + { + "msgtype": "m.text", + "body": "some message in the thread", + "m.relates_to": { + "rel_type": RelationTypes.THREAD, + "event_id": thread_root_id, + }, + }, + type="m.room.message", + ), + "alice is subscribed to thread so should be notified", + ) + + def test_with_disabled_thread_subscriptions(self) -> None: + """ + Test what happens with threaded events when MSC4306 is disabled. + + FUTURE: If MSC4306 becomes enabled-by-default/accepted, this test is to be removed. + """ + bulk_evaluator = BulkPushRuleEvaluator(self.hs) + (thread_root_id,) = self.helper.send_messages(self.room_id, 1, tok=self.token) + + # When MSC4306 is not enabled, a threaded message generates a notification + # by default. + self.assertTrue( + self._create_and_process( + bulk_evaluator, + { + "msgtype": "m.text", + "body": "test message before subscription", + "m.relates_to": { + "rel_type": RelationTypes.THREAD, + "event_id": thread_root_id, + }, + }, + type="m.room.message", + ) + ) diff --git a/tests/push/test_email.py b/tests/push/test_email.py index 4fafb71897..4d9e42ac2c 100644 --- a/tests/push/test_email.py +++ b/tests/push/test_email.py @@ -18,16 +18,16 @@ # # import email.message +import importlib.resources as importlib_resources import os from http import HTTPStatus from typing import Any, Dict, List, Sequence, Tuple import attr -import pkg_resources from parameterized import parameterized from twisted.internet.defer import Deferred -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes, SynapseError @@ -59,11 +59,12 @@ class EmailPusherTests(HomeserverTestCase): def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: config = self.default_config() + templates = ( + importlib_resources.files("synapse").joinpath("res").joinpath("templates") + ) config["email"] = { "enable_notifs": True, - "template_dir": os.path.abspath( - pkg_resources.resource_filename("synapse", "res/templates") - ), + "template_dir": os.path.abspath(str(templates)), "expiry_template_html": "notice_expiry.html", "expiry_template_text": "notice_expiry.txt", "notif_template_html": "notif_mail.html", diff --git a/tests/push/test_http.py b/tests/push/test_http.py index bcca472617..370233c730 100644 --- a/tests/push/test_http.py +++ b/tests/push/test_http.py @@ -17,11 +17,13 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import Any, List, Tuple +from typing import Any, Dict, List, Tuple from unittest.mock import Mock +from parameterized import parameterized + from twisted.internet.defer import Deferred -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.logging.context import make_deferred_yieldable @@ -1085,3 +1087,161 @@ class HTTPPusherTests(HomeserverTestCase): self.pump() self.assertEqual(len(self.push_attempts), 11) + + @parameterized.expand( + [ + # Badge count disabled + (True, True), + (True, False), + # Badge count enabled + (False, True), + (False, False), + ] + ) + @override_config({"experimental_features": {"msc4076_enabled": True}}) + def test_msc4076_badge_count( + self, disable_badge_count: bool, event_id_only: bool + ) -> None: + # Register the user who gets notified + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Register the user who sends the message + other_user_id = self.register_user("otheruser", "pass") + other_access_token = self.login("otheruser", "pass") + + # Register the pusher with disable_badge_count set to True + user_tuple = self.get_success( + self.hs.get_datastores().main.get_user_by_access_token(access_token) + ) + assert user_tuple is not None + device_id = user_tuple.device_id + + # Set the push data dict based on test input parameters + push_data: Dict[str, Any] = { + "url": "http://example.com/_matrix/push/v1/notify", + } + if disable_badge_count: + push_data["org.matrix.msc4076.disable_badge_count"] = True + if event_id_only: + push_data["format"] = "event_id_only" + + self.get_success( + self.hs.get_pusherpool().add_or_update_pusher( + user_id=user_id, + device_id=device_id, + kind="http", + app_id="m.http", + app_display_name="HTTP Push Notifications", + device_display_name="pushy push", + pushkey="a@example.com", + lang=None, + data=push_data, + ) + ) + + # Create a room + room = self.helper.create_room_as(user_id, tok=access_token) + + # The other user joins + self.helper.join(room=room, user=other_user_id, tok=other_access_token) + + # The other user sends a message + self.helper.send(room, body="Hi!", tok=other_access_token) + + # Advance time a bit, so the pusher will register something has happened + self.pump() + + # One push was attempted to be sent + self.assertEqual(len(self.push_attempts), 1) + self.assertEqual( + self.push_attempts[0][1], "http://example.com/_matrix/push/v1/notify" + ) + + if disable_badge_count: + # Verify that the notification DOESN'T contain a counts field + self.assertNotIn("counts", self.push_attempts[0][2]["notification"]) + else: + # Ensure that the notification DOES contain a counts field + self.assertIn("counts", self.push_attempts[0][2]["notification"]) + self.assertEqual( + self.push_attempts[0][2]["notification"]["counts"]["unread"], 1 + ) + + def test_push_backoff(self) -> None: + """ + The HTTP pusher will backoff correctly if it fails to contact the pusher. + """ + + # Register the user who gets notified + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Register the user who sends the message + other_user_id = self.register_user("otheruser", "pass") + other_access_token = self.login("otheruser", "pass") + + # Register the pusher + user_tuple = self.get_success( + self.hs.get_datastores().main.get_user_by_access_token(access_token) + ) + assert user_tuple is not None + device_id = user_tuple.device_id + + self.get_success( + self.hs.get_pusherpool().add_or_update_pusher( + user_id=user_id, + device_id=device_id, + kind="http", + app_id="m.http", + app_display_name="HTTP Push Notifications", + device_display_name="pushy push", + pushkey="a@example.com", + lang=None, + data={"url": "http://example.com/_matrix/push/v1/notify"}, + ) + ) + + # Create a room with the other user + room = self.helper.create_room_as(user_id, tok=access_token) + self.helper.join(room=room, user=other_user_id, tok=other_access_token) + + # The other user sends some messages + self.helper.send(room, body="Message 1", tok=other_access_token) + + # One push was attempted to be sent + self.assertEqual(len(self.push_attempts), 1) + self.assertEqual( + self.push_attempts[0][1], "http://example.com/_matrix/push/v1/notify" + ) + self.assertEqual( + self.push_attempts[0][2]["notification"]["content"]["body"], "Message 1" + ) + self.push_attempts[0][0].callback({}) + self.pump() + + # Send another message, this time it fails + self.helper.send(room, body="Message 2", tok=other_access_token) + self.assertEqual(len(self.push_attempts), 2) + self.push_attempts[1][0].errback(Exception("couldn't connect")) + self.pump() + + # Sending yet another message doesn't trigger a push immediately + self.helper.send(room, body="Message 3", tok=other_access_token) + self.pump() + self.assertEqual(len(self.push_attempts), 2) + + # .. but waiting for a bit will cause more pushes + self.reactor.advance(10) + self.assertEqual(len(self.push_attempts), 3) + self.assertEqual( + self.push_attempts[2][2]["notification"]["content"]["body"], "Message 2" + ) + self.push_attempts[2][0].callback({}) + self.pump() + + self.assertEqual(len(self.push_attempts), 4) + self.assertEqual( + self.push_attempts[3][2]["notification"]["content"]["body"], "Message 3" + ) + self.push_attempts[3][0].callback({}) diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py index 3898532acf..3a351acffa 100644 --- a/tests/push/test_push_rule_evaluator.py +++ b/tests/push/test_push_rule_evaluator.py @@ -21,7 +21,7 @@ from typing import Any, Dict, List, Optional, Union, cast -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EventTypes, HistoryVisibility, Membership @@ -150,6 +150,7 @@ class PushRuleEvaluatorTestCase(unittest.TestCase): *, related_events: Optional[JsonDict] = None, msc4210: bool = False, + msc4306: bool = False, ) -> PushRuleEvaluator: event = FrozenEvent( { @@ -176,6 +177,7 @@ class PushRuleEvaluatorTestCase(unittest.TestCase): room_version_feature_flags=event.room_version.msc3931_push_features, msc3931_enabled=True, msc4210_enabled=msc4210, + msc4306_enabled=msc4306, ) def test_display_name(self) -> None: @@ -806,6 +808,112 @@ class PushRuleEvaluatorTestCase(unittest.TestCase): ) ) + def test_thread_subscription_subscribed(self) -> None: + """ + Test MSC4306 thread subscription push rules against an event in a subscribed thread. + """ + evaluator = self._get_evaluator( + { + "msgtype": "m.text", + "body": "Squawk", + "m.relates_to": { + "event_id": "$threadroot", + "rel_type": "m.thread", + }, + }, + msc4306=True, + ) + self.assertTrue( + evaluator.matches( + { + "kind": "io.element.msc4306.thread_subscription", + "subscribed": True, + }, + None, + None, + msc4306_thread_subscription_state=True, + ) + ) + self.assertFalse( + evaluator.matches( + { + "kind": "io.element.msc4306.thread_subscription", + "subscribed": False, + }, + None, + None, + msc4306_thread_subscription_state=True, + ) + ) + + def test_thread_subscription_unsubscribed(self) -> None: + """ + Test MSC4306 thread subscription push rules against an event in an unsubscribed thread. + """ + evaluator = self._get_evaluator( + { + "msgtype": "m.text", + "body": "Squawk", + "m.relates_to": { + "event_id": "$threadroot", + "rel_type": "m.thread", + }, + }, + msc4306=True, + ) + self.assertFalse( + evaluator.matches( + { + "kind": "io.element.msc4306.thread_subscription", + "subscribed": True, + }, + None, + None, + msc4306_thread_subscription_state=False, + ) + ) + self.assertTrue( + evaluator.matches( + { + "kind": "io.element.msc4306.thread_subscription", + "subscribed": False, + }, + None, + None, + msc4306_thread_subscription_state=False, + ) + ) + + def test_thread_subscription_unthreaded(self) -> None: + """ + Test MSC4306 thread subscription push rules against an unthreaded event. + """ + evaluator = self._get_evaluator( + {"msgtype": "m.text", "body": "Squawk"}, msc4306=True + ) + self.assertFalse( + evaluator.matches( + { + "kind": "io.element.msc4306.thread_subscription", + "subscribed": True, + }, + None, + None, + msc4306_thread_subscription_state=None, + ) + ) + self.assertFalse( + evaluator.matches( + { + "kind": "io.element.msc4306.thread_subscription", + "subscribed": False, + }, + None, + None, + msc4306_thread_subscription_state=None, + ) + ) + class TestBulkPushRuleEvaluator(unittest.HomeserverTestCase): """Tests for the bulk push rule evaluator""" @@ -823,9 +931,9 @@ class TestBulkPushRuleEvaluator(unittest.HomeserverTestCase): # Define an application service so that we can register appservice users self._service_token = "some_token" self._service = ApplicationService( - self._service_token, - "as1", - "@as.sender:test", + token=self._service_token, + id="as1", + sender=UserID.from_string("@as.sender:test"), namespaces={ "users": [ {"regex": "@_as_.*:test", "exclusive": True}, diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 8437da1cdd..e756021937 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -23,8 +23,8 @@ from typing import Any, Dict, List, Optional, Set, Tuple from twisted.internet.address import IPv4Address from twisted.internet.protocol import Protocol, connectionDone +from twisted.internet.testing import MemoryReactor from twisted.python.failure import Failure -from twisted.test.proto_helpers import MemoryReactor from twisted.web.resource import Resource from synapse.app.generic_worker import GenericWorkerServer @@ -32,7 +32,6 @@ from synapse.config.workers import InstanceTcpLocationConfig, InstanceUnixLocati from synapse.http.site import SynapseRequest, SynapseSite from synapse.replication.http import ReplicationRestResource from synapse.replication.tcp.client import ReplicationDataHandler -from synapse.replication.tcp.handler import ReplicationCommandHandler from synapse.replication.tcp.protocol import ( ClientReplicationStreamProtocol, ServerReplicationStreamProtocol, @@ -97,7 +96,7 @@ class BaseStreamTestCase(unittest.HomeserverTestCase): self.test_handler = self._build_replication_data_handler() self.worker_hs._replication_data_handler = self.test_handler # type: ignore[attr-defined] - repl_handler = ReplicationCommandHandler(self.worker_hs) + repl_handler = self.worker_hs.get_replication_command_handler() self.client = ClientReplicationStreamProtocol( self.worker_hs, "client", @@ -220,7 +219,7 @@ class BaseStreamTestCase(unittest.HomeserverTestCase): fetching updates for given stream. """ - path: bytes = request.path # type: ignore + path: bytes = request.path self.assertRegex( path, rb"^/_synapse/replication/get_repl_stream_updates/%s/[^/]+$" diff --git a/tests/replication/http/test__base.py b/tests/replication/http/test__base.py index 2eaad3707a..31d3163c01 100644 --- a/tests/replication/http/test__base.py +++ b/tests/replication/http/test__base.py @@ -46,7 +46,7 @@ class CancellableReplicationEndpoint(ReplicationEndpoint): self.clock = hs.get_clock() @staticmethod - async def _serialize_payload() -> JsonDict: + async def _serialize_payload(**kwargs: ReplicationEndpoint) -> JsonDict: return {} @cancellable @@ -68,7 +68,7 @@ class UncancellableReplicationEndpoint(ReplicationEndpoint): self.clock = hs.get_clock() @staticmethod - async def _serialize_payload() -> JsonDict: + async def _serialize_payload(**kwargs: ReplicationEndpoint) -> JsonDict: return {} async def _handle_request( # type: ignore[override] diff --git a/tests/replication/storage/_base.py b/tests/replication/storage/_base.py index 27dff0034f..97e744127c 100644 --- a/tests/replication/storage/_base.py +++ b/tests/replication/storage/_base.py @@ -22,7 +22,7 @@ from typing import Any, Callable, Iterable, Optional from unittest.mock import Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer from synapse.util import Clock diff --git a/tests/replication/storage/test_events.py b/tests/replication/storage/test_events.py index 1afe523d02..b3ca204995 100644 --- a/tests/replication/storage/test_events.py +++ b/tests/replication/storage/test_events.py @@ -24,7 +24,7 @@ from typing import Any, Iterable, List, Optional, Tuple from canonicaljson import encode_canonical_json from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import ReceiptTypes from synapse.api.room_versions import RoomVersions diff --git a/tests/replication/tcp/streams/test_events.py b/tests/replication/tcp/streams/test_events.py index fdc74efb5a..cd6fe53a96 100644 --- a/tests/replication/tcp/streams/test_events.py +++ b/tests/replication/tcp/streams/test_events.py @@ -22,7 +22,7 @@ from typing import Any, List, Optional from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, Membership from synapse.events import EventBase @@ -324,7 +324,7 @@ class EventsStreamTestCase(BaseStreamTestCase): pls = self.helper.get_state( self.room_id, EventTypes.PowerLevels, tok=self.user_tok ) - pls["users"].update({u: 50 for u in user_ids}) + pls["users"].update(dict.fromkeys(user_ids, 50)) self.helper.send_state( self.room_id, EventTypes.PowerLevels, diff --git a/tests/replication/tcp/streams/test_thread_subscriptions.py b/tests/replication/tcp/streams/test_thread_subscriptions.py new file mode 100644 index 0000000000..7283aa851e --- /dev/null +++ b/tests/replication/tcp/streams/test_thread_subscriptions.py @@ -0,0 +1,157 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from twisted.internet.testing import MemoryReactor + +from synapse.replication.tcp.streams._base import ( + _STREAM_UPDATE_TARGET_ROW_COUNT, + ThreadSubscriptionsStream, +) +from synapse.server import HomeServer +from synapse.storage.database import LoggingTransaction +from synapse.util import Clock + +from tests.replication._base import BaseStreamTestCase + + +class ThreadSubscriptionsStreamTestCase(BaseStreamTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + # Postgres + def f(txn: LoggingTransaction) -> None: + txn.execute( + """ + ALTER TABLE thread_subscriptions + DROP CONSTRAINT thread_subscriptions_fk_users, + DROP CONSTRAINT thread_subscriptions_fk_rooms, + DROP CONSTRAINT thread_subscriptions_fk_events; + """, + ) + + self.get_success( + self.hs.get_datastores().main.db_pool.runInteraction( + "disable_foreign_keys", f + ) + ) + + def test_thread_subscription_updates(self) -> None: + """Test replication with thread subscription updates""" + store = self.hs.get_datastores().main + + # Create thread subscription updates + updates = [] + room_id = "!test_room:example.com" + + # Generate several thread subscription updates + for i in range(_STREAM_UPDATE_TARGET_ROW_COUNT + 5): + thread_root_id = f"$thread_{i}:example.com" + self.get_success( + store.subscribe_user_to_thread( + "@test_user:example.org", + room_id, + thread_root_id, + automatic_event_orderings=None, + ) + ) + updates.append(thread_root_id) + + # Also add one in a different room + other_room_id = "!other_room:example.com" + other_thread_root_id = "$other_thread:example.com" + self.get_success( + store.subscribe_user_to_thread( + "@test_user:example.org", + other_room_id, + other_thread_root_id, + automatic_event_orderings=None, + ) + ) + + # Not yet connected: no rows should yet have been received + self.assertEqual([], self.test_handler.received_rdata_rows) + + # Now reconnect to pull the updates + self.reconnect() + self.replicate() + + # We should have received all the expected rows in the right order + # Filter the updates to only include thread subscription changes + received_rows = [ + upd + for upd in self.test_handler.received_rdata_rows + if upd[0] == ThreadSubscriptionsStream.NAME + ] + + # Verify all the thread subscription updates + for thread_id in updates: + (stream_name, token, row) = received_rows.pop(0) + self.assertEqual(stream_name, ThreadSubscriptionsStream.NAME) + self.assertIsInstance(row, ThreadSubscriptionsStream.ROW_TYPE) + self.assertEqual(row.user_id, "@test_user:example.org") + self.assertEqual(row.room_id, room_id) + self.assertEqual(row.event_id, thread_id) + + # Verify the last update in the different room + (stream_name, token, row) = received_rows.pop(0) + self.assertEqual(stream_name, ThreadSubscriptionsStream.NAME) + self.assertIsInstance(row, ThreadSubscriptionsStream.ROW_TYPE) + self.assertEqual(row.user_id, "@test_user:example.org") + self.assertEqual(row.room_id, other_room_id) + self.assertEqual(row.event_id, other_thread_root_id) + + self.assertEqual([], received_rows) + + def test_multiple_users_thread_subscription_updates(self) -> None: + """Test replication with thread subscription updates for multiple users""" + store = self.hs.get_datastores().main + room_id = "!test_room:example.com" + thread_root_id = "$thread_root:example.com" + + # Create updates for multiple users + users = ["@user1:example.com", "@user2:example.com", "@user3:example.com"] + for user_id in users: + self.get_success( + store.subscribe_user_to_thread( + user_id, room_id, thread_root_id, automatic_event_orderings=None + ) + ) + + # Check no rows have been received yet + self.replicate() + self.assertEqual([], self.test_handler.received_rdata_rows) + + # Not yet connected: no rows should yet have been received + self.reconnect() + self.replicate() + + # We should have received all the expected rows + # Filter the updates to only include thread subscription changes + received_rows = [ + upd + for upd in self.test_handler.received_rdata_rows + if upd[0] == ThreadSubscriptionsStream.NAME + ] + + # Should have one update per user + self.assertEqual(len(received_rows), len(users)) + + # Verify all updates + for i, user_id in enumerate(users): + (stream_name, token, row) = received_rows[i] + self.assertEqual(stream_name, ThreadSubscriptionsStream.NAME) + self.assertIsInstance(row, ThreadSubscriptionsStream.ROW_TYPE) + self.assertEqual(row.user_id, user_id) + self.assertEqual(row.room_id, room_id) + self.assertEqual(row.event_id, thread_root_id) diff --git a/tests/replication/tcp/streams/test_typing.py b/tests/replication/tcp/streams/test_typing.py index b1c2f5b03b..e2b2299106 100644 --- a/tests/replication/tcp/streams/test_typing.py +++ b/tests/replication/tcp/streams/test_typing.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +import logging from unittest.mock import Mock from synapse.handlers.typing import RoomMember, TypingWriterHandler @@ -99,73 +100,86 @@ class TypingStreamTestCase(BaseStreamTestCase): This is emulated by jumping the stream ahead, then reconnecting (which sends the proper position and RDATA). """ - typing = self.hs.get_typing_handler() - assert isinstance(typing, TypingWriterHandler) + # FIXME: Because huge RDATA log line is triggered in this test, + # trial breaks, sometimes (flakily) failing the test run. + # ref: https://github.com/twisted/twisted/issues/12482 + # To remove this, we would need to fix the above issue and + # update, including in olddeps (so several years' wait). + server_logger = logging.getLogger("tests.server") + server_logger_was_disabled = server_logger.disabled + server_logger.disabled = True + try: + typing = self.hs.get_typing_handler() + assert isinstance(typing, TypingWriterHandler) - # Create a typing update before we reconnect so that there is a missing - # update to fetch. - typing._push_update(member=RoomMember(ROOM_ID, USER_ID), typing=True) + # Create a typing update before we reconnect so that there is a missing + # update to fetch. + typing._push_update(member=RoomMember(ROOM_ID, USER_ID), typing=True) - self.reconnect() + self.reconnect() - typing._push_update(member=RoomMember(ROOM_ID, USER_ID), typing=True) + typing._push_update(member=RoomMember(ROOM_ID, USER_ID), typing=True) - self.reactor.advance(0) + self.reactor.advance(0) - # We should now see an attempt to connect to the master - request = self.handle_http_replication_attempt() - self.assert_request_is_get_repl_stream_updates(request, "typing") + # We should now see an attempt to connect to the master + request = self.handle_http_replication_attempt() + self.assert_request_is_get_repl_stream_updates(request, "typing") - self.mock_handler.on_rdata.assert_called_once() - stream_name, _, token, rdata_rows = self.mock_handler.on_rdata.call_args[0] - self.assertEqual(stream_name, "typing") - self.assertEqual(1, len(rdata_rows)) - row: TypingStream.TypingStreamRow = rdata_rows[0] - self.assertEqual(ROOM_ID, row.room_id) - self.assertEqual([USER_ID], row.user_ids) + self.mock_handler.on_rdata.assert_called_once() + stream_name, _, token, rdata_rows = self.mock_handler.on_rdata.call_args[0] + self.assertEqual(stream_name, "typing") + self.assertEqual(1, len(rdata_rows)) + row: TypingStream.TypingStreamRow = rdata_rows[0] + self.assertEqual(ROOM_ID, row.room_id) + self.assertEqual([USER_ID], row.user_ids) - # Push the stream forward a bunch so it can be reset. - for i in range(100): - typing._push_update( - member=RoomMember(ROOM_ID, "@test%s:blue" % i), typing=True + # Push the stream forward a bunch so it can be reset. + for i in range(100): + typing._push_update( + member=RoomMember(ROOM_ID, "@test%s:blue" % i), typing=True + ) + self.reactor.advance(0) + + # Disconnect. + self.disconnect() + + # Reset the typing handler + self.hs.get_replication_streams()["typing"].last_token = 0 + self.hs.get_replication_command_handler()._streams["typing"].last_token = 0 + typing._latest_room_serial = 0 + typing._typing_stream_change_cache = StreamChangeCache( + name="TypingStreamChangeCache", + server_name=self.hs.hostname, + current_stream_pos=typing._latest_room_serial, ) - self.reactor.advance(0) + typing._reset() - # Disconnect. - self.disconnect() + # Reconnect. + self.reconnect() + self.pump(0.1) - # Reset the typing handler - self.hs.get_replication_streams()["typing"].last_token = 0 - self.hs.get_replication_command_handler()._streams["typing"].last_token = 0 - typing._latest_room_serial = 0 - typing._typing_stream_change_cache = StreamChangeCache( - "TypingStreamChangeCache", typing._latest_room_serial - ) - typing._reset() + # We should now see an attempt to connect to the master + request = self.handle_http_replication_attempt() + self.assert_request_is_get_repl_stream_updates(request, "typing") - # Reconnect. - self.reconnect() - self.pump(0.1) + # Reset the test code. + self.mock_handler.on_rdata.reset_mock() + self.mock_handler.on_rdata.assert_not_called() - # We should now see an attempt to connect to the master - request = self.handle_http_replication_attempt() - self.assert_request_is_get_repl_stream_updates(request, "typing") + # Push additional data. + typing._push_update(member=RoomMember(ROOM_ID_2, USER_ID_2), typing=False) + self.reactor.advance(0) - # Reset the test code. - self.mock_handler.on_rdata.reset_mock() - self.mock_handler.on_rdata.assert_not_called() + self.mock_handler.on_rdata.assert_called_once() + stream_name, _, token, rdata_rows = self.mock_handler.on_rdata.call_args[0] + self.assertEqual(stream_name, "typing") + self.assertEqual(1, len(rdata_rows)) + row = rdata_rows[0] + self.assertEqual(ROOM_ID_2, row.room_id) + self.assertEqual([], row.user_ids) - # Push additional data. - typing._push_update(member=RoomMember(ROOM_ID_2, USER_ID_2), typing=False) - self.reactor.advance(0) - - self.mock_handler.on_rdata.assert_called_once() - stream_name, _, token, rdata_rows = self.mock_handler.on_rdata.call_args[0] - self.assertEqual(stream_name, "typing") - self.assertEqual(1, len(rdata_rows)) - row = rdata_rows[0] - self.assertEqual(ROOM_ID_2, row.room_id) - self.assertEqual([], row.user_ids) - - # The token should have been reset. - self.assertEqual(token, 1) + # The token should have been reset. + self.assertEqual(token, 1) + finally: + server_logger.disabled = server_logger_was_disabled diff --git a/tests/replication/test_auth.py b/tests/replication/test_auth.py index 7820de8acc..640ed4e8f3 100644 --- a/tests/replication/test_auth.py +++ b/tests/replication/test_auth.py @@ -20,7 +20,7 @@ # import logging -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.rest.client import register from synapse.server import HomeServer diff --git a/tests/replication/test_federation_ack.py b/tests/replication/test_federation_ack.py index 14c9483f2b..440c1d45af 100644 --- a/tests/replication/test_federation_ack.py +++ b/tests/replication/test_federation_ack.py @@ -21,7 +21,7 @@ from unittest import mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.app.generic_worker import GenericWorkerServer from synapse.replication.tcp.commands import FederationAckCommand diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 4429d0f4e2..1fed4ec631 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -22,14 +22,26 @@ import logging from unittest.mock import AsyncMock, Mock from netaddr import IPSet +from signedjson.key import ( + encode_verify_key_base64, + generate_signing_key, + get_verify_key, +) + +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, Membership -from synapse.events.builder import EventBuilderFactory +from synapse.api.room_versions import RoomVersion +from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.events import EventBase, make_event_from_dict from synapse.handlers.typing import TypingWriterHandler from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent from synapse.rest.admin import register_servlets_for_client_rest_resource from synapse.rest.client import login, room -from synapse.types import UserID, create_requester +from synapse.server import HomeServer +from synapse.storage.keys import FetchKeyResult +from synapse.types import JsonDict, UserID, create_requester +from synapse.util import Clock from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import get_clock @@ -56,13 +68,18 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): reactor, _ = get_clock() self.matrix_federation_agent = MatrixFederationAgent( - reactor, + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, ip_blocklist=IPSet(), + proxy_config=None, ) + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.storage_controllers = hs.get_storage_controllers() + def test_send_event_single_sender(self) -> None: """Test that using a single federation sender worker correctly sends a new event. @@ -243,35 +260,92 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): self.assertTrue(sent_on_1) self.assertTrue(sent_on_2) + def create_fake_event_from_remote_server( + self, remote_server_name: str, event_dict: JsonDict, room_version: RoomVersion + ) -> EventBase: + """ + This is similar to what `FederatingHomeserverTestCase` is doing but we don't + need all of the extra baggage and we want to be able to create an event from + many remote servers. + """ + + # poke the other server's signing key into the key store, so that we don't + # make requests for it + other_server_signature_key = generate_signing_key("test") + verify_key = get_verify_key(other_server_signature_key) + verify_key_id = "%s:%s" % (verify_key.alg, verify_key.version) + + self.get_success( + self.hs.get_datastores().main.store_server_keys_response( + remote_server_name, + from_server=remote_server_name, + ts_added_ms=self.clock.time_msec(), + verify_keys={ + verify_key_id: FetchKeyResult( + verify_key=verify_key, + valid_until_ts=self.clock.time_msec() + 10000, + ), + }, + response_json={ + "verify_keys": { + verify_key_id: {"key": encode_verify_key_base64(verify_key)} + } + }, + ) + ) + + add_hashes_and_signatures( + room_version=room_version, + event_dict=event_dict, + signature_name=remote_server_name, + signing_key=other_server_signature_key, + ) + event = make_event_from_dict( + event_dict, + room_version=room_version, + ) + + return event + def create_room_with_remote_server( self, user: str, token: str, remote_server: str = "other_server" ) -> str: - room = self.helper.create_room_as(user, tok=token) + room_id = self.helper.create_room_as(user, tok=token) store = self.hs.get_datastores().main federation = self.hs.get_federation_event_handler() - prev_event_ids = self.get_success(store.get_latest_event_ids_in_room(room)) - room_version = self.get_success(store.get_room_version(room)) + room_version = self.get_success(store.get_room_version(room_id)) - factory = EventBuilderFactory(self.hs) - factory.hostname = remote_server + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) + ) + + # Figure out what the forward extremities in the room are (the most recent + # events that aren't tied into the DAG) + prev_event_ids = self.get_success(store.get_latest_event_ids_in_room(room_id)) user_id = UserID("user", remote_server).to_string() - event_dict = { - "type": EventTypes.Member, - "state_key": user_id, - "content": {"membership": Membership.JOIN}, - "sender": user_id, - "room_id": room, - } - - builder = factory.for_room_version(room_version, event_dict) - join_event = self.get_success( - builder.build(prev_event_ids=list(prev_event_ids), auth_event_ids=None) + join_event = self.create_fake_event_from_remote_server( + remote_server_name=remote_server, + event_dict={ + "room_id": room_id, + "sender": user_id, + "type": EventTypes.Member, + "state_key": user_id, + "depth": 1000, + "origin_server_ts": 1, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + ], + "prev_events": list(prev_event_ids), + }, + room_version=room_version, ) self.get_success(federation.on_send_membership_event(remote_server, join_event)) self.replicate() - return room + return room_id diff --git a/tests/replication/test_module_cache_invalidation.py b/tests/replication/test_module_cache_invalidation.py index 1e7183edaa..8d5d0cce9a 100644 --- a/tests/replication/test_module_cache_invalidation.py +++ b/tests/replication/test_module_cache_invalidation.py @@ -35,6 +35,7 @@ KEY = "mykey" class TestCache: current_value = FIRST_VALUE + server_name = "test_server" # nb must be called this for @cached @cached() async def cached_function(self, user_id: str) -> str: diff --git a/tests/replication/test_multi_media_repo.py b/tests/replication/test_multi_media_repo.py index f36af877c4..228a803c1d 100644 --- a/tests/replication/test_multi_media_repo.py +++ b/tests/replication/test_multi_media_repo.py @@ -23,7 +23,7 @@ import os from typing import Any, Optional, Tuple from twisted.internet.protocol import Factory -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.http import HTTPChannel from twisted.web.server import Request diff --git a/tests/replication/test_pusher_shard.py b/tests/replication/test_pusher_shard.py index 1b0bdc262a..d63054c631 100644 --- a/tests/replication/test_pusher_shard.py +++ b/tests/replication/test_pusher_shard.py @@ -22,7 +22,7 @@ import logging from unittest.mock import Mock from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, room diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py index ce6ad75901..797ad003ef 100644 --- a/tests/replication/test_sharded_event_persister.py +++ b/tests/replication/test_sharded_event_persister.py @@ -21,7 +21,7 @@ import logging from unittest.mock import patch -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, room, sync diff --git a/tests/replication/test_sharded_receipts.py b/tests/replication/test_sharded_receipts.py index e400267819..6b3ecdad78 100644 --- a/tests/replication/test_sharded_receipts.py +++ b/tests/replication/test_sharded_receipts.py @@ -20,7 +20,7 @@ # import logging -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import ReceiptTypes from synapse.rest import admin diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 5483f8f37f..b74e8388e9 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -20,11 +20,11 @@ # import urllib.parse -from typing import Dict +from typing import Dict, cast from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource import synapse.rest.admin @@ -32,6 +32,7 @@ from synapse.http.server import JsonResource from synapse.rest.admin import VersionServlet from synapse.rest.client import login, media, room from synapse.server import HomeServer +from synapse.types import UserID from synapse.util import Clock from tests import unittest @@ -227,10 +228,25 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): # Upload some media response_1 = self.helper.upload_media(SMALL_PNG, tok=non_admin_user_tok) response_2 = self.helper.upload_media(SMALL_PNG, tok=non_admin_user_tok) + response_3 = self.helper.upload_media(SMALL_PNG, tok=non_admin_user_tok) # Extract media IDs server_and_media_id_1 = response_1["content_uri"][6:] server_and_media_id_2 = response_2["content_uri"][6:] + server_and_media_id_3 = response_3["content_uri"][6:] + + # Remove the hash from the media to simulate historic media. + self.get_success( + self.hs.get_datastores().main.update_local_media( + media_id=server_and_media_id_3.split("/")[1], + media_type="image/png", + upload_name=None, + media_length=123, + user_id=UserID.from_string(non_admin_user), + # Hack to force some media to have no hash. + sha256=cast(str, None), + ) + ) # Quarantine all media by this user url = "/_synapse/admin/v1/user/%s/media/quarantine" % urllib.parse.quote( @@ -244,12 +260,13 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): self.pump(1.0) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual( - channel.json_body, {"num_quarantined": 2}, "Expected 2 quarantined items" + channel.json_body, {"num_quarantined": 3}, "Expected 3 quarantined items" ) # Attempt to access each piece of media self._ensure_quarantined(admin_user_tok, server_and_media_id_1) self._ensure_quarantined(admin_user_tok, server_and_media_id_2) + self._ensure_quarantined(admin_user_tok, server_and_media_id_3) def test_cannot_quarantine_safe_media(self) -> None: self.register_user("user_admin", "pass", admin=True) diff --git a/tests/rest/admin/test_background_updates.py b/tests/rest/admin/test_background_updates.py index f33aada64b..dd116e79f1 100644 --- a/tests/rest/admin/test_background_updates.py +++ b/tests/rest/admin/test_background_updates.py @@ -22,7 +22,7 @@ from typing import Collection from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes diff --git a/tests/rest/admin/test_device.py b/tests/rest/admin/test_device.py index a88c77bd19..c564e0c9a7 100644 --- a/tests/rest/admin/test_device.py +++ b/tests/rest/admin/test_device.py @@ -22,12 +22,12 @@ import urllib.parse from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes -from synapse.handlers.device import DeviceHandler -from synapse.rest.client import login +from synapse.handlers.device import DeviceWriterHandler +from synapse.rest.client import devices, login from synapse.server import HomeServer from synapse.util import Clock @@ -42,7 +42,7 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) + assert isinstance(handler, DeviceWriterHandler) self.handler = handler self.admin_user = self.register_user("admin", "pass", admin=True) @@ -299,6 +299,7 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): class DevicesRestTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, + devices.register_servlets, login.register_servlets, ] @@ -390,15 +391,63 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): self.assertEqual(0, channel.json_body["total"]) self.assertEqual(0, len(channel.json_body["devices"])) + @unittest.override_config( + {"experimental_features": {"msc2697_enabled": False, "msc3814_enabled": True}} + ) def test_get_devices(self) -> None: """ Tests that a normal lookup for devices is successfully """ # Create devices number_devices = 5 - for _ in range(number_devices): + # we create 2 fewer devices in the loop, because we will create another + # login after the loop, and we will create a dehydrated device + for _ in range(number_devices - 2): self.login("user", "pass") + other_user_token = self.login("user", "pass") + dehydrated_device_url = ( + "/_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device" + ) + content = { + "device_data": { + "algorithm": "m.dehydration.v1.olm", + }, + "device_id": "dehydrated_device", + "initial_device_display_name": "foo bar", + "device_keys": { + "user_id": "@user:test", + "device_id": "dehydrated_device", + "valid_until_ts": "80", + "algorithms": [ + "m.olm.curve25519-aes-sha2", + ], + "keys": { + ":": "", + }, + "signatures": { + "@user:test": {":": ""} + }, + }, + "fallback_keys": { + "alg1:device1": "f4llb4ckk3y", + "signed_:": { + "fallback": "true", + "key": "f4llb4ckk3y", + "signatures": { + "@user:test": {":": ""} + }, + }, + }, + "one_time_keys": {"alg1:k1": "0net1m3k3y"}, + } + self.make_request( + "PUT", + dehydrated_device_url, + access_token=other_user_token, + content=content, + ) + # Get devices channel = self.make_request( "GET", @@ -410,13 +459,22 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): self.assertEqual(number_devices, channel.json_body["total"]) self.assertEqual(number_devices, len(channel.json_body["devices"])) self.assertEqual(self.other_user, channel.json_body["devices"][0]["user_id"]) - # Check that all fields are available + # Check that all fields are available, and that the dehydrated device is marked as dehydrated + found_dehydrated = False for d in channel.json_body["devices"]: self.assertIn("user_id", d) self.assertIn("device_id", d) self.assertIn("display_name", d) self.assertIn("last_seen_ip", d) self.assertIn("last_seen_ts", d) + if d["device_id"] == "dehydrated_device": + self.assertTrue(d.get("dehydrated")) + found_dehydrated = True + else: + # Either the field is not present, or set to False + self.assertFalse(d.get("dehydrated")) + + self.assertTrue(found_dehydrated) class DeleteDevicesRestTestCase(unittest.HomeserverTestCase): diff --git a/tests/rest/admin/test_event_reports.py b/tests/rest/admin/test_event_reports.py index feb410a11d..a6f958658f 100644 --- a/tests/rest/admin/test_event_reports.py +++ b/tests/rest/admin/test_event_reports.py @@ -20,7 +20,7 @@ # from typing import List -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes @@ -378,6 +378,41 @@ class EventReportsTestCase(unittest.HomeserverTestCase): self.assertEqual(len(channel.json_body["event_reports"]), 1) self.assertNotIn("next_token", channel.json_body) + def test_filter_against_event_sender(self) -> None: + """ + Tests filtering by the sender of the reported event + """ + # first grab all the reports + channel = self.make_request( + "GET", + self.url, + access_token=self.admin_user_tok, + ) + self.assertEqual(channel.code, 200) + + # filter out set of report ids of events sent by one of the users + locally_filtered_report_ids = set() + for event_report in channel.json_body["event_reports"]: + if event_report["sender"] == self.other_user: + locally_filtered_report_ids.add(event_report["id"]) + + # grab the report ids by sender and compare to filtered report ids + channel = self.make_request( + "GET", + f"{self.url}?event_sender_user_id={self.other_user}", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + self.assertEqual(channel.json_body["total"], len(locally_filtered_report_ids)) + + event_reports = channel.json_body["event_reports"] + server_filtered_report_ids = set() + for event_report in event_reports: + server_filtered_report_ids.add(event_report["id"]) + self.assertIncludes( + locally_filtered_report_ids, server_filtered_report_ids, exact=True + ) + def _create_event_and_report(self, room_id: str, user_tok: str) -> None: """Create and report events""" resp = self.helper.send(room_id, tok=user_tok) diff --git a/tests/rest/admin/test_federation.py b/tests/rest/admin/test_federation.py index d5ae3345f5..cfea480bf0 100644 --- a/tests/rest/admin/test_federation.py +++ b/tests/rest/admin/test_federation.py @@ -22,7 +22,7 @@ from typing import List, Optional from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index 19c244cfcf..f863b5f8e7 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -24,7 +24,7 @@ from typing import Dict from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource import synapse.rest.admin @@ -35,7 +35,7 @@ from synapse.server import HomeServer from synapse.util import Clock from tests import unittest -from tests.test_utils import SMALL_PNG +from tests.test_utils import SMALL_CMYK_JPEG, SMALL_PNG from tests.unittest import override_config VALID_TIMESTAMP = 1609459200000 # 2021-01-01 in milliseconds @@ -598,23 +598,27 @@ class DeleteMediaByDateSizeTestCase(_AdminMediaTests): class QuarantineMediaByIDTestCase(_AdminMediaTests): + def upload_media_and_return_media_id(self, data: bytes) -> str: + # Upload some media into the room + response = self.helper.upload_media( + data, + tok=self.admin_user_tok, + expect_code=200, + ) + # Extract media ID from the response + server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' + return server_and_media_id.split("/")[1] + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastores().main self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") - - # Upload some media into the room - response = self.helper.upload_media( - SMALL_PNG, - tok=self.admin_user_tok, - expect_code=200, - ) - # Extract media ID from the response - server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' - self.media_id = server_and_media_id.split("/")[1] - + self.media_id = self.upload_media_and_return_media_id(SMALL_PNG) + self.media_id_2 = self.upload_media_and_return_media_id(SMALL_PNG) + self.media_id_3 = self.upload_media_and_return_media_id(SMALL_PNG) + self.media_id_other = self.upload_media_and_return_media_id(SMALL_CMYK_JPEG) self.url = "/_synapse/admin/v1/media/%s/%s/%s" @parameterized.expand(["quarantine", "unquarantine"]) @@ -686,6 +690,52 @@ class QuarantineMediaByIDTestCase(_AdminMediaTests): assert media_info is not None self.assertFalse(media_info.quarantined_by) + def test_quarantine_media_match_hash(self) -> None: + """ + Tests that quarantining removes all media with the same hash + """ + + media_info = self.get_success(self.store.get_local_media(self.media_id)) + assert media_info is not None + self.assertFalse(media_info.quarantined_by) + + # quarantining + channel = self.make_request( + "POST", + self.url % ("quarantine", self.server_name, self.media_id), + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertFalse(channel.json_body) + + # Test that ALL similar media was quarantined. + for media in [self.media_id, self.media_id_2, self.media_id_3]: + media_info = self.get_success(self.store.get_local_media(media)) + assert media_info is not None + self.assertTrue(media_info.quarantined_by) + + # Test that other media was not. + media_info = self.get_success(self.store.get_local_media(self.media_id_other)) + assert media_info is not None + self.assertFalse(media_info.quarantined_by) + + # remove from quarantine + channel = self.make_request( + "POST", + self.url % ("unquarantine", self.server_name, self.media_id), + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertFalse(channel.json_body) + + # Test that ALL similar media is now reset. + for media in [self.media_id, self.media_id_2, self.media_id_3]: + media_info = self.get_success(self.store.get_local_media(media)) + assert media_info is not None + self.assertFalse(media_info.quarantined_by) + def test_quarantine_protected_media(self) -> None: """ Tests that quarantining from protected media fails diff --git a/tests/rest/admin/test_registration_tokens.py b/tests/rest/admin/test_registration_tokens.py index 67d1db8ff8..b8e111c804 100644 --- a/tests/rest/admin/test_registration_tokens.py +++ b/tests/rest/admin/test_registration_tokens.py @@ -22,7 +22,7 @@ import random import string from typing import Optional -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 95ed736451..ee5d0419ab 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -28,17 +28,22 @@ from unittest.mock import AsyncMock, Mock from parameterized import parameterized from twisted.internet.task import deferLater -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EventTypes, Membership, RoomTypes from synapse.api.errors import Codes +from synapse.api.room_versions import RoomVersions from synapse.handlers.pagination import ( PURGE_ROOM_ACTION_NAME, SHUTDOWN_AND_PURGE_ROOM_ACTION_NAME, ) from synapse.rest.client import directory, events, knock, login, room, sync from synapse.server import HomeServer +from synapse.storage.databases.main.purge_events import ( + purge_room_tables_with_event_id_index, + purge_room_tables_with_room_id_column, +) from synapse.types import UserID from synapse.util import Clock from synapse.util.task_scheduler import TaskScheduler @@ -369,6 +374,47 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(200, channel.code, msg=channel.json_body) self._is_blocked(room_id) + def test_invited_users_not_joined_to_new_room(self) -> None: + """ + Test that when a new room id is provided, users who are only invited + but have not joined original room are not moved to new room. + """ + invitee = self.register_user("invitee", "pass") + + self.helper.invite( + self.room_id, self.other_user, invitee, tok=self.other_user_tok + ) + + # verify that user is invited + channel = self.make_request( + "GET", + f"/_matrix/client/v3/rooms/{self.room_id}/members?membership=invite", + access_token=self.other_user_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(len(channel.json_body["chunk"]), 1) + invite = channel.json_body["chunk"][0] + self.assertEqual(invite["state_key"], invitee) + + # shutdown room + channel = self.make_request( + "DELETE", + self.url, + {"new_room_user_id": self.admin_user}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(len(channel.json_body["kicked_users"]), 2) + + # joined member is moved to new room but invited user is not + users_in_room = self.get_success( + self.store.get_users_in_room(channel.json_body["new_room_id"]) + ) + self.assertNotIn(invitee, users_in_room) + self.assertIn(self.other_user, users_in_room) + self._is_purged(self.room_id) + self._has_no_members(self.room_id) + def test_shutdown_room_consent(self) -> None: """Test that we can shutdown rooms with local users who have not yet accepted the privacy policy. This used to fail when we tried to @@ -505,7 +551,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): def _is_purged(self, room_id: str) -> None: """Test that the following tables have been purged of all rows related to the room.""" - for table in PURGE_TABLES: + for table in purge_room_tables_with_room_id_column: count = self.get_success( self.store.db_pool.simple_select_one_onecol( table=table, @@ -514,7 +560,21 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): desc="test_purge_room", ) ) + self.assertEqual(count, 0, msg=f"Rows not purged in {table}") + for table in purge_room_tables_with_event_id_index: + rows = self.get_success( + self.store.db_pool.execute( + "find_event_count_for_table", + f""" + SELECT COUNT(*) FROM {table} WHERE event_id IN ( + SELECT event_id FROM events WHERE room_id=? + ) + """, + room_id, + ) + ) + count = rows[0][0] self.assertEqual(count, 0, msg=f"Rows not purged in {table}") def _assert_peek(self, room_id: str, expect_code: int) -> None: @@ -758,6 +818,8 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual(2, len(channel.json_body["results"])) self.assertEqual("complete", channel.json_body["results"][0]["status"]) self.assertEqual("complete", channel.json_body["results"][1]["status"]) + self.assertEqual(self.room_id, channel.json_body["results"][0]["room_id"]) + self.assertEqual(self.room_id, channel.json_body["results"][1]["room_id"]) delete_ids = {delete_id1, delete_id2} self.assertTrue(channel.json_body["results"][0]["delete_id"] in delete_ids) delete_ids.remove(channel.json_body["results"][0]["delete_id"]) @@ -777,6 +839,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual(1, len(channel.json_body["results"])) self.assertEqual("complete", channel.json_body["results"][0]["status"]) self.assertEqual(delete_id2, channel.json_body["results"][0]["delete_id"]) + self.assertEqual(self.room_id, channel.json_body["results"][0]["room_id"]) # get status after more than clearing time for all tasks self.reactor.advance(TaskScheduler.KEEP_TASKS_FOR_MS / 1000 / 2) @@ -1184,7 +1247,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): def _is_purged(self, room_id: str) -> None: """Test that the following tables have been purged of all rows related to the room.""" - for table in PURGE_TABLES: + for table in purge_room_tables_with_room_id_column: count = self.get_success( self.store.db_pool.simple_select_one_onecol( table=table, @@ -1193,7 +1256,21 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): desc="test_purge_room", ) ) + self.assertEqual(count, 0, msg=f"Rows not purged in {table}") + for table in purge_room_tables_with_event_id_index: + rows = self.get_success( + self.store.db_pool.execute( + "find_event_count_for_table", + f""" + SELECT COUNT(*) FROM {table} WHERE event_id IN ( + SELECT event_id FROM events WHERE room_id=? + ) + """, + room_id, + ) + ) + count = rows[0][0] self.assertEqual(count, 0, msg=f"Rows not purged in {table}") def _assert_peek(self, room_id: str, expect_code: int) -> None: @@ -1237,6 +1314,9 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual( delete_id, channel_room_id.json_body["results"][0]["delete_id"] ) + self.assertEqual( + self.room_id, channel_room_id.json_body["results"][0]["room_id"] + ) # get information by delete_id channel_delete_id = self.make_request( @@ -1249,6 +1329,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): channel_delete_id.code, msg=channel_delete_id.json_body, ) + self.assertEqual(self.room_id, channel_delete_id.json_body["room_id"]) # test values that are the same in both responses for content in [ @@ -1282,6 +1363,7 @@ class RoomTestCase(unittest.HomeserverTestCase): self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") + @unittest.override_config({"room_list_publication_rules": [{"action": "allow"}]}) def test_list_rooms(self) -> None: """Test that we can list rooms""" # Create 3 test rooms @@ -1311,7 +1393,7 @@ class RoomTestCase(unittest.HomeserverTestCase): # Check that response json body contains a "rooms" key self.assertTrue( "rooms" in channel.json_body, - msg="Response body does not " "contain a 'rooms' key", + msg="Response body does not contain a 'rooms' key", ) # Check that 3 rooms were returned @@ -1795,6 +1877,7 @@ class RoomTestCase(unittest.HomeserverTestCase): self.assertEqual(room_id, channel.json_body["rooms"][0].get("room_id")) self.assertEqual("ж", channel.json_body["rooms"][0].get("name")) + @unittest.override_config({"room_list_publication_rules": [{"action": "allow"}]}) def test_filter_public_rooms(self) -> None: self.helper.create_room_as( self.admin_user, tok=self.admin_user_tok, is_public=True @@ -1872,6 +1955,7 @@ class RoomTestCase(unittest.HomeserverTestCase): self.assertEqual(1, response.json_body["total_rooms"]) self.assertEqual(1, len(response.json_body["rooms"])) + @unittest.override_config({"room_list_publication_rules": [{"action": "allow"}]}) def test_single_room(self) -> None: """Test that a single room can be requested correctly""" # Create two test rooms @@ -2035,6 +2119,52 @@ class RoomTestCase(unittest.HomeserverTestCase): # the create_room already does the right thing, so no need to verify that we got # the state events it created. + def test_room_state_param(self) -> None: + """Test that filtering by state event type works when requesting state""" + room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/rooms/{room_id}/state?type=m.room.member", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + state = channel.json_body["state"] + # only one member has joined so there should be one membership event + self.assertEqual(1, len(state)) + event = state[0] + self.assertEqual(event["type"], "m.room.member") + self.assertEqual(event["state_key"], self.admin_user) + + def test_room_state_param_empty(self) -> None: + """Test that passing an empty string as state filter param returns no state events""" + room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/rooms/{room_id}/state?type=", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + state = channel.json_body["state"] + self.assertEqual(5, len(state)) + + def test_room_state_param_not_in_room(self) -> None: + """ + Test that passing a state filter param for a state event not in the room + returns no state events + """ + room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/rooms/{room_id}/state?type=m.room.custom", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + state = channel.json_body["state"] + self.assertEqual(0, len(state)) + def _set_canonical_alias( self, room_id: str, test_alias: str, admin_user_tok: str ) -> None: @@ -2114,7 +2244,7 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase): def test_topo_token_is_accepted(self) -> None: """Test Topo Token is accepted.""" - token = "t1-0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), @@ -2128,7 +2258,7 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase): def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: """Test that stream token is accepted for forward pagination.""" - token = "s0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), @@ -2795,6 +2925,63 @@ class MakeRoomAdminTestCase(unittest.HomeserverTestCase): "No local admin user in room with power to update power levels.", ) + def test_v12_room(self) -> None: + """Test that you can be promoted to admin in v12 rooms which won't have the admin the PL event.""" + room_id = self.helper.create_room_as( + self.creator, + tok=self.creator_tok, + room_version=RoomVersions.V12.identifier, + ) + + channel = self.make_request( + "POST", + f"/_synapse/admin/v1/rooms/{room_id}/make_room_admin", + content={}, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + + # Now we test that we can join the room and that the admin user has PL 100. + self.helper.join(room_id, self.admin_user, tok=self.admin_user_tok) + pl = self.helper.get_state( + room_id, EventTypes.PowerLevels, tok=self.creator_tok + ) + self.assertEquals(pl["users"][self.admin_user], 100) + + def test_v12_room_with_many_user_pls(self) -> None: + """Test that you can be promoted to the admin user's PL in v12 rooms that contain a range of user PLs.""" + room_id = self.helper.create_room_as( + self.creator, + tok=self.creator_tok, + room_version=RoomVersions.V12.identifier, + is_public=True, + extra_content={ + "power_level_content_override": { + "users": { + self.second_user_id: 50, + }, + }, + }, + ) + + self.helper.join(room_id, self.admin_user, tok=self.admin_user_tok) + self.helper.join(room_id, self.second_user_id, tok=self.second_tok) + + channel = self.make_request( + "POST", + f"/_synapse/admin/v1/rooms/{room_id}/make_room_admin", + content={}, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + + pl = self.helper.get_state( + room_id, EventTypes.PowerLevels, tok=self.creator_tok + ) + self.assertEquals(pl["users"][self.admin_user], 100) + class BlockRoomTestCase(unittest.HomeserverTestCase): servlets = [ @@ -3022,35 +3209,3 @@ class BlockRoomTestCase(unittest.HomeserverTestCase): """Block a room in database""" self.get_success(self._store.block_room(room_id, self.other_user)) self._is_blocked(room_id, expect=True) - - -PURGE_TABLES = [ - "current_state_events", - "event_backward_extremities", - "event_forward_extremities", - "event_json", - "event_push_actions", - "event_search", - "events", - "receipts_graph", - "receipts_linearized", - "room_aliases", - "room_depth", - "room_memberships", - "room_stats_state", - "room_stats_current", - "room_stats_earliest_token", - "rooms", - "stream_ordering_to_exterm", - "users_in_public_rooms", - "users_who_share_private_rooms", - "appservice_room_list", - "e2e_room_keys", - "event_push_summary", - "pusher_throttle", - "room_account_data", - "room_tags", - # "state_groups", # Current impl leaves orphaned state groups around. - "state_groups_state", - "federation_inbound_events_staging", -] diff --git a/tests/rest/admin/test_scheduled_tasks.py b/tests/rest/admin/test_scheduled_tasks.py new file mode 100644 index 0000000000..ea7afc0101 --- /dev/null +++ b/tests/rest/admin/test_scheduled_tasks.py @@ -0,0 +1,192 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +# +from typing import Mapping, Optional, Tuple + +from twisted.internet.testing import MemoryReactor + +import synapse.rest.admin +from synapse.api.errors import Codes +from synapse.rest.client import login +from synapse.server import HomeServer +from synapse.types import JsonMapping, ScheduledTask, TaskStatus +from synapse.util import Clock + +from tests import unittest + + +class ScheduledTasksAdminApiTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + self._task_scheduler = hs.get_task_scheduler() + + # create and schedule a few tasks + async def _test_task( + task: ScheduledTask, + ) -> Tuple[TaskStatus, Optional[JsonMapping], Optional[str]]: + return TaskStatus.ACTIVE, None, None + + async def _finished_test_task( + task: ScheduledTask, + ) -> Tuple[TaskStatus, Optional[JsonMapping], Optional[str]]: + return TaskStatus.COMPLETE, None, None + + async def _failed_test_task( + task: ScheduledTask, + ) -> Tuple[TaskStatus, Optional[JsonMapping], Optional[str]]: + return TaskStatus.FAILED, None, "Everything failed" + + self._task_scheduler.register_action(_test_task, "test_task") + self.get_success( + self._task_scheduler.schedule_task("test_task", resource_id="test") + ) + + self._task_scheduler.register_action(_finished_test_task, "finished_test_task") + self.get_success( + self._task_scheduler.schedule_task( + "finished_test_task", resource_id="finished_task" + ) + ) + + self._task_scheduler.register_action(_failed_test_task, "failed_test_task") + self.get_success( + self._task_scheduler.schedule_task( + "failed_test_task", resource_id="failed_task" + ) + ) + + def check_scheduled_tasks_response(self, scheduled_tasks: Mapping) -> list: + result = [] + for task in scheduled_tasks: + if task["resource_id"] == "test": + self.assertEqual(task["status"], TaskStatus.ACTIVE) + self.assertEqual(task["action"], "test_task") + result.append(task) + if task["resource_id"] == "finished_task": + self.assertEqual(task["status"], TaskStatus.COMPLETE) + self.assertEqual(task["action"], "finished_test_task") + result.append(task) + if task["resource_id"] == "failed_task": + self.assertEqual(task["status"], TaskStatus.FAILED) + self.assertEqual(task["action"], "failed_test_task") + result.append(task) + + return result + + def test_requester_is_not_admin(self) -> None: + """ + If the user is not a server admin, an error 403 is returned. + """ + + self.register_user("user", "pass", admin=False) + other_user_tok = self.login("user", "pass") + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks", + content={}, + access_token=other_user_tok, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_scheduled_tasks(self) -> None: + """ + Test that endpoint returns scheduled tasks. + """ + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + + # make sure we got back all the scheduled tasks + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + self.assertEqual(len(found_tasks), 3) + + def test_filtering_scheduled_tasks(self) -> None: + """ + Test that filtering the scheduled tasks response via query params works as expected. + """ + # filter via job_status + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?job_status=active", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + + # only the active task should have been returned + self.assertEqual(len(found_tasks), 1) + self.assertEqual(found_tasks[0]["status"], "active") + + # filter via action_name + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?action_name=test_task", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + + # only test_task should have been returned + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + self.assertEqual(len(found_tasks), 1) + self.assertEqual(found_tasks[0]["action"], "test_task") + + # filter via max_timestamp + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?max_timestamp=0", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + + # none should have been returned + self.assertEqual(len(found_tasks), 0) + + # filter via resource id + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?resource_id=failed_task", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + + # only the task with the matching resource id should have been returned + self.assertEqual(len(found_tasks), 1) + self.assertEqual(found_tasks[0]["resource_id"], "failed_task") diff --git a/tests/rest/admin/test_server_notice.py b/tests/rest/admin/test_server_notice.py index 150caeeee2..1f77e31d48 100644 --- a/tests/rest/admin/test_server_notice.py +++ b/tests/rest/admin/test_server_notice.py @@ -20,7 +20,7 @@ # from typing import List, Sequence -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes diff --git a/tests/rest/admin/test_statistics.py b/tests/rest/admin/test_statistics.py index 07ec49c4e5..10efc4ef8b 100644 --- a/tests/rest/admin/test_statistics.py +++ b/tests/rest/admin/test_statistics.py @@ -21,7 +21,7 @@ # from typing import Dict, List, Optional -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource import synapse.rest.admin diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 6d050e7784..4432b6a7a0 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -32,11 +32,17 @@ from unittest.mock import AsyncMock, Mock, patch from parameterized import parameterized, parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource import synapse.rest.admin -from synapse.api.constants import ApprovalNoticeMedium, EventTypes, LoginType, UserTypes +from synapse.api.constants import ( + ApprovalNoticeMedium, + EventContentFields, + EventTypes, + LoginType, + UserTypes, +) from synapse.api.errors import Codes, HttpResponseException, ResourceLimitError from synapse.api.room_versions import RoomVersions from synapse.media.filepath import MediaFilePaths @@ -60,6 +66,7 @@ from synapse.util import Clock from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.test_utils import SMALL_PNG +from tests.test_utils.event_injection import inject_event from tests.unittest import override_config @@ -321,6 +328,61 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Invalid user type", channel.json_body["error"]) + @override_config( + { + "user_types": { + "extra_user_types": ["extra1", "extra2"], + } + } + ) + def test_extra_user_type(self) -> None: + """ + Check that the extra user type can be used when registering a user. + """ + + def nonce_mac(user_type: str) -> tuple[str, str]: + """ + Get a nonce and the expected HMAC for that nonce. + """ + channel = self.make_request("GET", self.url) + nonce = channel.json_body["nonce"] + + want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) + want_mac.update( + nonce.encode("ascii") + + b"\x00alice\x00abc123\x00notadmin\x00" + + user_type.encode("ascii") + ) + want_mac_str = want_mac.hexdigest() + + return nonce, want_mac_str + + nonce, mac = nonce_mac("extra1") + # Valid user_type + body = { + "nonce": nonce, + "username": "alice", + "password": "abc123", + "user_type": "extra1", + "mac": mac, + } + channel = self.make_request("POST", self.url, body) + self.assertEqual(200, channel.code, msg=channel.json_body) + + nonce, mac = nonce_mac("extra3") + # Invalid user_type + body = { + "nonce": nonce, + "username": "alice", + "password": "abc123", + "user_type": "extra3", + "mac": mac, + } + channel = self.make_request("POST", self.url, body) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual("Invalid user type", channel.json_body["error"]) + def test_displayname(self) -> None: """ Test that displayname of new user is set @@ -1179,6 +1241,80 @@ class UsersListTestCase(unittest.HomeserverTestCase): not_user_types=["custom"], ) + @override_config( + { + "user_types": { + "extra_user_types": ["extra1", "extra2"], + } + } + ) + def test_filter_not_user_types_with_extra(self) -> None: + """Tests that the endpoint handles the not_user_types param when extra_user_types are configured""" + + regular_user_id = self.register_user("normalo", "secret") + + extra1_user_id = self.register_user("extra1", "secret") + self.make_request( + "PUT", + "/_synapse/admin/v2/users/" + urllib.parse.quote(extra1_user_id), + {"user_type": "extra1"}, + access_token=self.admin_user_tok, + ) + + def test_user_type( + expected_user_ids: List[str], not_user_types: Optional[List[str]] = None + ) -> None: + """Runs a test for the not_user_types param + Args: + expected_user_ids: Ids of the users that are expected to be returned + not_user_types: List of values for the not_user_types param + """ + + user_type_query = "" + + if not_user_types is not None: + user_type_query = "&".join( + [f"not_user_type={u}" for u in not_user_types] + ) + + test_url = f"{self.url}?{user_type_query}" + channel = self.make_request( + "GET", + test_url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code) + self.assertEqual(channel.json_body["total"], len(expected_user_ids)) + self.assertEqual( + expected_user_ids, + [u["name"] for u in channel.json_body["users"]], + ) + + # Request without user_types → all users expected + test_user_type([self.admin_user, extra1_user_id, regular_user_id]) + + # Request and exclude extra1 user type + test_user_type( + [self.admin_user, regular_user_id], + not_user_types=["extra1"], + ) + + # Request and exclude extra1 and extra2 user types + test_user_type( + [self.admin_user, regular_user_id], + not_user_types=["extra1", "extra2"], + ) + + # Request and exclude empty user types → only expected the extra1 user + test_user_type([extra1_user_id], not_user_types=[""]) + + # Request and exclude an unregistered type → expect all users + test_user_type( + [self.admin_user, extra1_user_id, regular_user_id], + not_user_types=["extra3"], + ) + def test_erasure_status(self) -> None: # Create a new user. user_id = self.register_user("eraseme", "eraseme") @@ -2710,6 +2846,16 @@ class UserRestTestCase(unittest.HomeserverTestCase): self.assertEqual(Codes.USER_LOCKED, channel.json_body["errcode"]) self.assertTrue(channel.json_body["soft_logout"]) + # User is not authorized to log in anymore + channel = self.make_request( + "POST", + "/_matrix/client/r0/login", + {"type": "m.login.password", "user": "user", "password": "pass"}, + ) + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.USER_LOCKED, channel.json_body["errcode"]) + self.assertTrue(channel.json_body["soft_logout"]) + @override_config({"user_directory": {"enabled": True, "search_all_users": True}}) def test_locked_user_not_in_user_dir(self) -> None: # User is available in the user dir @@ -2970,56 +3116,66 @@ class UserRestTestCase(unittest.HomeserverTestCase): self.assertEqual("@user:test", channel.json_body["name"]) self.assertTrue(channel.json_body["admin"]) + def set_user_type(self, user_type: Optional[str]) -> None: + # Set to user_type + channel = self.make_request( + "PUT", + self.url_other_user, + access_token=self.admin_user_tok, + content={"user_type": user_type}, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual("@user:test", channel.json_body["name"]) + self.assertEqual(user_type, channel.json_body["user_type"]) + + # Get user + channel = self.make_request( + "GET", + self.url_other_user, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual("@user:test", channel.json_body["name"]) + self.assertEqual(user_type, channel.json_body["user_type"]) + def test_set_user_type(self) -> None: """ Test changing user type. """ # Set to support type - channel = self.make_request( - "PUT", - self.url_other_user, - access_token=self.admin_user_tok, - content={"user_type": UserTypes.SUPPORT}, - ) - - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual("@user:test", channel.json_body["name"]) - self.assertEqual(UserTypes.SUPPORT, channel.json_body["user_type"]) - - # Get user - channel = self.make_request( - "GET", - self.url_other_user, - access_token=self.admin_user_tok, - ) - - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual("@user:test", channel.json_body["name"]) - self.assertEqual(UserTypes.SUPPORT, channel.json_body["user_type"]) + self.set_user_type(UserTypes.SUPPORT) # Change back to a regular user + self.set_user_type(None) + + @override_config({"user_types": {"extra_user_types": ["extra1", "extra2"]}}) + def test_set_user_type_with_extras(self) -> None: + """ + Test changing user type with extra_user_types configured. + """ + + # Check that we can still set to support type + self.set_user_type(UserTypes.SUPPORT) + + # Check that we can set to an extra user type + self.set_user_type("extra2") + + # Change back to a regular user + self.set_user_type(None) + + # Try setting to invalid type channel = self.make_request( "PUT", self.url_other_user, access_token=self.admin_user_tok, - content={"user_type": None}, + content={"user_type": "extra3"}, ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual("@user:test", channel.json_body["name"]) - self.assertIsNone(channel.json_body["user_type"]) - - # Get user - channel = self.make_request( - "GET", - self.url_other_user, - access_token=self.admin_user_tok, - ) - - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual("@user:test", channel.json_body["name"]) - self.assertIsNone(channel.json_body["user_type"]) + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual("Invalid user type", channel.json_body["error"]) def test_accidental_deactivation_prevention(self) -> None: """ @@ -3222,6 +3378,7 @@ class UserRestTestCase(unittest.HomeserverTestCase): self.assertIn("consent_ts", content) self.assertIn("external_ids", content) self.assertIn("last_seen_ts", content) + self.assertIn("suspended", content) # This key was removed intentionally. Ensure it is not accidentally re-included. self.assertNotIn("password_hash", content) @@ -3893,9 +4050,7 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): image_data1 = SMALL_PNG # Resolution: 1×1, MIME type: image/gif, Extension: gif, Size: 35 B image_data2 = unhexlify( - b"47494638376101000100800100000000" - b"ffffff2c00000000010001000002024c" - b"01003b" + b"47494638376101000100800100000000ffffff2c00000000010001000002024c01003b" ) # Resolution: 1×1, MIME type: image/bmp, Extension: bmp, Size: 54 B image_data3 = unhexlify( @@ -5030,7 +5185,6 @@ class UserSuspensionTestCase(unittest.HomeserverTestCase): self.store = hs.get_datastores().main - @override_config({"experimental_features": {"msc3823_account_suspension": True}}) def test_suspend_user(self) -> None: # test that suspending user works channel = self.make_request( @@ -5408,6 +5562,159 @@ class UserRedactionTestCase(unittest.HomeserverTestCase): # we redacted 6 messages self.assertEqual(len(matches), 6) + def test_redactions_for_remote_user_succeed_with_admin_priv_in_room(self) -> None: + """ + Test that if the admin requester has privileges in a room, redaction requests + succeed for a remote user + """ + + # inject some messages from remote user and collect event ids + original_message_ids = [] + for i in range(5): + event = self.get_success( + inject_event( + self.hs, + room_id=self.rm1, + type="m.room.message", + sender="@remote:remote_server", + content={"msgtype": "m.text", "body": f"nefarious_chatter{i}"}, + ) + ) + original_message_ids.append(event.event_id) + + # send a request to redact a remote user's messages in a room. + # the server admin created this room and has admin privilege in room + channel = self.make_request( + "POST", + "/_synapse/admin/v1/user/@remote:remote_server/redact", + content={"rooms": [self.rm1]}, + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + id = channel.json_body.get("redact_id") + + # check that there were no failed redactions + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/user/redact_status/{id}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body.get("status"), "complete") + failed_redactions = channel.json_body.get("failed_redactions") + self.assertEqual(failed_redactions, {}) + + filter = json.dumps({"types": [EventTypes.Redaction]}) + channel = self.make_request( + "GET", + f"rooms/{self.rm1}/messages?filter={filter}&limit=50", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + for event in channel.json_body["chunk"]: + for event_id in original_message_ids: + if event["type"] == "m.room.redaction" and event["redacts"] == event_id: + original_message_ids.remove(event_id) + break + # we originally sent 5 messages so 5 should be redacted + self.assertEqual(len(original_message_ids), 0) + + def test_redact_redacts_encrypted_messages(self) -> None: + """ + Test that user's encrypted messages are redacted + """ + encrypted_room = self.helper.create_room_as( + self.admin, tok=self.admin_tok, room_version="7" + ) + self.helper.send_state( + encrypted_room, + EventTypes.RoomEncryption, + {EventContentFields.ENCRYPTION_ALGORITHM: "m.megolm.v1.aes-sha2"}, + tok=self.admin_tok, + ) + # join room send some messages + originals = [] + join = self.helper.join(encrypted_room, self.bad_user, tok=self.bad_user_tok) + originals.append(join["event_id"]) + for _ in range(15): + res = self.helper.send_event( + encrypted_room, "m.room.encrypted", {}, tok=self.bad_user_tok + ) + originals.append(res["event_id"]) + + # redact user's events + channel = self.make_request( + "POST", + f"/_synapse/admin/v1/user/{self.bad_user}/redact", + content={"rooms": []}, + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + matched = [] + filter = json.dumps({"types": [EventTypes.Redaction]}) + channel = self.make_request( + "GET", + f"rooms/{encrypted_room}/messages?filter={filter}&limit=50", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + for event in channel.json_body["chunk"]: + for event_id in originals: + if event["type"] == "m.room.redaction" and event["redacts"] == event_id: + matched.append(event_id) + self.assertEqual(len(matched), len(originals)) + + def test_use_admin_param_for_redactions(self) -> None: + """ + Test that if the `use_admin` param is set to true, the admin user is used to issue + the redactions and that they succeed in a room where the admin user has sufficient + power to issue redactions + """ + + originals = [] + join = self.helper.join(self.rm1, self.bad_user, tok=self.bad_user_tok) + originals.append(join["event_id"]) + for i in range(15): + event = {"body": f"hello{i}", "msgtype": "m.text"} + res = self.helper.send_event( + self.rm1, "m.room.message", event, tok=self.bad_user_tok + ) + originals.append(res["event_id"]) + + # redact messages + channel = self.make_request( + "POST", + f"/_synapse/admin/v1/user/{self.bad_user}/redact", + content={"rooms": [self.rm1], "use_admin": True}, + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + # messages are redacted, and redactions are issued by the admin user + filter = json.dumps({"types": [EventTypes.Redaction]}) + channel = self.make_request( + "GET", + f"rooms/{self.rm1}/messages?filter={filter}&limit=50", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + matches = [] + for event in channel.json_body["chunk"]: + for event_id in originals: + if event["type"] == "m.room.redaction" and event["redacts"] == event_id: + matches.append((event_id, event)) + # we redacted 16 messages + self.assertEqual(len(matches), 16) + + for redaction_tuple in matches: + redaction = redaction_tuple[1] + if redaction["sender"] != self.admin: + self.fail("Redaction was not issued by admin account") + class UserRedactionBackgroundTaskTestCase(BaseMultiWorkerStreamTestCase): servlets = [ @@ -5502,3 +5809,254 @@ class UserRedactionBackgroundTaskTestCase(BaseMultiWorkerStreamTestCase): redaction_ids.add(event["redacts"]) self.assertIncludes(redaction_ids, original_event_ids, exact=True) + + +class GetInvitesFromUserTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + admin.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin = self.register_user("thomas", "pass", True) + self.admin_tok = self.login("thomas", "pass") + + self.bad_user = self.register_user("teresa", "pass") + self.bad_user_tok = self.login("teresa", "pass") + + self.random_users = [] + for i in range(4): + self.random_users.append(self.register_user(f"user{i}", f"pass{i}")) + + self.room1 = self.helper.create_room_as(self.bad_user, tok=self.bad_user_tok) + self.room2 = self.helper.create_room_as(self.bad_user, tok=self.bad_user_tok) + self.room3 = self.helper.create_room_as(self.bad_user, tok=self.bad_user_tok) + + @unittest.override_config( + {"rc_invites": {"per_issuer": {"per_second": 1000, "burst_count": 1000}}} + ) + def test_get_user_invite_count_new_invites_test_case(self) -> None: + """ + Test that new invites that arrive after a provided timestamp are counted + """ + # grab a current timestamp + before_invites_sent_ts = self.hs.get_clock().time_msec() + + # bad user sends some invites + for room_id in [self.room1, self.room2]: + for user in self.random_users: + self.helper.invite(room_id, self.bad_user, user, tok=self.bad_user_tok) + + # fetch using timestamp, all should be returned + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={before_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 8) + + # send some more invites, they should show up in addition to original 8 using same timestamp + for user in self.random_users: + self.helper.invite( + self.room3, src=self.bad_user, targ=user, tok=self.bad_user_tok + ) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={before_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 12) + + def test_get_user_invite_count_invites_before_ts_test_case(self) -> None: + """ + Test that invites sent before provided ts are not counted + """ + # bad user sends some invites + for room_id in [self.room1, self.room2]: + for user in self.random_users: + self.helper.invite(room_id, self.bad_user, user, tok=self.bad_user_tok) + + # add a msec between last invite and ts + after_invites_sent_ts = self.hs.get_clock().time_msec() + 1 + + # fetch invites with timestamp, none should be returned + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={after_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 0) + + def test_user_invite_count_kick_ban_not_counted(self) -> None: + """ + Test that kicks and bans are not counted in invite count + """ + to_kick_user_id = self.register_user("kick_me", "pass") + to_kick_tok = self.login("kick_me", "pass") + + self.helper.join(self.room1, to_kick_user_id, tok=to_kick_tok) + + # grab a current timestamp + before_invites_sent_ts = self.hs.get_clock().time_msec() + + # bad user sends some invites (8) + for room_id in [self.room1, self.room2]: + for user in self.random_users: + self.helper.invite( + room_id, src=self.bad_user, targ=user, tok=self.bad_user_tok + ) + + # fetch using timestamp, all invites sent should be counted + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={before_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 8) + + # send a kick and some bans and make sure these aren't counted against invite total + for user in self.random_users: + self.helper.ban( + self.room1, src=self.bad_user, targ=user, tok=self.bad_user_tok + ) + + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{self.room1}/kick", + content={"user_id": to_kick_user_id}, + access_token=self.bad_user_tok, + ) + self.assertEqual(channel.code, 200) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={before_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 8) + + +class GetCumulativeJoinedRoomCountForUserTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + admin.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin = self.register_user("thomas", "pass", True) + self.admin_tok = self.login("thomas", "pass") + + self.bad_user = self.register_user("teresa", "pass") + self.bad_user_tok = self.login("teresa", "pass") + + def test_user_cumulative_joined_room_count(self) -> None: + """ + Tests proper count returned from /cumulative_joined_room_count endpoint + """ + # Create rooms and join, grab timestamp before room creation + before_room_creation_timestamp = self.hs.get_clock().time_msec() + + joined_rooms = [] + for _ in range(3): + room = self.helper.create_room_as(self.admin, tok=self.admin_tok) + self.helper.join( + room, user=self.bad_user, expect_code=200, tok=self.bad_user_tok + ) + joined_rooms.append(room) + + # get a timestamp after room creation and join, add a msec between last join and ts + after_room_creation = self.hs.get_clock().time_msec() + 1 + + # Get rooms using this timestamp, there should be none since all rooms were created and joined + # before provided timestamp + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/cumulative_joined_room_count?from_ts={int(after_room_creation)}", + access_token=self.admin_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(0, channel.json_body["cumulative_joined_room_count"]) + + # fetch rooms with the older timestamp before they were created and joined, this should + # return the rooms + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/cumulative_joined_room_count?from_ts={int(before_room_creation_timestamp)}", + access_token=self.admin_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual( + len(joined_rooms), channel.json_body["cumulative_joined_room_count"] + ) + + def test_user_joined_room_count_includes_left_and_banned_rooms(self) -> None: + """ + Tests proper count returned from /joined_room_count endpoint when user has left + or been banned from joined rooms + """ + # Create rooms and join, grab timestamp before room creation + before_room_creation_timestamp = self.hs.get_clock().time_msec() + + joined_rooms = [] + for _ in range(3): + room = self.helper.create_room_as(self.admin, tok=self.admin_tok) + self.helper.join( + room, user=self.bad_user, expect_code=200, tok=self.bad_user_tok + ) + joined_rooms.append(room) + + # fetch rooms with the older timestamp before they were created and joined + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/cumulative_joined_room_count?from_ts={int(before_room_creation_timestamp)}", + access_token=self.admin_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual( + len(joined_rooms), channel.json_body["cumulative_joined_room_count"] + ) + + # have the user banned from/leave the joined rooms + self.helper.ban( + joined_rooms[0], + src=self.admin, + targ=self.bad_user, + expect_code=200, + tok=self.admin_tok, + ) + self.helper.change_membership( + joined_rooms[1], + src=self.bad_user, + targ=self.bad_user, + membership="leave", + expect_code=200, + tok=self.bad_user_tok, + ) + self.helper.ban( + joined_rooms[2], + src=self.admin, + targ=self.bad_user, + expect_code=200, + tok=self.admin_tok, + ) + + # fetch the joined room count again, the number should remain the same as the collected joined rooms + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/cumulative_joined_room_count?from_ts={int(before_room_creation_timestamp)}", + access_token=self.admin_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual( + len(joined_rooms), channel.json_body["cumulative_joined_room_count"] + ) diff --git a/tests/rest/admin/test_username_available.py b/tests/rest/admin/test_username_available.py index 4dd5de33d3..9c3ab3e64c 100644 --- a/tests/rest/admin/test_username_available.py +++ b/tests/rest/admin/test_username_available.py @@ -20,7 +20,7 @@ # from typing import Optional -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes, SynapseError diff --git a/tests/rest/client/sliding_sync/test_connection_tracking.py b/tests/rest/client/sliding_sync/test_connection_tracking.py index 5b819103c2..f8ce1104a8 100644 --- a/tests/rest/client/sliding_sync/test_connection_tracking.py +++ b/tests/rest/client/sliding_sync/test_connection_tracking.py @@ -15,7 +15,7 @@ import logging from parameterized import parameterized, parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EventTypes diff --git a/tests/rest/client/sliding_sync/test_extension_account_data.py b/tests/rest/client/sliding_sync/test_extension_account_data.py index 799fbb1856..5949065722 100644 --- a/tests/rest/client/sliding_sync/test_extension_account_data.py +++ b/tests/rest/client/sliding_sync/test_extension_account_data.py @@ -17,7 +17,7 @@ import logging from parameterized import parameterized, parameterized_class from typing_extensions import assert_never -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import AccountDataTypes diff --git a/tests/rest/client/sliding_sync/test_extension_e2ee.py b/tests/rest/client/sliding_sync/test_extension_e2ee.py index 7ce6592d8f..baf6a5882e 100644 --- a/tests/rest/client/sliding_sync/test_extension_e2ee.py +++ b/tests/rest/client/sliding_sync/test_extension_e2ee.py @@ -15,7 +15,7 @@ import logging from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.rest.client import devices, login, room, sync diff --git a/tests/rest/client/sliding_sync/test_extension_receipts.py b/tests/rest/client/sliding_sync/test_extension_receipts.py index 6e7700b533..1bba3038db 100644 --- a/tests/rest/client/sliding_sync/test_extension_receipts.py +++ b/tests/rest/client/sliding_sync/test_extension_receipts.py @@ -15,7 +15,7 @@ import logging from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EduTypes, ReceiptTypes diff --git a/tests/rest/client/sliding_sync/test_extension_thread_subscriptions.py b/tests/rest/client/sliding_sync/test_extension_thread_subscriptions.py new file mode 100644 index 0000000000..775c4f96c9 --- /dev/null +++ b/tests/rest/client/sliding_sync/test_extension_thread_subscriptions.py @@ -0,0 +1,497 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import logging +from http import HTTPStatus +from typing import List, Optional, Tuple, cast + +from twisted.test.proto_helpers import MemoryReactor + +import synapse.rest.admin +from synapse.rest.client import login, room, sync, thread_subscriptions +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock + +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase + +logger = logging.getLogger(__name__) + + +# The name of the extension. Currently unstable-prefixed. +EXT_NAME = "io.element.msc4308.thread_subscriptions" + + +class SlidingSyncThreadSubscriptionsExtensionTestCase(SlidingSyncBase): + """ + Test the thread subscriptions extension in the Sliding Sync API. + """ + + maxDiff = None + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + thread_subscriptions.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = {"msc4306_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + super().prepare(reactor, clock, hs) + + def test_no_data_initial_sync(self) -> None: + """ + Test enabling thread subscriptions extension during initial sync with no data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + sync_body = { + "lists": {}, + "extensions": { + EXT_NAME: { + "enabled": True, + } + }, + } + + # Sync + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Assert + self.assertNotIn(EXT_NAME, response_body["extensions"]) + + def test_no_data_incremental_sync(self) -> None: + """ + Test enabling thread subscriptions extension during incremental sync with no data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + initial_sync_body: JsonDict = { + "lists": {}, + } + + # Initial sync + response_body, sync_pos = self.do_sync(initial_sync_body, tok=user1_tok) + + # Incremental sync with extension enabled + sync_body = { + "lists": {}, + "extensions": { + EXT_NAME: { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok, since=sync_pos) + + # Assert + self.assertNotIn( + EXT_NAME, + response_body["extensions"], + response_body, + ) + + def test_thread_subscription_initial_sync(self) -> None: + """ + Test thread subscriptions appear in initial sync response. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + thread_root_resp = self.helper.send(room_id, body="Thread root", tok=user1_tok) + thread_root_id = thread_root_resp["event_id"] + + # get the baseline stream_id of the thread_subscriptions stream + # before we write any data. + # Required because the initial value differs between SQLite and Postgres. + base = self.store.get_max_thread_subscriptions_stream_id() + + self._subscribe_to_thread(user1_id, room_id, thread_root_id) + sync_body = { + "lists": {}, + "extensions": { + EXT_NAME: { + "enabled": True, + } + }, + } + + # Sync + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Assert + self.assertEqual( + response_body["extensions"][EXT_NAME], + { + "subscribed": { + room_id: { + thread_root_id: { + "automatic": False, + "bump_stamp": base + 1, + } + } + } + }, + ) + + def test_thread_subscription_incremental_sync(self) -> None: + """ + Test new thread subscriptions appear in incremental sync response. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + sync_body = { + "lists": {}, + "extensions": { + EXT_NAME: { + "enabled": True, + } + }, + } + thread_root_resp = self.helper.send(room_id, body="Thread root", tok=user1_tok) + thread_root_id = thread_root_resp["event_id"] + + # get the baseline stream_id of the thread_subscriptions stream + # before we write any data. + # Required because the initial value differs between SQLite and Postgres. + base = self.store.get_max_thread_subscriptions_stream_id() + + # Initial sync + _, sync_pos = self.do_sync(sync_body, tok=user1_tok) + logger.info("Synced to: %r, now subscribing to thread", sync_pos) + + # Subscribe + self._subscribe_to_thread(user1_id, room_id, thread_root_id) + + # Incremental sync + response_body, sync_pos = self.do_sync(sync_body, tok=user1_tok, since=sync_pos) + logger.info("Synced to: %r", sync_pos) + + # Assert + self.assertEqual( + response_body["extensions"][EXT_NAME], + { + "subscribed": { + room_id: { + thread_root_id: { + "automatic": False, + "bump_stamp": base + 1, + } + } + } + }, + ) + + def test_unsubscribe_from_thread(self) -> None: + """ + Test unsubscribing from a thread. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + thread_root_resp = self.helper.send(room_id, body="Thread root", tok=user1_tok) + thread_root_id = thread_root_resp["event_id"] + + # get the baseline stream_id of the thread_subscriptions stream + # before we write any data. + # Required because the initial value differs between SQLite and Postgres. + base = self.store.get_max_thread_subscriptions_stream_id() + + self._subscribe_to_thread(user1_id, room_id, thread_root_id) + sync_body = { + "lists": {}, + "extensions": { + EXT_NAME: { + "enabled": True, + } + }, + } + + response_body, sync_pos = self.do_sync(sync_body, tok=user1_tok) + + # Assert: Subscription present + self.assertIn(EXT_NAME, response_body["extensions"]) + self.assertEqual( + response_body["extensions"][EXT_NAME], + { + "subscribed": { + room_id: { + thread_root_id: {"automatic": False, "bump_stamp": base + 1} + } + } + }, + ) + + # Unsubscribe + self._unsubscribe_from_thread(user1_id, room_id, thread_root_id) + + # Incremental sync + response_body, sync_pos = self.do_sync(sync_body, tok=user1_tok, since=sync_pos) + + # Assert: Unsubscription present + self.assertEqual( + response_body["extensions"][EXT_NAME], + {"unsubscribed": {room_id: {thread_root_id: {"bump_stamp": base + 2}}}}, + ) + + def test_multiple_thread_subscriptions(self) -> None: + """ + Test handling of multiple thread subscriptions. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + + # Create thread roots + thread_root_resp1 = self.helper.send( + room_id, body="Thread root 1", tok=user1_tok + ) + thread_root_id1 = thread_root_resp1["event_id"] + thread_root_resp2 = self.helper.send( + room_id, body="Thread root 2", tok=user1_tok + ) + thread_root_id2 = thread_root_resp2["event_id"] + thread_root_resp3 = self.helper.send( + room_id, body="Thread root 3", tok=user1_tok + ) + thread_root_id3 = thread_root_resp3["event_id"] + + # get the baseline stream_id of the thread_subscriptions stream + # before we write any data. + # Required because the initial value differs between SQLite and Postgres. + base = self.store.get_max_thread_subscriptions_stream_id() + + # Subscribe to threads + self._subscribe_to_thread(user1_id, room_id, thread_root_id1) + self._subscribe_to_thread(user1_id, room_id, thread_root_id2) + self._subscribe_to_thread(user1_id, room_id, thread_root_id3) + + sync_body = { + "lists": {}, + "extensions": { + EXT_NAME: { + "enabled": True, + } + }, + } + + # Sync + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Assert + self.assertEqual( + response_body["extensions"][EXT_NAME], + { + "subscribed": { + room_id: { + thread_root_id1: { + "automatic": False, + "bump_stamp": base + 1, + }, + thread_root_id2: { + "automatic": False, + "bump_stamp": base + 2, + }, + thread_root_id3: { + "automatic": False, + "bump_stamp": base + 3, + }, + } + } + }, + ) + + def test_limit_parameter(self) -> None: + """ + Test limit parameter in thread subscriptions extension. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + + # Create 5 thread roots and subscribe to each + thread_root_ids = [] + for i in range(5): + thread_root_resp = self.helper.send( + room_id, body=f"Thread root {i}", tok=user1_tok + ) + thread_root_ids.append(thread_root_resp["event_id"]) + self._subscribe_to_thread(user1_id, room_id, thread_root_ids[-1]) + + sync_body = { + "lists": {}, + "extensions": {EXT_NAME: {"enabled": True, "limit": 3}}, + } + + # Sync + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Assert + thread_subscriptions = response_body["extensions"][EXT_NAME] + self.assertEqual( + len(thread_subscriptions["subscribed"][room_id]), 3, thread_subscriptions + ) + + def test_limit_and_companion_backpagination(self) -> None: + """ + Create 1 thread subscription, do a sync, create 4 more, + then sync with a limit of 2 and fill in the gap + using the companion /thread_subscriptions endpoint. + """ + + thread_root_ids: List[str] = [] + + def make_subscription() -> None: + thread_root_resp = self.helper.send( + room_id, body="Some thread root", tok=user1_tok + ) + thread_root_ids.append(thread_root_resp["event_id"]) + self._subscribe_to_thread(user1_id, room_id, thread_root_ids[-1]) + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + + # get the baseline stream_id of the thread_subscriptions stream + # before we write any data. + # Required because the initial value differs between SQLite and Postgres. + base = self.store.get_max_thread_subscriptions_stream_id() + + # Make our first subscription + make_subscription() + + # Sync for the first time + sync_body = { + "lists": {}, + "extensions": {EXT_NAME: {"enabled": True, "limit": 2}}, + } + + sync_resp, first_sync_pos = self.do_sync(sync_body, tok=user1_tok) + + thread_subscriptions = sync_resp["extensions"][EXT_NAME] + self.assertEqual( + thread_subscriptions["subscribed"], + { + room_id: { + thread_root_ids[0]: {"automatic": False, "bump_stamp": base + 1}, + } + }, + ) + + # Get our pos for the next sync + first_sync_pos = sync_resp["pos"] + + # Create 5 more thread subscriptions and subscribe to each + for _ in range(5): + make_subscription() + + # Now sync again. Our limit is 2, + # so we should get the latest 2 subscriptions, + # with a gap of 3 more subscriptions in the middle + sync_resp, _pos = self.do_sync(sync_body, tok=user1_tok, since=first_sync_pos) + + thread_subscriptions = sync_resp["extensions"][EXT_NAME] + self.assertEqual( + thread_subscriptions["subscribed"], + { + room_id: { + thread_root_ids[4]: {"automatic": False, "bump_stamp": base + 5}, + thread_root_ids[5]: {"automatic": False, "bump_stamp": base + 6}, + } + }, + ) + # 1st backpagination: expecting a page with 2 subscriptions + page, end_tok = self._do_backpaginate( + from_tok=thread_subscriptions["prev_batch"], + to_tok=first_sync_pos, + limit=2, + access_token=user1_tok, + ) + self.assertIsNotNone(end_tok, "backpagination should continue") + self.assertEqual( + page["subscribed"], + { + room_id: { + thread_root_ids[2]: {"automatic": False, "bump_stamp": base + 3}, + thread_root_ids[3]: {"automatic": False, "bump_stamp": base + 4}, + } + }, + ) + + # 2nd backpagination: expecting a page with only 1 subscription + # and no other token for further backpagination + assert end_tok is not None + page, end_tok = self._do_backpaginate( + from_tok=end_tok, to_tok=first_sync_pos, limit=2, access_token=user1_tok + ) + self.assertIsNone(end_tok, "backpagination should have finished") + self.assertEqual( + page["subscribed"], + { + room_id: { + thread_root_ids[1]: {"automatic": False, "bump_stamp": base + 2}, + } + }, + ) + + def _do_backpaginate( + self, *, from_tok: str, to_tok: str, limit: int, access_token: str + ) -> Tuple[JsonDict, Optional[str]]: + channel = self.make_request( + "GET", + "/_matrix/client/unstable/io.element.msc4308/thread_subscriptions" + f"?from={from_tok}&to={to_tok}&limit={limit}&dir=b", + access_token=access_token, + ) + + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + body = channel.json_body + return body, cast(Optional[str], body.get("end")) + + def _subscribe_to_thread( + self, user_id: str, room_id: str, thread_root_id: str + ) -> None: + """ + Helper method to subscribe a user to a thread. + """ + self.get_success( + self.store.subscribe_user_to_thread( + user_id=user_id, + room_id=room_id, + thread_root_event_id=thread_root_id, + automatic_event_orderings=None, + ) + ) + + def _unsubscribe_from_thread( + self, user_id: str, room_id: str, thread_root_id: str + ) -> None: + """ + Helper method to unsubscribe a user from a thread. + """ + self.get_success( + self.store.unsubscribe_user_from_thread( + user_id=user_id, + room_id=room_id, + thread_root_event_id=thread_root_id, + ) + ) diff --git a/tests/rest/client/sliding_sync/test_extension_to_device.py b/tests/rest/client/sliding_sync/test_extension_to_device.py index 790abb739d..151a5be665 100644 --- a/tests/rest/client/sliding_sync/test_extension_to_device.py +++ b/tests/rest/client/sliding_sync/test_extension_to_device.py @@ -16,7 +16,7 @@ from typing import List from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.rest.client import login, sendtodevice, sync diff --git a/tests/rest/client/sliding_sync/test_extension_typing.py b/tests/rest/client/sliding_sync/test_extension_typing.py index f87c3c8b17..37c90d6ec2 100644 --- a/tests/rest/client/sliding_sync/test_extension_typing.py +++ b/tests/rest/client/sliding_sync/test_extension_typing.py @@ -15,7 +15,7 @@ import logging from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EduTypes diff --git a/tests/rest/client/sliding_sync/test_extensions.py b/tests/rest/client/sliding_sync/test_extensions.py index 30230e5c4b..0643596e59 100644 --- a/tests/rest/client/sliding_sync/test_extensions.py +++ b/tests/rest/client/sliding_sync/test_extensions.py @@ -17,7 +17,7 @@ from typing import Literal from parameterized import parameterized, parameterized_class from typing_extensions import assert_never -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import ReceiptTypes diff --git a/tests/rest/client/sliding_sync/test_lists_filters.py b/tests/rest/client/sliding_sync/test_lists_filters.py index c59f6aedc4..57d00a2a7a 100644 --- a/tests/rest/client/sliding_sync/test_lists_filters.py +++ b/tests/rest/client/sliding_sync/test_lists_filters.py @@ -15,7 +15,7 @@ import logging from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import ( diff --git a/tests/rest/client/sliding_sync/test_room_subscriptions.py b/tests/rest/client/sliding_sync/test_room_subscriptions.py index 285fdaaf78..b78e4f2045 100644 --- a/tests/rest/client/sliding_sync/test_room_subscriptions.py +++ b/tests/rest/client/sliding_sync/test_room_subscriptions.py @@ -16,7 +16,7 @@ from http import HTTPStatus from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EventTypes, HistoryVisibility diff --git a/tests/rest/client/sliding_sync/test_rooms_invites.py b/tests/rest/client/sliding_sync/test_rooms_invites.py index 882762ca29..a0f4ccd2cc 100644 --- a/tests/rest/client/sliding_sync/test_rooms_invites.py +++ b/tests/rest/client/sliding_sync/test_rooms_invites.py @@ -15,7 +15,7 @@ import logging from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EventTypes, HistoryVisibility diff --git a/tests/rest/client/sliding_sync/test_rooms_meta.py b/tests/rest/client/sliding_sync/test_rooms_meta.py index 0a8b2c02c2..4559bc7646 100644 --- a/tests/rest/client/sliding_sync/test_rooms_meta.py +++ b/tests/rest/client/sliding_sync/test_rooms_meta.py @@ -15,7 +15,7 @@ import logging from parameterized import parameterized, parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EventContentFields, EventTypes, Membership diff --git a/tests/rest/client/sliding_sync/test_rooms_required_state.py b/tests/rest/client/sliding_sync/test_rooms_required_state.py index ecea5f2d5b..cfff167c6e 100644 --- a/tests/rest/client/sliding_sync/test_rooms_required_state.py +++ b/tests/rest/client/sliding_sync/test_rooms_required_state.py @@ -11,16 +11,17 @@ # See the GNU Affero General Public License for more details: # . # +import enum import logging from parameterized import parameterized, parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventContentFields, EventTypes, JoinRules, Membership from synapse.handlers.sliding_sync import StateValues -from synapse.rest.client import login, room, sync +from synapse.rest.client import knock, login, room, sync from synapse.server import HomeServer from synapse.util import Clock @@ -30,6 +31,17 @@ from tests.test_utils.event_injection import mark_event_as_partial_state logger = logging.getLogger(__name__) +# Inherit from `str` so that they show up in the test description when we +# `@parameterized.expand(...)` the first parameter +class MembershipAction(str, enum.Enum): + INVITE = "invite" + JOIN = "join" + KNOCK = "knock" + LEAVE = "leave" + BAN = "ban" + KICK = "kick" + + # FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the # foreground update for # `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` (tracked by @@ -52,6 +64,7 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, + knock.register_servlets, room.register_servlets, sync.register_servlets, ] @@ -496,6 +509,153 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): ) self.assertIsNone(response_body["rooms"][room_id1].get("invite_state")) + @parameterized.expand( + [ + (MembershipAction.LEAVE,), + (MembershipAction.INVITE,), + (MembershipAction.KNOCK,), + (MembershipAction.JOIN,), + (MembershipAction.BAN,), + (MembershipAction.KICK,), + ] + ) + def test_rooms_required_state_changed_membership_in_timeline_lazy_loading_room_members_incremental_sync( + self, + room_membership_action: str, + ) -> None: + """ + On incremental sync, test `rooms.required_state` returns people relevant to the + timeline when lazy-loading room members, `["m.room.member","$LAZY"]` **including + changes to membership**. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + user3_id = self.register_user("user3", "pass") + user3_tok = self.login(user3_id, "pass") + user4_id = self.register_user("user4", "pass") + user4_tok = self.login(user4_id, "pass") + user5_id = self.register_user("user5", "pass") + user5_tok = self.login(user5_id, "pass") + + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # If we're testing knocks, set the room to knock + if room_membership_action == MembershipAction.KNOCK: + self.helper.send_state( + room_id1, + EventTypes.JoinRules, + {"join_rule": JoinRules.KNOCK}, + tok=user2_tok, + ) + + # Join the test users to the room + self.helper.invite(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.invite(room_id1, src=user2_id, targ=user3_id, tok=user2_tok) + self.helper.join(room_id1, user3_id, tok=user3_tok) + self.helper.invite(room_id1, src=user2_id, targ=user4_id, tok=user2_tok) + self.helper.join(room_id1, user4_id, tok=user4_tok) + if room_membership_action in ( + MembershipAction.LEAVE, + MembershipAction.BAN, + MembershipAction.JOIN, + ): + self.helper.invite(room_id1, src=user2_id, targ=user5_id, tok=user2_tok) + self.helper.join(room_id1, user5_id, tok=user5_tok) + + # Send some messages to fill up the space + self.helper.send(room_id1, "1", tok=user2_tok) + self.helper.send(room_id1, "2", tok=user2_tok) + self.helper.send(room_id1, "3", tok=user2_tok) + + # Make the Sliding Sync request with lazy loading for the room members + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Create, ""], + [EventTypes.Member, StateValues.LAZY], + ], + "timeline_limit": 3, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Send more timeline events into the room + self.helper.send(room_id1, "4", tok=user2_tok) + self.helper.send(room_id1, "5", tok=user4_tok) + # The third event will be our membership event concerning user5 + if room_membership_action == MembershipAction.LEAVE: + # User 5 leaves + self.helper.leave(room_id1, user5_id, tok=user5_tok) + elif room_membership_action == MembershipAction.INVITE: + # User 5 is invited + self.helper.invite(room_id1, src=user2_id, targ=user5_id, tok=user2_tok) + elif room_membership_action == MembershipAction.KNOCK: + # User 5 knocks + self.helper.knock(room_id1, user5_id, tok=user5_tok) + # The admin of the room accepts the knock + self.helper.invite(room_id1, src=user2_id, targ=user5_id, tok=user2_tok) + elif room_membership_action == MembershipAction.JOIN: + # Update the display name of user5 (causing a membership change) + self.helper.send_state( + room_id1, + event_type=EventTypes.Member, + state_key=user5_id, + body={ + EventContentFields.MEMBERSHIP: Membership.JOIN, + EventContentFields.MEMBERSHIP_DISPLAYNAME: "quick changer", + }, + tok=user5_tok, + ) + elif room_membership_action == MembershipAction.BAN: + self.helper.ban(room_id1, src=user2_id, targ=user5_id, tok=user2_tok) + elif room_membership_action == MembershipAction.KICK: + # Kick user5 from the room + self.helper.change_membership( + room=room_id1, + src=user2_id, + targ=user5_id, + tok=user2_tok, + membership=Membership.LEAVE, + extra_data={ + "reason": "Bad manners", + }, + ) + else: + raise AssertionError( + f"Unknown room_membership_action: {room_membership_action}" + ) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + + # Only user2, user4, and user5 sent events in the last 3 events we see in the + # `timeline`. + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + # This appears because *some* membership in the room changed and the + # heroes are recalculated and is thrown in because we have it. But this + # is technically optional and not needed because we've already seen user2 + # in the last sync (and their membership hasn't changed). + state_map[(EventTypes.Member, user2_id)], + # Appears because there is a message in the timeline from this user + state_map[(EventTypes.Member, user4_id)], + # Appears because there is a membership event in the timeline from this user + state_map[(EventTypes.Member, user5_id)], + }, + exact=True, + ) + self.assertIsNone(response_body["rooms"][room_id1].get("invite_state")) + def test_rooms_required_state_expand_lazy_loading_room_members_incremental_sync( self, ) -> None: @@ -751,9 +911,10 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): self.assertIsNone(response_body["rooms"][room_id1].get("invite_state")) @parameterized.expand([(Membership.LEAVE,), (Membership.BAN,)]) - def test_rooms_required_state_leave_ban(self, stop_membership: str) -> None: + def test_rooms_required_state_leave_ban_initial(self, stop_membership: str) -> None: """ - Test `rooms.required_state` should not return state past a leave/ban event. + Test `rooms.required_state` should not return state past a leave/ban event when + it's the first "initial" time the room is being sent down the connection. """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -788,6 +949,13 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): body={"foo": "bar"}, tok=user2_tok, ) + self.helper.send_state( + room_id1, + event_type="org.matrix.bar_state", + state_key="", + body={"bar": "bar"}, + tok=user2_tok, + ) if stop_membership == Membership.LEAVE: # User 1 leaves @@ -796,6 +964,8 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): # User 1 is banned self.helper.ban(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) + # Get the state_map before we change the state as this is the final state we + # expect User1 to be able to see state_map = self.get_success( self.storage_controllers.state.get_current_state(room_id1) ) @@ -808,12 +978,36 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): body={"foo": "qux"}, tok=user2_tok, ) + self.helper.send_state( + room_id1, + event_type="org.matrix.bar_state", + state_key="", + body={"bar": "qux"}, + tok=user2_tok, + ) self.helper.leave(room_id1, user3_id, tok=user3_tok) # Make an incremental Sliding Sync request + # + # Also expand the required state to include the `org.matrix.bar_state` event. + # This is just an extra complication of the test. + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Create, ""], + [EventTypes.Member, "*"], + ["org.matrix.foo_state", ""], + ["org.matrix.bar_state", ""], + ], + "timeline_limit": 3, + } + } + } response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) - # Only user2 and user3 sent events in the 3 events we see in the `timeline` + # We should only see the state up to the leave/ban event self._assertRequiredStateIncludes( response_body["rooms"][room_id1]["required_state"], { @@ -822,6 +1016,126 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): state_map[(EventTypes.Member, user2_id)], state_map[(EventTypes.Member, user3_id)], state_map[("org.matrix.foo_state", "")], + state_map[("org.matrix.bar_state", "")], + }, + exact=True, + ) + self.assertIsNone(response_body["rooms"][room_id1].get("invite_state")) + + @parameterized.expand([(Membership.LEAVE,), (Membership.BAN,)]) + def test_rooms_required_state_leave_ban_incremental( + self, stop_membership: str + ) -> None: + """ + Test `rooms.required_state` should not return state past a leave/ban event on + incremental sync. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + user3_id = self.register_user("user3", "pass") + user3_tok = self.login(user3_id, "pass") + + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.join(room_id1, user3_id, tok=user3_tok) + + self.helper.send_state( + room_id1, + event_type="org.matrix.foo_state", + state_key="", + body={"foo": "bar"}, + tok=user2_tok, + ) + self.helper.send_state( + room_id1, + event_type="org.matrix.bar_state", + state_key="", + body={"bar": "bar"}, + tok=user2_tok, + ) + + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Create, ""], + [EventTypes.Member, "*"], + ["org.matrix.foo_state", ""], + ], + "timeline_limit": 3, + } + } + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + if stop_membership == Membership.LEAVE: + # User 1 leaves + self.helper.leave(room_id1, user1_id, tok=user1_tok) + elif stop_membership == Membership.BAN: + # User 1 is banned + self.helper.ban(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) + + # Get the state_map before we change the state as this is the final state we + # expect User1 to be able to see + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + + # Change the state after user 1 leaves + self.helper.send_state( + room_id1, + event_type="org.matrix.foo_state", + state_key="", + body={"foo": "qux"}, + tok=user2_tok, + ) + self.helper.send_state( + room_id1, + event_type="org.matrix.bar_state", + state_key="", + body={"bar": "qux"}, + tok=user2_tok, + ) + self.helper.leave(room_id1, user3_id, tok=user3_tok) + + # Make an incremental Sliding Sync request + # + # Also expand the required state to include the `org.matrix.bar_state` event. + # This is just an extra complication of the test. + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Create, ""], + [EventTypes.Member, "*"], + ["org.matrix.foo_state", ""], + ["org.matrix.bar_state", ""], + ], + "timeline_limit": 3, + } + } + } + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + + # User1 should only see the state up to the leave/ban event + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + # User1 should see their leave/ban membership + state_map[(EventTypes.Member, user1_id)], + state_map[("org.matrix.bar_state", "")], + # The commented out state events were already returned in the initial + # sync so we shouldn't see them again on the incremental sync. And we + # shouldn't see the state events that changed after the leave/ban event. + # + # state_map[(EventTypes.Create, "")], + # state_map[(EventTypes.Member, user2_id)], + # state_map[(EventTypes.Member, user3_id)], + # state_map[("org.matrix.foo_state", "")], }, exact=True, ) @@ -1243,7 +1557,7 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): # Update the room name self.helper.send_state( - room_id1, "m.room.name", {"name": "Bar"}, state_key="", tok=user1_tok + room_id1, EventTypes.Name, {"name": "Bar"}, state_key="", tok=user1_tok ) # Update the sliding sync requests to exclude the room name again diff --git a/tests/rest/client/sliding_sync/test_rooms_timeline.py b/tests/rest/client/sliding_sync/test_rooms_timeline.py index 2293994793..3d950eb20b 100644 --- a/tests/rest/client/sliding_sync/test_rooms_timeline.py +++ b/tests/rest/client/sliding_sync/test_rooms_timeline.py @@ -16,7 +16,7 @@ from typing import List, Optional from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EventTypes @@ -309,8 +309,8 @@ class SlidingSyncRoomsTimelineTestCase(SlidingSyncBase): self.assertEqual( response_body["rooms"][room_id1]["limited"], False, - f'Our `timeline_limit` was {sync_body["lists"]["foo-list"]["timeline_limit"]} ' - + f'and {len(response_body["rooms"][room_id1]["timeline"])} events were returned in the timeline. ' + f"Our `timeline_limit` was {sync_body['lists']['foo-list']['timeline_limit']} " + + f"and {len(response_body['rooms'][room_id1]['timeline'])} events were returned in the timeline. " + str(response_body["rooms"][room_id1]), ) # Check to make sure the latest events are returned @@ -387,7 +387,7 @@ class SlidingSyncRoomsTimelineTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["limited"], True, f"Our `timeline_limit` was {timeline_limit} " - + f'and {len(response_body["rooms"][room_id1]["timeline"])} events were returned in the timeline. ' + + f"and {len(response_body['rooms'][room_id1]['timeline'])} events were returned in the timeline. " + str(response_body["rooms"][room_id1]), ) # Check to make sure that the "live" and historical events are returned diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index 578cb384cd..ea4ee16359 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -18,7 +18,7 @@ from unittest.mock import AsyncMock from parameterized import parameterized, parameterized_class from typing_extensions import assert_never -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import ( @@ -790,6 +790,64 @@ class SlidingSyncTestCase(SlidingSyncBase): exact=True, ) + def test_reject_remote_invite(self) -> None: + """Test that rejecting a remote invite comes down incremental sync""" + + user_id = self.register_user("user1", "pass") + user_tok = self.login(user_id, "pass") + + # Create a remote room invite (out-of-band membership) + room_id = "!room:remote.server" + self._create_remote_invite_room_for_user(user_id, None, room_id) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [(EventTypes.Member, StateValues.ME)], + "timeline_limit": 3, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user_tok) + # We should see the room (like normal) + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id}, + exact=True, + ) + + # Reject the remote room invite + self.helper.leave(room_id, user_id, tok=user_tok) + + # Sync again after rejecting the invite + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user_tok) + + # The fix to add the leave event to incremental sync when rejecting a remote + # invite relies on the new tables to work. + if self.use_new_tables: + # We should see the newly_left room + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id}, + exact=True, + ) + # We should see the leave state for the room so clients don't end up with stuck + # invites + self.assertIncludes( + { + ( + state["type"], + state["state_key"], + state["content"].get("membership"), + ) + for state in response_body["rooms"][room_id]["required_state"] + }, + {(EventTypes.Member, user_id, Membership.LEAVE)}, + exact=True, + ) + def test_ignored_user_invites_initial_sync(self) -> None: """ Make sure we ignore invites if they are from one of the `m.ignored_user_list` on @@ -1169,12 +1227,6 @@ class SlidingSyncTestCase(SlidingSyncBase): self.persistence.persist_event(join_rule_event, join_rule_context) ) - # FIXME: We're manually busting the cache since - # https://github.com/element-hq/synapse/issues/17368 is not solved yet - self.store._membership_stream_cache.entity_has_changed( - user1_id, join_rule_event_pos.stream - ) - # Ensure that the state reset worked and only user2 is in the room now users_in_room = self.get_success(self.store.get_users_in_room(room_id1)) self.assertIncludes(set(users_in_room), {user2_id}, exact=True) @@ -1322,12 +1374,6 @@ class SlidingSyncTestCase(SlidingSyncBase): self.persistence.persist_event(join_rule_event, join_rule_context) ) - # FIXME: We're manually busting the cache since - # https://github.com/element-hq/synapse/issues/17368 is not solved yet - self.store._membership_stream_cache.entity_has_changed( - user1_id, join_rule_event_pos.stream - ) - # Ensure that the state reset worked and only user2 is in the room now users_in_room = self.get_success(self.store.get_users_in_room(space_room_id)) self.assertIncludes(set(users_in_room), {user2_id}, exact=True) @@ -1506,12 +1552,6 @@ class SlidingSyncTestCase(SlidingSyncBase): self.persistence.persist_event(join_rule_event, join_rule_context) ) - # FIXME: We're manually busting the cache since - # https://github.com/element-hq/synapse/issues/17368 is not solved yet - self.store._membership_stream_cache.entity_has_changed( - user1_id, join_rule_event_pos.stream - ) - # Ensure that the state reset worked and only user2 is in the room now users_in_room = self.get_success(self.store.get_users_in_room(space_room_id)) self.assertIncludes(set(users_in_room), {user2_id}, exact=True) @@ -1576,3 +1616,55 @@ class SlidingSyncTestCase(SlidingSyncBase): {space_room_id, space_room_id2}, exact=True, ) + + def test_exclude_rooms_from_sync(self) -> None: + """Tests that sliding sync honours the `exclude_rooms_from_sync` config + option. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id_to_exclude = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + room_id_to_include = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + + # We cheekily modify the stored config here, as we can't add it to the + # raw config since we don't know the room ID before we start up. + self.hs.get_sliding_sync_handler().rooms_to_exclude_globally.append( + room_id_to_exclude + ) + self.hs.get_sliding_sync_handler().room_lists.rooms_to_exclude_globally.append( + room_id_to_exclude + ) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 99]], + "required_state": [], + "timeline_limit": 0, + }, + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Make sure response only contains room_id_to_include + self.assertIncludes( + set(response_body["rooms"].keys()), + {room_id_to_include}, + exact=True, + ) + + # Test that the excluded room is not in the list ops + # Make sure the list is sorted in the way we expect + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id_to_include}, + exact=True, + ) diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index a85ea994de..9a3202bd93 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +import importlib.resources as importlib_resources import os import re from email.parser import Parser @@ -25,10 +26,8 @@ from http import HTTPStatus from typing import Any, Dict, List, Optional, Union from unittest.mock import Mock -import pkg_resources - from twisted.internet.interfaces import IReactorTCP -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import LoginType, Membership @@ -59,11 +58,12 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): config = self.default_config() # Email config. + templates = ( + importlib_resources.files("synapse").joinpath("res").joinpath("templates") + ) config["email"] = { "enable_notifs": False, - "template_dir": os.path.abspath( - pkg_resources.resource_filename("synapse", "res/templates") - ), + "template_dir": os.path.abspath(str(templates)), "smtp_host": "127.0.0.1", "smtp_port": 20, "require_transport_security": False, @@ -764,10 +764,10 @@ class WhoamiTestCase(unittest.HomeserverTestCase): as_token = "i_am_an_app_service" appservice = ApplicationService( - as_token, + token=as_token, id="1234", namespaces={"users": [{"regex": user_id, "exclusive": True}]}, - sender=user_id, + sender=UserID.from_string(user_id), ) self.hs.get_datastores().main.services_cache.append(appservice) @@ -798,11 +798,12 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): config = self.default_config() # Email config. + templates = ( + importlib_resources.files("synapse").joinpath("res").joinpath("templates") + ) config["email"] = { "enable_notifs": False, - "template_dir": os.path.abspath( - pkg_resources.resource_filename("synapse", "res/templates") - ), + "template_dir": os.path.abspath(str(templates)), "smtp_host": "127.0.0.1", "smtp_port": 20, "require_transport_security": False, diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py index 0b5daf4bb4..4fe506845c 100644 --- a/tests/rest/client/test_auth.py +++ b/tests/rest/client/test_auth.py @@ -23,7 +23,7 @@ from http import HTTPStatus from typing import Any, Dict, List, Optional, Tuple, Union from twisted.internet.defer import succeed -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource import synapse.rest.admin diff --git a/tests/rest/client/test_auth_issuer.py b/tests/rest/client/test_auth_issuer.py deleted file mode 100644 index d6f334a7ab..0000000000 --- a/tests/rest/client/test_auth_issuer.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2023 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from http import HTTPStatus -from unittest.mock import AsyncMock - -from synapse.rest.client import auth_issuer - -from tests.unittest import HomeserverTestCase, override_config, skip_unless -from tests.utils import HAS_AUTHLIB - -ISSUER = "https://account.example.com/" - - -class AuthIssuerTestCase(HomeserverTestCase): - servlets = [ - auth_issuer.register_servlets, - ] - - def test_returns_404_when_msc3861_disabled(self) -> None: - # Make an unauthenticated request for the discovery info. - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) - - @skip_unless(HAS_AUTHLIB, "requires authlib") - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": "David Lister", - "client_auth_method": "client_secret_post", - "client_secret": "Who shot Mister Burns?", - } - }, - } - ) - def test_returns_issuer_when_oidc_enabled(self) -> None: - # Patch the HTTP client to return the issuer metadata - req_mock = AsyncMock(return_value={"issuer": ISSUER}) - self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] - - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual(channel.json_body, {"issuer": ISSUER}) - - req_mock.assert_called_with( - "https://account.example.com/.well-known/openid-configuration" - ) - req_mock.reset_mock() - - # Second call it should use the cached value - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual(channel.json_body, {"issuer": ISSUER}) - req_mock.assert_not_called() diff --git a/tests/rest/client/test_auth_metadata.py b/tests/rest/client/test_auth_metadata.py new file mode 100644 index 0000000000..c13d410636 --- /dev/null +++ b/tests/rest/client/test_auth_metadata.py @@ -0,0 +1,145 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2023 The Matrix.org Foundation C.I.C +# Copyright (C) 2023-2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +from http import HTTPStatus +from typing import ClassVar +from unittest.mock import AsyncMock + +from parameterized import parameterized_class + +from synapse.rest.client import auth_metadata + +from tests.unittest import HomeserverTestCase, override_config, skip_unless +from tests.utils import HAS_AUTHLIB + +ISSUER = "https://account.example.com/" + + +class AuthIssuerTestCase(HomeserverTestCase): + servlets = [ + auth_metadata.register_servlets, + ] + + def test_returns_404_when_msc3861_disabled(self) -> None: + # Make an unauthenticated request for the discovery info. + channel = self.make_request( + "GET", + "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + + @skip_unless(HAS_AUTHLIB, "requires authlib") + @override_config( + { + "disable_registration": True, + "experimental_features": { + "msc3861": { + "enabled": True, + "issuer": ISSUER, + "client_id": "David Lister", + "client_auth_method": "client_secret_post", + "client_secret": "Who shot Mister Burns?", + } + }, + } + ) + def test_returns_issuer_when_oidc_enabled(self) -> None: + # Patch the HTTP client to return the issuer metadata + req_mock = AsyncMock(return_value={"issuer": ISSUER}) + self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] + + channel = self.make_request( + "GET", + "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"issuer": ISSUER}) + + req_mock.assert_called_with( + "https://account.example.com/.well-known/openid-configuration" + ) + req_mock.reset_mock() + + # Second call it should use the cached value + channel = self.make_request( + "GET", + "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"issuer": ISSUER}) + req_mock.assert_not_called() + + +@parameterized_class( + ("endpoint",), + [ + ("/_matrix/client/unstable/org.matrix.msc2965/auth_metadata",), + ("/_matrix/client/v1/auth_metadata",), + ], +) +class AuthMetadataTestCase(HomeserverTestCase): + endpoint: ClassVar[str] + servlets = [ + auth_metadata.register_servlets, + ] + + def test_returns_404_when_msc3861_disabled(self) -> None: + # Make an unauthenticated request for the discovery info. + channel = self.make_request("GET", self.endpoint) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + + @skip_unless(HAS_AUTHLIB, "requires authlib") + @override_config( + { + "disable_registration": True, + "experimental_features": { + "msc3861": { + "enabled": True, + "issuer": ISSUER, + "client_id": "David Lister", + "client_auth_method": "client_secret_post", + "client_secret": "Who shot Mister Burns?", + } + }, + } + ) + def test_returns_issuer_when_oidc_enabled(self) -> None: + # Patch the HTTP client to return the issuer metadata + req_mock = AsyncMock( + return_value={ + "issuer": ISSUER, + "authorization_endpoint": "https://example.com/auth", + "token_endpoint": "https://example.com/token", + } + ) + self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] + + channel = self.make_request("GET", self.endpoint) + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + channel.json_body, + { + "issuer": ISSUER, + "authorization_endpoint": "https://example.com/auth", + "token_endpoint": "https://example.com/token", + }, + ) diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index bbe8ab1a7c..8ae1cc935a 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -19,7 +19,7 @@ # from http import HTTPStatus -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.room_versions import KNOWN_ROOM_VERSIONS @@ -130,6 +130,10 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_displayname"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["m.profile_fields"]["disallowed"], ["displayname"] + ) @override_config({"enable_set_avatar_url": False}) def test_get_set_avatar_url_capabilities_avatar_url_disabled(self) -> None: @@ -141,6 +145,58 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_avatar_url"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual(capabilities["m.profile_fields"]["disallowed"], ["avatar_url"]) + + @override_config( + { + "enable_set_displayname": False, + "experimental_features": {"msc4133_enabled": True}, + } + ) + def test_get_set_displayname_capabilities_displayname_disabled_msc4133( + self, + ) -> None: + """Test if set displayname is disabled that the server responds it.""" + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse(capabilities["m.set_displayname"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["m.profile_fields"]["disallowed"], ["displayname"] + ) + self.assertTrue(capabilities["uk.tcpip.msc4133.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["uk.tcpip.msc4133.profile_fields"]["disallowed"], + ["displayname"], + ) + + @override_config( + { + "enable_set_avatar_url": False, + "experimental_features": {"msc4133_enabled": True}, + } + ) + def test_get_set_avatar_url_capabilities_avatar_url_disabled_msc4133(self) -> None: + """Test if set avatar_url is disabled that the server responds it.""" + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse(capabilities["m.set_avatar_url"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual(capabilities["m.profile_fields"]["disallowed"], ["avatar_url"]) + self.assertTrue(capabilities["uk.tcpip.msc4133.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["uk.tcpip.msc4133.profile_fields"]["disallowed"], + ["avatar_url"], + ) @override_config({"enable_3pid_changes": False}) def test_get_change_3pid_capabilities_3pid_disabled(self) -> None: @@ -220,3 +276,43 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertTrue(capabilities["m.get_login_token"]["enabled"]) + + @override_config( + { + "experimental_features": {"msc4267_enabled": True}, + "forget_rooms_on_leave": True, + } + ) + def test_get_forget_forced_upon_leave_with_auto_forget(self) -> None: + # Server auto-forgets on /leave, expect enabled client capability + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertTrue( + capabilities["org.matrix.msc4267.forget_forced_upon_leave"]["enabled"] + ) + + @override_config( + { + "experimental_features": {"msc4267_enabled": True}, + "forget_rooms_on_leave": False, + } + ) + def test_get_forget_forced_upon_leave_without_auto_forget(self) -> None: + # Server doesn't auto-forget on /leave, expect disabled client capability + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse( + capabilities["org.matrix.msc4267.forget_forced_upon_leave"]["enabled"] + ) diff --git a/tests/rest/client/test_consent.py b/tests/rest/client/test_consent.py index 5f4168c56c..1a64b3984f 100644 --- a/tests/rest/client/test_consent.py +++ b/tests/rest/client/test_consent.py @@ -21,7 +21,7 @@ import os from http import HTTPStatus -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.urls import ConsentURIBuilder diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index 1793b38c4a..4b338d333f 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -19,10 +19,11 @@ from typing import List from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import Codes -from synapse.rest.client import delayed_events, room, versions +from synapse.rest import admin +from synapse.rest.client import delayed_events, login, room, versions from synapse.server import HomeServer from synapse.types import JsonDict from synapse.util import Clock @@ -32,7 +33,6 @@ from tests.unittest import HomeserverTestCase PATH_PREFIX = "/_matrix/client/unstable/org.matrix.msc4140/delayed_events" -_HS_NAME = "red" _EVENT_TYPE = "com.example.test" @@ -54,23 +54,41 @@ class DelayedEventsUnstableSupportTestCase(HomeserverTestCase): class DelayedEventsTestCase(HomeserverTestCase): """Tests getting and managing delayed events.""" - servlets = [delayed_events.register_servlets, room.register_servlets] - user_id = f"@sid1:{_HS_NAME}" + servlets = [ + admin.register_servlets, + delayed_events.register_servlets, + login.register_servlets, + room.register_servlets, + ] def default_config(self) -> JsonDict: config = super().default_config() - config["server_name"] = _HS_NAME config["max_event_delay_duration"] = "24h" return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user1_user_id = self.register_user("user1", "pass") + self.user1_access_token = self.login("user1", "pass") + self.user2_user_id = self.register_user("user2", "pass") + self.user2_access_token = self.login("user2", "pass") + self.room_id = self.helper.create_room_as( - self.user_id, + self.user1_user_id, + tok=self.user1_access_token, extra_content={ - "preset": "trusted_private_chat", + "preset": "public_chat", + "power_level_content_override": { + "events": { + _EVENT_TYPE: 0, + } + }, }, ) + self.helper.join( + room=self.room_id, user=self.user2_user_id, tok=self.user2_access_token + ) + def test_delayed_events_empty_on_startup(self) -> None: self.assertListEqual([], self._get_delayed_events()) @@ -85,6 +103,7 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: setter_expected, }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) events = self._get_delayed_events() @@ -94,7 +113,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -104,15 +123,39 @@ class DelayedEventsTestCase(HomeserverTestCase): content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, ) self.assertEqual(setter_expected, content.get(setter_key), content) + @unittest.override_config( + {"rc_delayed_event_mgmt": {"per_second": 0.5, "burst_count": 1}} + ) + def test_get_delayed_events_ratelimit(self) -> None: + args = ("GET", PATH_PREFIX, b"", self.user1_access_token) + + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) + ) + + # Test that the request isn't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + def test_update_delayed_event_without_id(self) -> None: channel = self.make_request( "POST", f"{PATH_PREFIX}/", + access_token=self.user1_access_token, ) self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, channel.result) @@ -120,6 +163,7 @@ class DelayedEventsTestCase(HomeserverTestCase): channel = self.make_request( "POST", f"{PATH_PREFIX}/abc", + access_token=self.user1_access_token, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) self.assertEqual( @@ -132,6 +176,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/abc", {}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) self.assertEqual( @@ -144,6 +189,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/abc", {"action": "oops"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) self.assertEqual( @@ -157,6 +203,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/abc", {"action": action}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, channel.result) @@ -171,6 +218,7 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: setter_expected, }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) delay_id = channel.json_body.get("delay_id") @@ -184,7 +232,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -193,6 +241,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_id}", {"action": "cancel"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertListEqual([], self._get_delayed_events()) @@ -201,11 +250,56 @@ class DelayedEventsTestCase(HomeserverTestCase): content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) + @unittest.override_config( + {"rc_delayed_event_mgmt": {"per_second": 0.5, "burst_count": 1}} + ) + def test_cancel_delayed_event_ratelimit(self) -> None: + delay_ids = [] + for _ in range(2): + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), + {}, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + delay_id = channel.json_body.get("delay_id") + self.assertIsNotNone(delay_id) + delay_ids.append(delay_id) + + channel = self.make_request( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "cancel"}, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + args = ( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "cancel"}, + self.user1_access_token, + ) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) + ) + + # Test that the request isn't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + def test_send_delayed_state_event(self) -> None: state_key = "to_send_on_request" @@ -217,6 +311,7 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: setter_expected, }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) delay_id = channel.json_body.get("delay_id") @@ -230,7 +325,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -239,17 +334,61 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_id}", {"action": "send"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertListEqual([], self._get_delayed_events()) content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, ) self.assertEqual(setter_expected, content.get(setter_key), content) + @unittest.override_config({"rc_message": {"per_second": 3.5, "burst_count": 4}}) + def test_send_delayed_event_ratelimit(self) -> None: + delay_ids = [] + for _ in range(2): + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), + {}, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + delay_id = channel.json_body.get("delay_id") + self.assertIsNotNone(delay_id) + delay_ids.append(delay_id) + + channel = self.make_request( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "send"}, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + args = ( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "send"}, + self.user1_access_token, + ) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) + ) + + # Test that the request isn't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + def test_restart_delayed_state_event(self) -> None: state_key = "to_send_on_restarted_timeout" @@ -261,6 +400,7 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: setter_expected, }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) delay_id = channel.json_body.get("delay_id") @@ -274,7 +414,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -283,6 +423,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_id}", {"action": "restart"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @@ -294,7 +435,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -304,13 +445,100 @@ class DelayedEventsTestCase(HomeserverTestCase): content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, ) self.assertEqual(setter_expected, content.get(setter_key), content) - def test_delayed_state_events_are_cancelled_by_more_recent_state(self) -> None: - state_key = "to_be_cancelled" + @unittest.override_config( + {"rc_delayed_event_mgmt": {"per_second": 0.5, "burst_count": 1}} + ) + def test_restart_delayed_event_ratelimit(self) -> None: + delay_ids = [] + for _ in range(2): + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), + {}, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + delay_id = channel.json_body.get("delay_id") + self.assertIsNotNone(delay_id) + delay_ids.append(delay_id) + + channel = self.make_request( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "restart"}, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + args = ( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "restart"}, + self.user1_access_token, + ) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) + ) + + # Test that the request isn't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + def test_delayed_state_is_not_cancelled_by_new_state_from_same_user( + self, + ) -> None: + state_key = "to_not_be_cancelled_by_same_user" + + setter_key = "setter" + setter_expected = "on_timeout" + channel = self.make_request( + "PUT", + _get_path_for_delayed_state(self.room_id, _EVENT_TYPE, state_key, 900), + { + setter_key: setter_expected, + }, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + events = self._get_delayed_events() + self.assertEqual(1, len(events), events) + + self.helper.send_state( + self.room_id, + _EVENT_TYPE, + { + setter_key: "manual", + }, + self.user1_access_token, + state_key=state_key, + ) + events = self._get_delayed_events() + self.assertEqual(1, len(events), events) + + self.reactor.advance(1) + content = self.helper.get_state( + self.room_id, + _EVENT_TYPE, + self.user1_access_token, + state_key=state_key, + ) + self.assertEqual(setter_expected, content.get(setter_key), content) + + def test_delayed_state_is_cancelled_by_new_state_from_other_user( + self, + ) -> None: + state_key = "to_be_cancelled_by_other_user" setter_key = "setter" channel = self.make_request( @@ -319,19 +547,20 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: "on_timeout", }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) events = self._get_delayed_events() self.assertEqual(1, len(events), events) - setter_expected = "manual" + setter_expected = "other_user" self.helper.send_state( self.room_id, _EVENT_TYPE, { setter_key: setter_expected, }, - None, + self.user2_access_token, state_key=state_key, ) self.assertListEqual([], self._get_delayed_events()) @@ -340,7 +569,7 @@ class DelayedEventsTestCase(HomeserverTestCase): content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, ) self.assertEqual(setter_expected, content.get(setter_key), content) @@ -349,6 +578,7 @@ class DelayedEventsTestCase(HomeserverTestCase): channel = self.make_request( "GET", PATH_PREFIX, + access_token=self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @@ -374,3 +604,7 @@ def _get_path_for_delayed_state( room_id: str, event_type: str, state_key: str, delay_ms: int ) -> str: return f"rooms/{room_id}/state/{event_type}/{state_key}?org.matrix.msc4140.delay={delay_ms}" + + +def _get_path_for_delayed_send(room_id: str, event_type: str, delay_ms: int) -> str: + return f"rooms/{room_id}/send/{event_type}?org.matrix.msc4140.delay={delay_ms}" diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index a3ed12a38f..2c498e97e1 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -21,9 +21,10 @@ from http import HTTPStatus from twisted.internet.defer import ensureDeferred -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import NotFoundError +from synapse.appservice import ApplicationService from synapse.rest import admin, devices, sync from synapse.rest.client import keys, login, register from synapse.server import HomeServer @@ -455,3 +456,183 @@ class DehydratedDeviceTestCase(unittest.HomeserverTestCase): token, ) self.assertEqual(channel.json_body["device_keys"], {"@mikey:test": {}}) + + +class MSC4190AppserviceDevicesTestCase(unittest.HomeserverTestCase): + servlets = [ + register.register_servlets, + devices.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.hs = self.setup_test_homeserver() + + # This application service uses the new MSC4190 behaviours + self.msc4190_service = ApplicationService( + id="msc4190", + token="some_token", + hs_token="some_token", + sender=UserID.from_string("@as:example.com"), + namespaces={ + ApplicationService.NS_USERS: [{"regex": "@.*", "exclusive": False}] + }, + msc4190_device_management=True, + ) + # This application service doesn't use the new MSC4190 behaviours + self.pre_msc_service = ApplicationService( + id="regular", + token="other_token", + hs_token="other_token", + sender=UserID.from_string("@as2:example.com"), + namespaces={ + ApplicationService.NS_USERS: [{"regex": "@.*", "exclusive": False}] + }, + msc4190_device_management=False, + ) + self.hs.get_datastores().main.services_cache.append(self.msc4190_service) + self.hs.get_datastores().main.services_cache.append(self.pre_msc_service) + return self.hs + + def test_PUT_device(self) -> None: + self.register_appservice_user("alice", self.msc4190_service.token) + self.register_appservice_user("bob", self.pre_msc_service.token) + + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) + + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + content={"display_name": "Alice's device"}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 201, channel.json_body) + + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(len(channel.json_body["devices"]), 1) + self.assertEqual(channel.json_body["devices"][0]["device_id"], "AABBCCDD") + + # Doing a second time should return a 200 instead of a 201 + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + content={"display_name": "Alice's device"}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # On the regular service, that API should not allow for the + # creation of new devices. + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@bob:test", + content={"display_name": "Bob's device"}, + access_token=self.pre_msc_service.token, + ) + self.assertEqual(channel.code, 404, channel.json_body) + + def test_DELETE_device(self) -> None: + self.register_appservice_user("alice", self.msc4190_service.token) + + # There should be no device + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) + + # Create a device + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + content={}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 201, channel.json_body) + + # There should be one device + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(len(channel.json_body["devices"]), 1) + + # Delete the device. UIA should not be required. + channel = self.make_request( + "DELETE", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # There should be no device again + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) + + def test_POST_delete_devices(self) -> None: + self.register_appservice_user("alice", self.msc4190_service.token) + + # There should be no device + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) + + # Create a device + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + content={}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 201, channel.json_body) + + # There should be one device + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(len(channel.json_body["devices"]), 1) + + # Delete the device with delete_devices + # UIA should not be required. + channel = self.make_request( + "POST", + "/_matrix/client/v3/delete_devices?user_id=@alice:test", + content={"devices": ["AABBCCDD"]}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # There should be no device again + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) diff --git a/tests/rest/client/test_directory.py b/tests/rest/client/test_directory.py index ecf38493c3..6548ac6fa8 100644 --- a/tests/rest/client/test_directory.py +++ b/tests/rest/client/test_directory.py @@ -19,13 +19,13 @@ # from http import HTTPStatus -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.appservice import ApplicationService from synapse.rest import admin from synapse.rest.client import directory, login, room from synapse.server import HomeServer -from synapse.types import RoomAlias +from synapse.types import RoomAlias, UserID from synapse.util import Clock from synapse.util.stringutils import random_string @@ -140,7 +140,7 @@ class DirectoryTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"aliases": [{"regex": "#asns-*", "exclusive": True}]}, - sender=user_id, + sender=UserID.from_string(user_id), ) self.hs.get_datastores().main.services_cache.append(appservice) diff --git a/tests/rest/client/test_ephemeral_message.py b/tests/rest/client/test_ephemeral_message.py index 2d98fda67f..5b5c220825 100644 --- a/tests/rest/client/test_ephemeral_message.py +++ b/tests/rest/client/test_ephemeral_message.py @@ -19,7 +19,7 @@ # from http import HTTPStatus -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventContentFields, EventTypes from synapse.rest import admin diff --git a/tests/rest/client/test_events.py b/tests/rest/client/test_events.py index 039144fdbe..142509bbf7 100644 --- a/tests/rest/client/test_events.py +++ b/tests/rest/client/test_events.py @@ -23,7 +23,7 @@ from unittest.mock import Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EduTypes diff --git a/tests/rest/client/test_filter.py b/tests/rest/client/test_filter.py index 9cfc6b224f..4153fb322d 100644 --- a/tests/rest/client/test_filter.py +++ b/tests/rest/client/test_filter.py @@ -19,7 +19,7 @@ # # -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import Codes from synapse.rest.client import filter diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py index 63c2c5923e..87af18f473 100644 --- a/tests/rest/client/test_identity.py +++ b/tests/rest/client/test_identity.py @@ -20,7 +20,7 @@ from http import HTTPStatus -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.rest.client import login, room diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index cbd6d8d4bf..8f9856fa2e 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -27,6 +27,7 @@ from typing import ( Collection, Dict, List, + Literal, Optional, Tuple, Union, @@ -35,14 +36,14 @@ from unittest.mock import Mock from urllib.parse import urlencode import pymacaroons -from typing_extensions import Literal -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource import synapse.rest.admin from synapse.api.constants import ApprovalNoticeMedium, LoginType from synapse.api.errors import Codes +from synapse.api.urls import LoginSSORedirectURIBuilder from synapse.appservice import ApplicationService from synapse.http.client import RawHeaders from synapse.module_api import ModuleApi @@ -50,7 +51,7 @@ from synapse.rest.client import account, devices, login, logout, profile, regist from synapse.rest.client.account import WhoamiRestServlet from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.server import HomeServer -from synapse.types import JsonDict, create_requester +from synapse.types import JsonDict, UserID, create_requester from synapse.util import Clock from tests import unittest @@ -69,6 +70,10 @@ try: except ImportError: HAS_JWT = False +import logging + +logger = logging.getLogger(__name__) + # synapse server name: used to populate public_baseurl in some tests SYNAPSE_SERVER_PUBLIC_HOSTNAME = "synapse" @@ -77,7 +82,7 @@ SYNAPSE_SERVER_PUBLIC_HOSTNAME = "synapse" # FakeChannel.isSecure() returns False, so synapse will see the requested uri as # http://..., so using http in the public_baseurl stops Synapse trying to redirect to # https://.... -BASE_URL = "http://%s/" % (SYNAPSE_SERVER_PUBLIC_HOSTNAME,) +PUBLIC_BASEURL = "http://%s/" % (SYNAPSE_SERVER_PUBLIC_HOSTNAME,) # CAS server used in some tests CAS_SERVER = "https://fake.test" @@ -109,6 +114,23 @@ ADDITIONAL_LOGIN_FLOWS = [ ] +def get_relative_uri_from_absolute_uri(absolute_uri: str) -> str: + """ + Peels off the path and query string from an absolute URI. Useful when interacting + with `make_request(...)` util function which expects a relative path instead of a + full URI. + """ + parsed_uri = urllib.parse.urlparse(absolute_uri) + # Sanity check that we're working with an absolute URI + assert parsed_uri.scheme == "http" or parsed_uri.scheme == "https" + + relative_uri = parsed_uri.path + if parsed_uri.query: + relative_uri += "?" + parsed_uri.query + + return relative_uri + + class TestSpamChecker: def __init__(self, config: None, api: ModuleApi): api.register_spam_checker_callbacks( @@ -614,7 +636,7 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): def default_config(self) -> Dict[str, Any]: config = super().default_config() - config["public_baseurl"] = BASE_URL + config["public_baseurl"] = PUBLIC_BASEURL config["cas_config"] = { "enabled": True, @@ -653,6 +675,9 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): ] return config + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.login_sso_redirect_url_builder = LoginSSORedirectURIBuilder(hs.config) + def create_resource_dict(self) -> Dict[str, Resource]: d = super().create_resource_dict() d.update(build_synapse_client_resource_tree(self.hs)) @@ -725,6 +750,32 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): + "&idp=cas", shorthand=False, ) + self.assertEqual(channel.code, 302, channel.result) + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers + sso_login_redirect_uri = location_headers[0] + + # it should redirect us to the standard login SSO redirect flow + self.assertEqual( + sso_login_redirect_uri, + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="cas", client_redirect_url=TEST_CLIENT_REDIRECT_URL + ), + ) + + # follow the redirect + channel = self.make_request( + "GET", + # We have to make this relative to be compatible with `make_request(...)` + get_relative_uri_from_absolute_uri(sso_login_redirect_uri), + # We have to set the Host header to match the `public_baseurl` to avoid + # the extra redirect in the `SsoRedirectServlet` in order for the + # cookies to be visible. + custom_headers=[ + ("Host", SYNAPSE_SERVER_PUBLIC_HOSTNAME), + ], + ) + self.assertEqual(channel.code, 302, channel.result) location_headers = channel.headers.getRawHeaders("Location") assert location_headers @@ -750,6 +801,32 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): + urllib.parse.quote_plus(TEST_CLIENT_REDIRECT_URL) + "&idp=saml", ) + self.assertEqual(channel.code, 302, channel.result) + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers + sso_login_redirect_uri = location_headers[0] + + # it should redirect us to the standard login SSO redirect flow + self.assertEqual( + sso_login_redirect_uri, + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="saml", client_redirect_url=TEST_CLIENT_REDIRECT_URL + ), + ) + + # follow the redirect + channel = self.make_request( + "GET", + # We have to make this relative to be compatible with `make_request(...)` + get_relative_uri_from_absolute_uri(sso_login_redirect_uri), + # We have to set the Host header to match the `public_baseurl` to avoid + # the extra redirect in the `SsoRedirectServlet` in order for the + # cookies to be visible. + custom_headers=[ + ("Host", SYNAPSE_SERVER_PUBLIC_HOSTNAME), + ], + ) + self.assertEqual(channel.code, 302, channel.result) location_headers = channel.headers.getRawHeaders("Location") assert location_headers @@ -773,13 +850,38 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): # pick the default OIDC provider channel = self.make_request( "GET", - "/_synapse/client/pick_idp?redirectUrl=" - + urllib.parse.quote_plus(TEST_CLIENT_REDIRECT_URL) - + "&idp=oidc", + f"/_synapse/client/pick_idp?redirectUrl={urllib.parse.quote_plus(TEST_CLIENT_REDIRECT_URL)}&idp=oidc", ) self.assertEqual(channel.code, 302, channel.result) location_headers = channel.headers.getRawHeaders("Location") assert location_headers + sso_login_redirect_uri = location_headers[0] + + # it should redirect us to the standard login SSO redirect flow + self.assertEqual( + sso_login_redirect_uri, + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="oidc", client_redirect_url=TEST_CLIENT_REDIRECT_URL + ), + ) + + with fake_oidc_server.patch_homeserver(hs=self.hs): + # follow the redirect + channel = self.make_request( + "GET", + # We have to make this relative to be compatible with `make_request(...)` + get_relative_uri_from_absolute_uri(sso_login_redirect_uri), + # We have to set the Host header to match the `public_baseurl` to avoid + # the extra redirect in the `SsoRedirectServlet` in order for the + # cookies to be visible. + custom_headers=[ + ("Host", SYNAPSE_SERVER_PUBLIC_HOSTNAME), + ], + ) + + self.assertEqual(channel.code, 302, channel.result) + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers oidc_uri = location_headers[0] oidc_uri_path, oidc_uri_query = oidc_uri.split("?", 1) @@ -837,14 +939,33 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): self.assertEqual(chan.code, 200, chan.result) self.assertEqual(chan.json_body["user_id"], "@user1:test") - def test_multi_sso_redirect_to_unknown(self) -> None: - """An unknown IdP should cause a 400""" + def test_multi_sso_redirect_unknown_idp(self) -> None: + """An unknown IdP should cause a 400 bad request error""" channel = self.make_request( "GET", "/_synapse/client/pick_idp?redirectUrl=http://x&idp=xyz", ) self.assertEqual(channel.code, 400, channel.result) + def test_multi_sso_redirect_unknown_idp_as_url(self) -> None: + """ + An unknown IdP that looks like a URL should cause a 400 bad request error (to + avoid open redirects). + + Ideally, we'd have another test for a known IdP with a URL as the `idp_id`, but + we can't configure that in our tests because the config validation on + `oidc_providers` only allows a subset of characters. If we could configure + `oidc_providers` with a URL as the `idp_id`, it should still be URL-encoded + properly to avoid open redirections. We do have `test_url_as_idp_id_is_escaped` + in the URL building tests to cover this case but is only a unit test vs + something at the REST layer here that covers things end-to-end. + """ + channel = self.make_request( + "GET", + "/_synapse/client/pick_idp?redirectUrl=something&idp=https://element.io/", + ) + self.assertEqual(channel.code, 400, channel.result) + def test_client_idp_redirect_to_unknown(self) -> None: """If the client tries to pick an unknown IdP, return a 404""" channel = self._make_sso_redirect_request("xxx") @@ -1134,18 +1255,18 @@ class JWTTestCase(unittest.HomeserverTestCase): channel = self.jwt_login({"sub": "kermit", "iss": "invalid"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: invalid_claim: Invalid claim "iss"', + r"^JWT validation failed: invalid_claim: Invalid claim [\"']iss[\"']$", ) # Not providing an issuer. channel = self.jwt_login({"sub": "kermit"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: missing_claim: Missing "iss" claim', + r"^JWT validation failed: missing_claim: Missing [\"']iss[\"'] claim$", ) def test_login_iss_no_config(self) -> None: @@ -1166,18 +1287,18 @@ class JWTTestCase(unittest.HomeserverTestCase): channel = self.jwt_login({"sub": "kermit", "aud": "invalid"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: invalid_claim: Invalid claim "aud"', + r"^JWT validation failed: invalid_claim: Invalid claim [\"']aud[\"']$", ) # Not providing an audience. channel = self.jwt_login({"sub": "kermit"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: missing_claim: Missing "aud" claim', + r"^JWT validation failed: missing_claim: Missing [\"']aud[\"'] claim$", ) def test_login_aud_no_config(self) -> None: @@ -1185,9 +1306,9 @@ class JWTTestCase(unittest.HomeserverTestCase): channel = self.jwt_login({"sub": "kermit", "aud": "invalid"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: invalid_claim: Invalid claim "aud"', + r"^JWT validation failed: invalid_claim: Invalid claim [\"']aud[\"']$", ) def test_login_default_sub(self) -> None: @@ -1356,7 +1477,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.service = ApplicationService( id="unique_identifier", token="some_token", - sender="@asbot:example.com", + sender=UserID.from_string("@asbot:example.com"), namespaces={ ApplicationService.NS_USERS: [ {"regex": r"@as_user.*", "exclusive": False} @@ -1368,7 +1489,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.another_service = ApplicationService( id="another__identifier", token="another_token", - sender="@as2bot:example.com", + sender=UserID.from_string("@as2bot:example.com"), namespaces={ ApplicationService.NS_USERS: [ {"regex": r"@as2_user.*", "exclusive": False} @@ -1402,7 +1523,10 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): params = { "type": login.LoginRestServlet.APPSERVICE_TYPE, - "identifier": {"type": "m.id.user", "user": self.service.sender}, + "identifier": { + "type": "m.id.user", + "user": self.service.sender.to_string(), + }, } channel = self.make_request( b"POST", LOGIN_URL, params, access_token=self.service.token @@ -1473,7 +1597,7 @@ class UsernamePickerTestCase(HomeserverTestCase): def default_config(self) -> Dict[str, Any]: config = super().default_config() - config["public_baseurl"] = BASE_URL + config["public_baseurl"] = PUBLIC_BASEURL config["oidc_config"] = {} config["oidc_config"].update(TEST_OIDC_CONFIG) diff --git a/tests/rest/client/test_login_token_request.py b/tests/rest/client/test_login_token_request.py index fbacf9d869..202d2cf351 100644 --- a/tests/rest/client/test_login_token_request.py +++ b/tests/rest/client/test_login_token_request.py @@ -19,7 +19,7 @@ # # -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, login_token_request, versions diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 4060525efe..ec6760feea 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -24,14 +24,13 @@ import json import os import re import shutil -from typing import Any, BinaryIO, Dict, List, Optional, Sequence, Tuple, Type +from typing import Any, BinaryIO, ClassVar, Dict, List, Optional, Sequence, Tuple, Type from unittest.mock import MagicMock, Mock, patch from urllib import parse from urllib.parse import quote, urlencode from parameterized import parameterized, parameterized_class from PIL import Image as Image -from typing_extensions import ClassVar from twisted.internet import defer from twisted.internet._resolver import HostResolution @@ -39,14 +38,15 @@ from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred from twisted.internet.error import DNSLookupError from twisted.internet.interfaces import IAddress, IResolutionReceiver +from twisted.internet.testing import AccumulatingProtocol, MemoryReactor from twisted.python.failure import Failure -from twisted.test.proto_helpers import AccumulatingProtocol, MemoryReactor from twisted.web.http_headers import Headers from twisted.web.iweb import UNKNOWN_LENGTH, IResponse from twisted.web.resource import Resource from synapse.api.errors import HttpResponseException from synapse.api.ratelimiting import Ratelimiter +from synapse.config._base import Config from synapse.config.oembed import OEmbedEndpointConfig from synapse.http.client import MultipartResponse from synapse.http.types import QueryParams @@ -54,6 +54,7 @@ from synapse.logging.context import make_deferred_yieldable from synapse.media._base import FileInfo, ThumbnailInfo from synapse.media.thumbnailer import ThumbnailProvider from synapse.media.url_previewer import IMAGE_CACHE_EXPIRY_MS +from synapse.module_api import MediaUploadLimit from synapse.rest import admin from synapse.rest.client import login, media from synapse.server import HomeServer @@ -138,6 +139,7 @@ class MediaDomainBlockingTests(unittest.HomeserverTestCase): time_now_ms=clock.time_msec(), upload_name="test.png", filesystem_id=file_id, + sha256=file_id, ) ) self.register_user("user", "password") @@ -1006,7 +1008,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): data = base64.b64encode(SMALL_PNG) end_content = ( - b"" b'' b"" + b'' ) % (data,) channel = self.make_request( @@ -1618,6 +1620,63 @@ class MediaConfigTest(unittest.HomeserverTestCase): ) +class MediaConfigModuleCallbackTestCase(unittest.HomeserverTestCase): + servlets = [ + media.register_servlets, + admin.register_servlets, + login.register_servlets, + ] + + def make_homeserver( + self, reactor: ThreadedMemoryReactorClock, clock: Clock + ) -> HomeServer: + config = self.default_config() + + self.storage_path = self.mktemp() + self.media_store_path = self.mktemp() + os.mkdir(self.storage_path) + os.mkdir(self.media_store_path) + config["media_store_path"] = self.media_store_path + + provider_config = { + "module": "synapse.media.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + + config["media_storage_providers"] = [provider_config] + + return self.setup_test_homeserver(config=config) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user = self.register_user("user", "password") + self.tok = self.login("user", "password") + + hs.get_module_api().register_media_repository_callbacks( + get_media_config_for_user=self.get_media_config_for_user, + ) + + async def get_media_config_for_user( + self, + user_id: str, + ) -> Optional[JsonDict]: + # We echo back the user_id and set a custom upload size. + return {"m.upload.size": 1024, "user_id": user_id} + + def test_media_config(self) -> None: + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/config", + shorthand=False, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["m.upload.size"], 1024) + self.assertEqual(channel.json_body["user_id"], self.user) + + class RemoteDownloadLimiterTestCase(unittest.HomeserverTestCase): servlets = [ media.register_servlets, @@ -1895,7 +1954,7 @@ class RemoteDownloadLimiterTestCase(unittest.HomeserverTestCase): def test_file_download(self) -> None: content = io.BytesIO(b"file_to_stream") content_uri = self.get_success( - self.repo.create_content( + self.repo.create_or_update_content( "text/plain", "test_upload", content, @@ -2594,6 +2653,7 @@ class AuthenticatedMediaTestCase(unittest.HomeserverTestCase): time_now_ms=self.clock.time_msec(), upload_name="remote_test.png", filesystem_id=file_id, + sha256=file_id, ) ) @@ -2677,3 +2737,424 @@ class AuthenticatedMediaTestCase(unittest.HomeserverTestCase): access_token=self.tok, ) self.assertEqual(channel10.code, 200) + + def test_authenticated_media_etag(self) -> None: + """Test that ETag works correctly with authenticated media over client + APIs""" + + # upload some local media with authentication on + channel = self.make_request( + "POST", + "_matrix/media/v3/upload?filename=test_png_upload", + SMALL_PNG, + self.tok, + shorthand=False, + content_type=b"image/png", + custom_headers=[("Content-Length", str(67))], + ) + self.assertEqual(channel.code, 200) + res = channel.json_body.get("content_uri") + assert res is not None + uri = res.split("mxc://")[1] + + # Check standard media endpoint + self._check_caching(f"/download/{uri}") + + # check thumbnails as well + params = "?width=32&height=32&method=crop" + self._check_caching(f"/thumbnail/{uri}{params}") + + # Inject a piece of remote media. + file_id = "abcdefg12345" + file_info = FileInfo(server_name="lonelyIsland", file_id=file_id) + + media_storage = self.hs.get_media_repository().media_storage + + ctx = media_storage.store_into_file(file_info) + (f, fname) = self.get_success(ctx.__aenter__()) + f.write(SMALL_PNG) + self.get_success(ctx.__aexit__(None, None, None)) + + # we write the authenticated status when storing media, so this should pick up + # config and authenticate the media + self.get_success( + self.store.store_cached_remote_media( + origin="lonelyIsland", + media_id="52", + media_type="image/png", + media_length=1, + time_now_ms=self.clock.time_msec(), + upload_name="remote_test.png", + filesystem_id=file_id, + sha256=file_id, + ) + ) + + # ensure we have thumbnails for the non-dynamic code path + if self.extra_config == {"dynamic_thumbnails": False}: + self.get_success( + self.repo._generate_thumbnails( + "lonelyIsland", "52", file_id, "image/png" + ) + ) + + self._check_caching("/download/lonelyIsland/52") + + params = "?width=32&height=32&method=crop" + self._check_caching(f"/thumbnail/lonelyIsland/52{params}") + + def _check_caching(self, path: str) -> None: + """ + Checks that: + 1. fetching the path returns an ETag header + 2. refetching with the ETag returns a 304 without a body + 3. refetching with the ETag but through unauthenticated endpoint + returns 404 + """ + + # Request media over authenticated endpoint, should be found + channel1 = self.make_request( + "GET", + f"/_matrix/client/v1/media{path}", + access_token=self.tok, + shorthand=False, + ) + self.assertEqual(channel1.code, 200) + + # Should have a single ETag field + etags = channel1.headers.getRawHeaders("ETag") + self.assertIsNotNone(etags) + assert etags is not None # For mypy + self.assertEqual(len(etags), 1) + etag = etags[0] + + # Refetching with the etag should result in 304 and empty body. + channel2 = self.make_request( + "GET", + f"/_matrix/client/v1/media{path}", + access_token=self.tok, + shorthand=False, + custom_headers=[("If-None-Match", etag)], + ) + self.assertEqual(channel2.code, 304) + self.assertEqual(channel2.is_finished(), True) + self.assertNotIn("body", channel2.result) + + # Refetching with the etag but no access token should result in 404. + channel3 = self.make_request( + "GET", + f"/_matrix/media/r0{path}", + shorthand=False, + custom_headers=[("If-None-Match", etag)], + ) + self.assertEqual(channel3.code, 404) + + +class MediaUploadLimits(unittest.HomeserverTestCase): + """ + This test case simulates a homeserver with media upload limits configured. + """ + + servlets = [ + media.register_servlets, + login.register_servlets, + admin.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + config = self.default_config() + + self.storage_path = self.mktemp() + self.media_store_path = self.mktemp() + os.mkdir(self.storage_path) + os.mkdir(self.media_store_path) + config["media_store_path"] = self.media_store_path + + provider_config = { + "module": "synapse.media.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + + config["media_storage_providers"] = [provider_config] + + # These are the limits that we are testing + config["media_upload_limits"] = [ + {"time_period": "1d", "max_size": "1K"}, + {"time_period": "1w", "max_size": "3K"}, + ] + + return self.setup_test_homeserver(config=config) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.repo = hs.get_media_repository() + self.client = hs.get_federation_http_client() + self.store = hs.get_datastores().main + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + + def create_resource_dict(self) -> Dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + def upload_media(self, size: int) -> FakeChannel: + """Helper to upload media of a given size.""" + return self.make_request( + "POST", + "/_matrix/media/v3/upload", + content=b"0" * size, + access_token=self.tok, + shorthand=False, + content_type=b"text/plain", + custom_headers=[("Content-Length", str(size))], + ) + + def test_upload_under_limit(self) -> None: + """Test that uploading media under the limit works.""" + channel = self.upload_media(67) + self.assertEqual(channel.code, 200) + + def test_over_day_limit(self) -> None: + """Test that uploading media over the daily limit fails.""" + channel = self.upload_media(500) + self.assertEqual(channel.code, 200) + + channel = self.upload_media(800) + self.assertEqual(channel.code, 400) + + def test_under_daily_limit(self) -> None: + """Test that uploading media under the daily limit fails.""" + channel = self.upload_media(500) + self.assertEqual(channel.code, 200) + + self.reactor.advance(60 * 60 * 24) # Advance by one day + + # This will succeed as the daily limit has reset + channel = self.upload_media(800) + self.assertEqual(channel.code, 200) + + self.reactor.advance(60 * 60 * 24) # Advance by one day + + # ... and again + channel = self.upload_media(800) + self.assertEqual(channel.code, 200) + + def test_over_weekly_limit(self) -> None: + """Test that uploading media over the weekly limit fails.""" + channel = self.upload_media(900) + self.assertEqual(channel.code, 200) + + self.reactor.advance(60 * 60 * 24) # Advance by one day + + channel = self.upload_media(900) + self.assertEqual(channel.code, 200) + + self.reactor.advance(2 * 60 * 60 * 24) # Advance by one day + + channel = self.upload_media(900) + self.assertEqual(channel.code, 200) + + self.reactor.advance(2 * 60 * 60 * 24) # Advance by one day + + # This will fail as the weekly limit has been exceeded + channel = self.upload_media(900) + self.assertEqual(channel.code, 400) + + # Reset the weekly limit by advancing a week + self.reactor.advance(7 * 60 * 60 * 24) # Advance by 7 days + + # This will succeed as the weekly limit has reset + channel = self.upload_media(900) + self.assertEqual(channel.code, 200) + + +class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): + """ + This test case simulates a homeserver with media upload limits being overridden by the module API. + """ + + servlets = [ + media.register_servlets, + login.register_servlets, + admin.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + config = self.default_config() + + self.storage_path = self.mktemp() + self.media_store_path = self.mktemp() + os.mkdir(self.storage_path) + os.mkdir(self.media_store_path) + config["media_store_path"] = self.media_store_path + + provider_config = { + "module": "synapse.media.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + + config["media_storage_providers"] = [provider_config] + + # default limits to use + config["media_upload_limits"] = [ + {"time_period": "1d", "max_size": "1K"}, + {"time_period": "1w", "max_size": "3K"}, + ] + + return self.setup_test_homeserver(config=config) + + async def _get_media_upload_limits_for_user( + self, + user_id: str, + ) -> Optional[List[MediaUploadLimit]]: + # user1 has custom limits + if user_id == self.user1: + # n.b. we return these in increasing duration order and Synapse will need to sort them correctly + return [ + MediaUploadLimit( + time_period_ms=Config.parse_duration("1d"), max_bytes=5000 + ), + MediaUploadLimit( + time_period_ms=Config.parse_duration("1w"), max_bytes=15000 + ), + ] + # user2 has no limits + if user_id == self.user2: + return [] + # otherwise use default + return None + + async def _on_media_upload_limit_exceeded( + self, + user_id: str, + limit: MediaUploadLimit, + sent_bytes: int, + attempted_bytes: int, + ) -> None: + self.last_media_upload_limit_exceeded: Optional[dict[str, object]] = { + "user_id": user_id, + "limit": limit, + "sent_bytes": sent_bytes, + "attempted_bytes": attempted_bytes, + } + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.repo = hs.get_media_repository() + self.client = hs.get_federation_http_client() + self.store = hs.get_datastores().main + self.user1 = self.register_user("user1", "pass") + self.tok1 = self.login("user1", "pass") + self.user2 = self.register_user("user2", "pass") + self.tok2 = self.login("user2", "pass") + self.user3 = self.register_user("user3", "pass") + self.tok3 = self.login("user3", "pass") + self.last_media_upload_limit_exceeded = None + self.hs.get_module_api().register_media_repository_callbacks( + get_media_upload_limits_for_user=self._get_media_upload_limits_for_user, + on_media_upload_limit_exceeded=self._on_media_upload_limit_exceeded, + ) + + def create_resource_dict(self) -> Dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + def upload_media(self, size: int, tok: str) -> FakeChannel: + """Helper to upload media of a given size with a given token.""" + return self.make_request( + "POST", + "/_matrix/media/v3/upload", + content=b"0" * size, + access_token=tok, + shorthand=False, + content_type=b"text/plain", + custom_headers=[("Content-Length", str(size))], + ) + + def test_upload_under_limit(self) -> None: + """Test that uploading media under the limit works.""" + + # User 1 uploads 100 bytes + channel = self.upload_media(100, self.tok1) + self.assertEqual(channel.code, 200) + + # User 2 (unlimited) uploads 100 bytes + channel = self.upload_media(100, self.tok2) + self.assertEqual(channel.code, 200) + + # User 3 (default) uploads 100 bytes + channel = self.upload_media(100, self.tok3) + self.assertEqual(channel.code, 200) + + self.assertEqual(self.last_media_upload_limit_exceeded, None) + + def test_uses_custom_limit(self) -> None: + """Test that uploading media over the module provided daily limit fails.""" + + # User 1 uploads 3000 bytes + channel = self.upload_media(3000, self.tok1) + self.assertEqual(channel.code, 200) + + # User 1 attempts to upload 4000 bytes taking it over the limit + channel = self.upload_media(4000, self.tok1) + self.assertEqual(channel.code, 400) + assert self.last_media_upload_limit_exceeded is not None + self.assertEqual(self.last_media_upload_limit_exceeded["user_id"], self.user1) + self.assertEqual( + self.last_media_upload_limit_exceeded["limit"], + MediaUploadLimit( + max_bytes=5000, time_period_ms=Config.parse_duration("1d") + ), + ) + self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 3000) + self.assertEqual(self.last_media_upload_limit_exceeded["attempted_bytes"], 4000) + + # User 1 attempts to upload 20000 bytes which is over the weekly limit + # This tests that the limits have been sorted as expected + channel = self.upload_media(20000, self.tok1) + self.assertEqual(channel.code, 400) + assert self.last_media_upload_limit_exceeded is not None + self.assertEqual(self.last_media_upload_limit_exceeded["user_id"], self.user1) + self.assertEqual( + self.last_media_upload_limit_exceeded["limit"], + MediaUploadLimit( + max_bytes=15000, time_period_ms=Config.parse_duration("1w") + ), + ) + self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 3000) + self.assertEqual( + self.last_media_upload_limit_exceeded["attempted_bytes"], 20000 + ) + + def test_uses_unlimited(self) -> None: + """Test that unlimited user is not limited when module returns [].""" + # User 2 uploads 10000 bytes which is over the default limit + channel = self.upload_media(10000, self.tok2) + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_media_upload_limit_exceeded, None) + + def test_uses_defaults(self) -> None: + """Test that the default limits are applied when module returned None.""" + # User 3 uploads 500 bytes + channel = self.upload_media(500, self.tok3) + self.assertEqual(channel.code, 200) + + # User 3 uploads 800 bytes which is over the limit + channel = self.upload_media(800, self.tok3) + self.assertEqual(channel.code, 400) + assert self.last_media_upload_limit_exceeded is not None + self.assertEqual(self.last_media_upload_limit_exceeded["user_id"], self.user3) + self.assertEqual( + self.last_media_upload_limit_exceeded["limit"], + MediaUploadLimit( + max_bytes=1024, time_period_ms=Config.parse_duration("1d") + ), + ) + self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 500) + self.assertEqual(self.last_media_upload_limit_exceeded["attempted_bytes"], 800) diff --git a/tests/rest/client/test_models.py b/tests/rest/client/test_models.py index f14585ccac..75479e6235 100644 --- a/tests/rest/client/test_models.py +++ b/tests/rest/client/test_models.py @@ -19,8 +19,7 @@ # # import unittest as stdlib_unittest - -from typing_extensions import Literal +from typing import Literal from synapse._pydantic_compat import BaseModel, ValidationError from synapse.types.rest.client import EmailRequestTokenBody diff --git a/tests/rest/client/test_mutual_rooms.py b/tests/rest/client/test_mutual_rooms.py index 637722ca0a..2e37284680 100644 --- a/tests/rest/client/test_mutual_rooms.py +++ b/tests/rest/client/test_mutual_rooms.py @@ -20,7 +20,7 @@ # from urllib.parse import quote -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.rest.client import login, mutual_rooms, room diff --git a/tests/rest/client/test_notifications.py b/tests/rest/client/test_notifications.py index e4b0455ce8..ec66567817 100644 --- a/tests/rest/client/test_notifications.py +++ b/tests/rest/client/test_notifications.py @@ -21,7 +21,7 @@ from typing import List, Optional, Tuple from unittest.mock import AsyncMock, Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.rest.client import login, notifications, receipts, room diff --git a/tests/rest/client/test_owned_state.py b/tests/rest/client/test_owned_state.py index 5fb5767676..386b95d616 100644 --- a/tests/rest/client/test_owned_state.py +++ b/tests/rest/client/test_owned_state.py @@ -2,7 +2,7 @@ from http import HTTPStatus from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import Codes from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions diff --git a/tests/rest/client/test_password_policy.py b/tests/rest/client/test_password_policy.py index f0ef733f7b..33bab684e3 100644 --- a/tests/rest/client/test_password_policy.py +++ b/tests/rest/client/test_password_policy.py @@ -21,7 +21,7 @@ from http import HTTPStatus -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import LoginType from synapse.api.errors import Codes diff --git a/tests/rest/client/test_power_levels.py b/tests/rest/client/test_power_levels.py index 1584c2e96c..39ea9acef6 100644 --- a/tests/rest/client/test_power_levels.py +++ b/tests/rest/client/test_power_levels.py @@ -20,7 +20,7 @@ # from http import HTTPStatus -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.errors import Codes from synapse.events.utils import CANONICALJSON_MAX_INT, CANONICALJSON_MIN_INT diff --git a/tests/rest/client/test_presence.py b/tests/rest/client/test_presence.py index 5ced8319e1..7138cc92c2 100644 --- a/tests/rest/client/test_presence.py +++ b/tests/rest/client/test_presence.py @@ -20,7 +20,7 @@ from http import HTTPStatus from unittest.mock import AsyncMock, Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.handlers.presence import PresenceHandler from synapse.rest.client import presence @@ -29,6 +29,7 @@ from synapse.types import UserID from synapse.util import Clock from tests import unittest +from tests.unittest import override_config class PresenceTestCase(unittest.HomeserverTestCase): @@ -95,3 +96,54 @@ class PresenceTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertEqual(self.presence_handler.set_state.call_count, 0) + + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def test_put_presence_over_ratelimit(self) -> None: + """ + Multiple PUTs to the status endpoint without sufficient delay will be rate limited. + """ + self.hs.config.server.presence_enabled = True + + body = {"presence": "here", "status_msg": "beep boop"} + channel = self.make_request( + "PUT", "/presence/%s/status" % (self.user_id,), body + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + + body = {"presence": "here", "status_msg": "beep boop"} + channel = self.make_request( + "PUT", "/presence/%s/status" % (self.user_id,), body + ) + + self.assertEqual(channel.code, HTTPStatus.TOO_MANY_REQUESTS) + self.assertEqual(self.presence_handler.set_state.call_count, 1) + + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def test_put_presence_within_ratelimit(self) -> None: + """ + Multiple PUTs to the status endpoint with sufficient delay should all call set_state. + """ + self.hs.config.server.presence_enabled = True + + body = {"presence": "here", "status_msg": "beep boop"} + channel = self.make_request( + "PUT", "/presence/%s/status" % (self.user_id,), body + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + + # Advance time a sufficient amount to avoid rate limiting. + self.reactor.advance(30) + + body = {"presence": "here", "status_msg": "beep boop"} + channel = self.make_request( + "PUT", "/presence/%s/status" % (self.user_id,), body + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(self.presence_handler.set_state.call_count, 2) diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index a92713d220..936e573bcd 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -21,20 +21,25 @@ """Tests REST events for /profile paths.""" +import logging import urllib.parse from http import HTTPStatus from typing import Any, Dict, Optional -from twisted.test.proto_helpers import MemoryReactor +from canonicaljson import encode_canonical_json + +from twisted.internet.testing import MemoryReactor from synapse.api.errors import Codes from synapse.rest import admin from synapse.rest.client import login, profile, room from synapse.server import HomeServer +from synapse.storage.databases.main.profile import MAX_PROFILE_SIZE from synapse.types import UserID from synapse.util import Clock from tests import unittest +from tests.utils import USE_POSTGRES_FOR_TESTS class ProfileTestCase(unittest.HomeserverTestCase): @@ -480,6 +485,299 @@ class ProfileTestCase(unittest.HomeserverTestCase): # The client requested ?propagate=true, so it should have happened. self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") + def test_get_missing_custom_field(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) + + def test_get_missing_custom_field_invalid_field_name(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}/[custom_field]", + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + def test_get_custom_field_rejects_bad_username(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{urllib.parse.quote('@alice:')}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + def test_set_custom_field(self) -> None: + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", + content={"custom_field": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"custom_field": "test"}) + + # Overwriting the field should work. + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", + content={"custom_field": "new_Value"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"custom_field": "new_Value"}) + + # Deleting the field should work. + channel = self.make_request( + "DELETE", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", + content={}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) + + def test_non_string(self) -> None: + """Non-string fields are supported for custom fields.""" + fields = { + "bool_field": True, + "array_field": ["test"], + "object_field": {"test": "test"}, + "numeric_field": 1, + "null_field": None, + } + + for key, value in fields.items(): + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/{key}", + content={key: value}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"displayname": "owner", **fields}) + + # Check getting individual fields works. + for key, value in fields.items(): + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}/{key}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {key: value}) + + def test_set_custom_field_noauth(self) -> None: + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", + content={"custom_field": "test"}, + ) + self.assertEqual(channel.code, 401, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.MISSING_TOKEN) + + def test_set_custom_field_size(self) -> None: + """ + Attempts to set a custom field name that is too long should get a 400 error. + """ + # Key is missing. + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/", + content={"": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + # Single key is too large. + key = "c" * 500 + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/{key}", + content={key: "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.KEY_TOO_LARGE) + + channel = self.make_request( + "DELETE", + f"/_matrix/client/v3/profile/{self.owner}/{key}", + content={key: "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.KEY_TOO_LARGE) + + # Key doesn't match body. + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", + content={"diff_key": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.MISSING_PARAM) + + def test_set_custom_field_profile_too_long(self) -> None: + """ + Attempts to set a custom field that would push the overall profile too large. + """ + # FIXME: Because we emit huge SQL log lines and trial can't handle these, + # sometimes (flakily) failing the test run, + # disable SQL logging for this test. + # ref: https://github.com/twisted/twisted/issues/12482 + # To remove this, we would need to fix the above issue and + # update, including in olddeps (so several years' wait). + sql_logger = logging.getLogger("synapse.storage.SQL") + sql_logger_was_disabled = sql_logger.disabled + sql_logger.disabled = True + try: + # Get right to the boundary: + # len("displayname") + len("owner") + 5 = 21 for the displayname + # 1 + 65498 + 5 for key "a" = 65504 + # 2 braces, 1 comma + # 3 + 21 + 65498 = 65522 < 65536. + key = "a" + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/{key}", + content={key: "a" * 65498}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Get the entire profile. + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + canonical_json = encode_canonical_json(channel.json_body) + # 6 is the minimum bytes to store a value: 4 quotes, 1 colon, 1 comma, an empty key. + # Be one below that so we can prove we're at the boundary. + self.assertEqual(len(canonical_json), MAX_PROFILE_SIZE - 8) + + # Postgres stores JSONB with whitespace, while SQLite doesn't. + if USE_POSTGRES_FOR_TESTS: + ADDITIONAL_CHARS = 0 + else: + ADDITIONAL_CHARS = 1 + + # The next one should fail, note the value has a (JSON) length of 2. + key = "b" + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/{key}", + content={key: "1" + "a" * ADDITIONAL_CHARS}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.PROFILE_TOO_LARGE) + + # Setting an avatar or (longer) display name should not work. + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/displayname", + content={"displayname": "owner12345678" + "a" * ADDITIONAL_CHARS}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.PROFILE_TOO_LARGE) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/avatar_url", + content={"avatar_url": "mxc://foo/bar"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.PROFILE_TOO_LARGE) + + # Removing a single byte should work. + key = "b" + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/{key}", + content={key: "" + "a" * ADDITIONAL_CHARS}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Finally, setting a field that already exists to a value that is <= in length should work. + key = "a" + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/{key}", + content={key: ""}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + finally: + sql_logger.disabled = sql_logger_was_disabled + + def test_set_custom_field_displayname(self) -> None: + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/displayname", + content={"displayname": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + displayname = self._get_displayname() + self.assertEqual(displayname, "test") + + def test_set_custom_field_avatar_url(self) -> None: + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.owner}/avatar_url", + content={"avatar_url": "mxc://test/good"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + avatar_url = self._get_avatar_url() + self.assertEqual(avatar_url, "mxc://test/good") + + def test_set_custom_field_other(self) -> None: + """Setting someone else's profile field should fail""" + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/profile/{self.other}/custom_field", + content={"custom_field": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + def _setup_local_files(self, names_and_props: Dict[str, Dict[str, Any]]) -> None: """Stores metadata about files in the database. diff --git a/tests/rest/client/test_push_rule_attrs.py b/tests/rest/client/test_push_rule_attrs.py index 9da0e7982f..53c36b7a9c 100644 --- a/tests/rest/client/test_push_rule_attrs.py +++ b/tests/rest/client/test_push_rule_attrs.py @@ -18,6 +18,8 @@ # [This file includes modifications made by New Vector Limited] # # +from http import HTTPStatus + import synapse from synapse.api.errors import Codes from synapse.rest.client import login, push_rule, room @@ -486,3 +488,23 @@ class PushRuleAttributesTestCase(HomeserverTestCase): }, channel.json_body, ) + + def test_no_user_defined_postcontent_rules(self) -> None: + """ + Tests that clients are not permitted to create MSC4306 `postcontent` rules. + """ + self.register_user("bob", "pass") + token = self.login("bob", "pass") + + channel = self.make_request( + "PUT", + "/pushrules/global/postcontent/some.user.rule", + {}, + access_token=token, + ) + + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) + self.assertEqual( + Codes.INVALID_PARAM, + channel.json_body["errcode"], + ) diff --git a/tests/rest/client/test_read_marker.py b/tests/rest/client/test_read_marker.py index 0b4ad685b3..a27eb9453b 100644 --- a/tests/rest/client/test_read_marker.py +++ b/tests/rest/client/test_read_marker.py @@ -18,7 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EventTypes diff --git a/tests/rest/client/test_receipts.py b/tests/rest/client/test_receipts.py index f0648289f1..ae4818c412 100644 --- a/tests/rest/client/test_receipts.py +++ b/tests/rest/client/test_receipts.py @@ -21,7 +21,7 @@ from http import HTTPStatus from typing import Optional -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EduTypes, EventTypes, HistoryVisibility, ReceiptTypes diff --git a/tests/rest/client/test_redactions.py b/tests/rest/client/test_redactions.py index b25e184786..d435a9e393 100644 --- a/tests/rest/client/test_redactions.py +++ b/tests/rest/client/test_redactions.py @@ -22,7 +22,7 @@ from typing import List, Optional from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, RelationTypes from synapse.api.room_versions import RoomVersion, RoomVersions diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index c091f403cc..70e005caf4 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -20,13 +20,12 @@ # # import datetime +import importlib.resources as importlib_resources import os from typing import Any, Dict, List, Tuple from unittest.mock import AsyncMock -import pkg_resources - -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import ( @@ -39,7 +38,7 @@ from synapse.appservice import ApplicationService from synapse.rest.client import account, account_validity, login, logout, register, sync from synapse.server import HomeServer from synapse.storage._base import db_to_json -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util import Clock from tests import unittest @@ -75,7 +74,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), ) self.hs.get_datastores().main.services_cache.append(appservice) @@ -99,7 +98,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), ) self.hs.get_datastores().main.services_cache.append(appservice) @@ -120,6 +119,34 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 401, msg=channel.result) + def test_POST_appservice_msc4190_enabled(self) -> None: + # With MSC4190 enabled, the registration should *not* return an access token + user_id = "@as_user_kermit:test" + as_token = "i_am_an_app_service" + + appservice = ApplicationService( + as_token, + id="1234", + namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, + sender=UserID.from_string("@as:test"), + msc4190_device_management=True, + ) + + self.hs.get_datastores().main.services_cache.append(appservice) + request_data = { + "username": "as_user_kermit", + "type": APP_SERVICE_REGISTRATION_TYPE, + } + + channel = self.make_request( + b"POST", self.url + b"?access_token=i_am_an_app_service", request_data + ) + + self.assertEqual(channel.code, 200, msg=channel.result) + det_data = {"user_id": user_id, "home_server": self.hs.hostname} + self.assertLessEqual(det_data.items(), channel.json_body.items()) + self.assertNotIn("access_token", channel.json_body) + def test_POST_bad_password(self) -> None: request_data = {"username": "kermit", "password": 666} channel = self.make_request(b"POST", self.url, request_data) @@ -953,11 +980,12 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase): # Email config. + templates = ( + importlib_resources.files("synapse").joinpath("res").joinpath("templates") + ) config["email"] = { "enable_notifs": True, - "template_dir": os.path.abspath( - pkg_resources.resource_filename("synapse", "res/templates") - ), + "template_dir": os.path.abspath(str(templates)), "expiry_template_html": "notice_expiry.html", "expiry_template_text": "notice_expiry.txt", "notif_template_html": "notif_mail.html", diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index f5a7602d0a..fd1e87296c 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -23,7 +23,7 @@ import urllib.parse from typing import Any, Callable, Dict, List, Optional, Tuple from unittest.mock import AsyncMock, patch -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import AccountDataTypes, EventTypes, RelationTypes from synapse.rest import admin @@ -1181,7 +1181,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): bundled_aggregations, ) - self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 6) + self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 7) def test_thread(self) -> None: """ @@ -1226,21 +1226,21 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): # The "user" sent the root event and is making queries for the bundled # aggregations: they have participated. - self._test_bundled_aggregations(RelationTypes.THREAD, _gen_assert(True), 6) + self._test_bundled_aggregations(RelationTypes.THREAD, _gen_assert(True), 7) # The "user2" sent replies in the thread and is making queries for the # bundled aggregations: they have participated. # # Note that this re-uses some cached values, so the total number of # queries is much smaller. self._test_bundled_aggregations( - RelationTypes.THREAD, _gen_assert(True), 3, access_token=self.user2_token + RelationTypes.THREAD, _gen_assert(True), 4, access_token=self.user2_token ) # A user with no interactions with the thread: they have not participated. user3_id, user3_token = self._create_user("charlie") self.helper.join(self.room, user=user3_id, tok=user3_token) self._test_bundled_aggregations( - RelationTypes.THREAD, _gen_assert(False), 3, access_token=user3_token + RelationTypes.THREAD, _gen_assert(False), 4, access_token=user3_token ) def test_thread_with_bundled_aggregations_for_latest(self) -> None: @@ -1287,7 +1287,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): bundled_aggregations["latest_event"].get("unsigned"), ) - self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 6) + self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 7) def test_nested_thread(self) -> None: """ diff --git a/tests/rest/client/test_rendezvous.py b/tests/rest/client/test_rendezvous.py index ab701680a6..01401f73da 100644 --- a/tests/rest/client/test_rendezvous.py +++ b/tests/rest/client/test_rendezvous.py @@ -22,7 +22,7 @@ from typing import Dict from urllib.parse import urlparse -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource from synapse.rest.client import rendezvous @@ -117,10 +117,11 @@ class RendezvousServletTestCase(unittest.HomeserverTestCase): headers = dict(channel.headers.getAllRawHeaders()) self.assertIn(b"ETag", headers) self.assertIn(b"Expires", headers) + self.assertIn(b"Content-Length", headers) self.assertEqual(headers[b"Content-Type"], [b"application/json"]) self.assertEqual(headers[b"Access-Control-Allow-Origin"], [b"*"]) self.assertEqual(headers[b"Access-Control-Expose-Headers"], [b"etag"]) - self.assertEqual(headers[b"Cache-Control"], [b"no-store"]) + self.assertEqual(headers[b"Cache-Control"], [b"no-store, no-transform"]) self.assertEqual(headers[b"Pragma"], [b"no-cache"]) self.assertIn("url", channel.json_body) self.assertTrue(channel.json_body["url"].startswith("https://")) @@ -141,9 +142,10 @@ class RendezvousServletTestCase(unittest.HomeserverTestCase): self.assertEqual(headers[b"ETag"], [etag]) self.assertIn(b"Expires", headers) self.assertEqual(headers[b"Content-Type"], [b"text/plain"]) + self.assertEqual(headers[b"Content-Length"], [b"7"]) self.assertEqual(headers[b"Access-Control-Allow-Origin"], [b"*"]) self.assertEqual(headers[b"Access-Control-Expose-Headers"], [b"etag"]) - self.assertEqual(headers[b"Cache-Control"], [b"no-store"]) + self.assertEqual(headers[b"Cache-Control"], [b"no-store, no-transform"]) self.assertEqual(headers[b"Pragma"], [b"no-cache"]) self.assertEqual(channel.text_body, "foo=bar") diff --git a/tests/rest/client/test_reporting.py b/tests/rest/client/test_reporting.py index 723553979f..5e5af34b42 100644 --- a/tests/rest/client/test_reporting.py +++ b/tests/rest/client/test_reporting.py @@ -18,8 +18,9 @@ # [This file includes modifications made by New Vector Limited] # # +from typing import Optional -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.rest.client import login, reporting, room @@ -28,6 +29,7 @@ from synapse.types import JsonDict from synapse.util import Clock from tests import unittest +from tests.unittest import override_config class ReportEventTestCase(unittest.HomeserverTestCase): @@ -80,6 +82,11 @@ class ReportEventTestCase(unittest.HomeserverTestCase): data = {"reason": None, "score": None} self._assert_status(400, data) + @override_config({"experimental_features": {"msc4277_enabled": True}}) + def test_score_str(self) -> None: + data = {"score": "string"} + self._assert_status(200, data) + def test_cannot_report_nonexistent_event(self) -> None: """ Tests that we don't accept event reports for events which do not exist. @@ -97,6 +104,19 @@ class ReportEventTestCase(unittest.HomeserverTestCase): msg=channel.result["body"], ) + @override_config({"experimental_features": {"msc4277_enabled": True}}) + def test_event_existence_hidden(self) -> None: + """ + Tests that the requester cannot infer the existence of an event. + """ + channel = self.make_request( + "POST", + f"rooms/{self.room_id}/report/$nonsenseeventid:test", + {"reason": "i am very sad"}, + access_token=self.other_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.result["body"]) + def test_cannot_report_event_if_not_in_room(self) -> None: """ Tests that we don't accept event reports for events that exist, but for which @@ -192,6 +212,20 @@ class ReportRoomTestCase(unittest.HomeserverTestCase): msg=channel.result["body"], ) + @override_config({"experimental_features": {"msc4277_enabled": True}}) + def test_room_existence_hidden(self) -> None: + """ + Tests that the requester cannot infer the existence of a room. + """ + channel = self.make_request( + "POST", + "/_matrix/client/v3/rooms/!bloop:example.org/report", + {"reason": "i am very sad"}, + access_token=self.other_user_tok, + shorthand=False, + ) + self.assertEqual(200, channel.code, msg=channel.result["body"]) + def _assert_status(self, response_status: int, data: JsonDict) -> None: channel = self.make_request( "POST", @@ -201,3 +235,91 @@ class ReportRoomTestCase(unittest.HomeserverTestCase): shorthand=False, ) self.assertEqual(response_status, channel.code, msg=channel.result["body"]) + + +class ReportUserTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + reporting.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.other_user = self.register_user("user", "pass") + self.other_user_tok = self.login("user", "pass") + + self.target_user_id = self.register_user("target_user", "pass") + + def test_reason_str(self) -> None: + data = {"reason": "this makes me sad"} + self._assert_status(200, data) + + rows = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_onecol( + table="user_reports", + keyvalues={"target_user_id": self.target_user_id}, + retcol="id", + desc="get_user_report_ids", + ) + ) + self.assertEqual(len(rows), 1) + + def test_no_reason(self) -> None: + data = {"not_reason": "for typechecking"} + self._assert_status(400, data) + + def test_reason_nonstring(self) -> None: + data = {"reason": 42} + self._assert_status(400, data) + + def test_reason_null(self) -> None: + data = {"reason": None} + self._assert_status(400, data) + + def test_reason_long(self) -> None: + data = {"reason": "x" * 1001} + self._assert_status(400, data) + + def test_cannot_report_nonlocal_user(self) -> None: + """ + Tests that we ignore reports for nonlocal users. + """ + target_user_id = "@bloop:example.org" + data = {"reason": "i am very sad"} + self._assert_status(200, data, target_user_id) + self._assert_no_reports_for_user(target_user_id) + + def test_can_report_nonexistent_user(self) -> None: + """ + Tests that we ignore reports for nonexistent users. + """ + target_user_id = f"@bloop:{self.hs.hostname}" + data = {"reason": "i am very sad"} + self._assert_status(200, data, target_user_id) + self._assert_no_reports_for_user(target_user_id) + + def _assert_no_reports_for_user(self, target_user_id: str) -> None: + rows = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_onecol( + table="user_reports", + keyvalues={"target_user_id": target_user_id}, + retcol="id", + desc="get_user_report_ids", + ) + ) + self.assertEqual(len(rows), 0) + + def _assert_status( + self, response_status: int, data: JsonDict, user_id: Optional[str] = None + ) -> None: + if user_id is None: + user_id = self.target_user_id + channel = self.make_request( + "POST", + f"/_matrix/client/v3/users/{user_id}/report", + data, + access_token=self.other_user_tok, + shorthand=False, + ) + self.assertEqual(response_status, channel.code, msg=channel.result["body"]) diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py index 1e5a1b0a4d..24b007f779 100644 --- a/tests/rest/client/test_retention.py +++ b/tests/rest/client/test_retention.py @@ -20,7 +20,7 @@ from typing import Any, Dict from unittest.mock import Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes from synapse.rest import admin diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 07600418ed..d3b5e26132 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -25,14 +25,13 @@ import json from http import HTTPStatus -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, Union from unittest.mock import AsyncMock, Mock, call, patch from urllib import parse as urlparse from parameterized import param, parameterized -from typing_extensions import Literal -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import ( @@ -44,8 +43,9 @@ from synapse.api.constants import ( RoomTypes, ) from synapse.api.errors import Codes, HttpResponseException +from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService -from synapse.events import EventBase +from synapse.events import EventBase, make_event_from_dict from synapse.events.snapshot import EventContext from synapse.rest import admin from synapse.rest.client import ( @@ -68,6 +68,7 @@ from tests.http.server._base import make_request_with_cancellation_test from tests.storage.test_stream import PaginationTestCase from tests.test_utils.event_injection import create_event from tests.unittest import override_config +from tests.utils import default_config PATH_PREFIX = b"/_matrix/client/api/v1" @@ -552,6 +553,51 @@ class RoomStateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.result["body"]) self.assertEqual(channel.json_body, {"membership": "join"}) + def test_get_state_format_content(self) -> None: + """Test response of a `/rooms/$room_id/state/$event_type?format=content` request.""" + room_id = self.helper.create_room_as(self.user_id) + channel1 = self.make_request( + "GET", + "/rooms/%s/state/m.room.member/%s?format=content" + % ( + room_id, + self.user_id, + ), + ) + self.assertEqual(channel1.code, HTTPStatus.OK, channel1.json_body) + self.assertEqual(channel1.json_body, {"membership": "join"}) + channel2 = self.make_request( + "GET", + "/rooms/%s/state/m.room.member/%s" + % ( + room_id, + self.user_id, + ), + ) + self.assertEqual(channel2.code, HTTPStatus.OK, channel2.json_body) + # "content" is the default format. + self.assertEqual(channel1.json_body, channel2.json_body) + + def test_get_state_format_event(self) -> None: + """Test response of a `/rooms/$room_id/state/$event_type?format=event` request.""" + room_id = self.helper.create_room_as(self.user_id) + channel = self.make_request( + "GET", + "/rooms/%s/state/m.room.member/%s?format=event" + % ( + room_id, + self.user_id, + ), + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + self.assertEqual(channel.json_body["content"], {"membership": "join"}) + self.assertEqual(channel.json_body["room_id"], room_id) + self.assertRegex(channel.json_body["event_id"], r"\$.+") + self.assertEqual(channel.json_body["type"], "m.room.member") + self.assertEqual(channel.json_body["sender"], self.user_id) + self.assertEqual(channel.json_body["state_key"], self.user_id) + self.assertTrue(type(channel.json_body["origin_server_ts"]) is int) + class RoomsMemberListTestCase(RoomBase): """Tests /rooms/$room_id/members/list REST events.""" @@ -742,7 +788,7 @@ class RoomsCreateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertTrue("room_id" in channel.json_body) assert channel.resource_usage is not None - self.assertEqual(33, channel.resource_usage.db_txn_count) + self.assertEqual(35, channel.resource_usage.db_txn_count) def test_post_room_initial_state(self) -> None: # POST with initial_state config key, expect new room id @@ -755,7 +801,60 @@ class RoomsCreateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertTrue("room_id" in channel.json_body) assert channel.resource_usage is not None - self.assertEqual(35, channel.resource_usage.db_txn_count) + self.assertEqual(37, channel.resource_usage.db_txn_count) + + def test_post_room_topic(self) -> None: + # POST with topic key, expect new room id + channel = self.make_request("POST", "/createRoom", b'{"topic":"shenanigans"}') + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertTrue("room_id" in channel.json_body) + room_id = channel.json_body["room_id"] + + # GET topic event, expect content from topic key + channel = self.make_request("GET", "/rooms/%s/state/m.room.topic" % (room_id,)) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertEqual( + {"topic": "shenanigans", "m.topic": {"m.text": [{"body": "shenanigans"}]}}, + channel.json_body, + ) + + def test_post_room_topic_initial_state(self) -> None: + # POST with m.room.topic in initial state, expect new room id + channel = self.make_request( + "POST", + "/createRoom", + b'{"initial_state":[{"type": "m.room.topic", "content": {"topic": "foobar"}}]}', + ) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertTrue("room_id" in channel.json_body) + room_id = channel.json_body["room_id"] + + # GET topic event, expect content from initial state + channel = self.make_request("GET", "/rooms/%s/state/m.room.topic" % (room_id,)) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertEqual( + {"topic": "foobar"}, + channel.json_body, + ) + + def test_post_room_topic_overriding_initial_state(self) -> None: + # POST with m.room.topic in initial state and topic key, expect new room id + channel = self.make_request( + "POST", + "/createRoom", + b'{"initial_state":[{"type": "m.room.topic", "content": {"topic": "foobar"}}], "topic":"shenanigans"}', + ) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertTrue("room_id" in channel.json_body) + room_id = channel.json_body["room_id"] + + # GET topic event, expect content from topic key + channel = self.make_request("GET", "/rooms/%s/state/m.room.topic" % (room_id,)) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertEqual( + {"topic": "shenanigans", "m.topic": {"m.text": [{"body": "shenanigans"}]}}, + channel.json_body, + ) def test_post_room_visibility_key(self) -> None: # POST with visibility config key, expect new room id @@ -1337,17 +1436,13 @@ class RoomJoinTestCase(RoomBase): "POST", f"/join/{self.room1}", access_token=self.tok2 ) self.assertEqual(channel.code, 403) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") channel = self.make_request( "POST", f"/rooms/{self.room1}/join", access_token=self.tok2 ) self.assertEqual(channel.code, 403) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") def test_suspended_user_cannot_knock_on_room(self) -> None: # set the user as suspended @@ -1361,9 +1456,7 @@ class RoomJoinTestCase(RoomBase): shorthand=False, ) self.assertEqual(channel.code, 403) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") def test_suspended_user_cannot_invite_to_room(self) -> None: # set the user as suspended @@ -1376,9 +1469,24 @@ class RoomJoinTestCase(RoomBase): access_token=self.tok1, content={"user_id": self.user2}, ) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") + + def test_suspended_user_can_leave_room(self) -> None: + channel = self.make_request( + "POST", f"/join/{self.room1}", access_token=self.tok1 ) + self.assertEqual(channel.code, 200) + + # set the user as suspended + self.get_success(self.store.set_user_suspended_status(self.user1, True)) + + # leave room + channel = self.make_request( + "POST", + f"/rooms/{self.room1}/leave", + access_token=self.tok1, + ) + self.assertEqual(channel.code, 200) class RoomAppserviceTsParamTestCase(unittest.HomeserverTestCase): @@ -1417,7 +1525,7 @@ class RoomAppserviceTsParamTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) mock_load_appservices = Mock(return_value=[self.appservice]) @@ -2137,7 +2245,7 @@ class RoomMessageListTestCase(RoomBase): self.room_id = self.helper.create_room_as(self.user_id) def test_topo_token_is_accepted(self) -> None: - token = "t1-0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2148,7 +2256,7 @@ class RoomMessageListTestCase(RoomBase): self.assertTrue("end" in channel.json_body) def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: - token = "s0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2390,6 +2498,41 @@ class RoomDelayedEventTestCase(RoomBase): ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "rc_message": {"per_second": 1, "burst_count": 2}, + } + ) + def test_add_delayed_event_ratelimit(self) -> None: + """Test that requests to schedule new delayed events are ratelimited by a RateLimiter, + which ratelimits them correctly, including by not limiting when the requester is + exempt from ratelimiting. + """ + + # Test that new delayed events are correctly ratelimited. + args = ( + "POST", + ( + "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=2000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + # Test that the new delayed events aren't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + class RoomSearchTestCase(unittest.HomeserverTestCase): servlets = [ @@ -2557,6 +2700,11 @@ class PublicRoomsRoomTypeFilterTestCase(unittest.HomeserverTestCase): tok=self.token, ) + def default_config(self) -> JsonDict: + config = default_config("test") + config["room_list_publication_rules"] = [{"action": "allow"}] + return config + def make_public_rooms_request( self, room_types: Optional[List[Union[str, None]]], @@ -3998,10 +4146,25 @@ class UserSuspensionTests(unittest.HomeserverTestCase): self.user2 = self.register_user("teresa", "hackme") self.tok2 = self.login("teresa", "hackme") - self.room1 = self.helper.create_room_as(room_creator=self.user1, tok=self.tok1) + self.admin = self.register_user("admin", "pass", True) + self.admin_tok = self.login("admin", "pass") + + self.room1 = self.helper.create_room_as( + room_creator=self.user1, tok=self.tok1, room_version="11" + ) self.store = hs.get_datastores().main - def test_suspended_user_cannot_send_message_to_room(self) -> None: + self.room2 = self.helper.create_room_as( + room_creator=self.user1, is_public=False, tok=self.tok1 + ) + self.helper.send_state( + self.room2, + EventTypes.RoomEncryption, + {EventContentFields.ENCRYPTION_ALGORITHM: "m.megolm.v1.aes-sha2"}, + tok=self.tok1, + ) + + def test_suspended_user_cannot_send_message_to_public_room(self) -> None: # set the user as suspended self.get_success(self.store.set_user_suspended_status(self.user1, True)) @@ -4011,9 +4174,25 @@ class UserSuspensionTests(unittest.HomeserverTestCase): access_token=self.tok1, content={"body": "hello", "msgtype": "m.text"}, ) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") + + def test_suspended_user_cannot_send_message_to_encrypted_room(self) -> None: + channel = self.make_request( + "PUT", + f"/_synapse/admin/v1/suspend/{self.user1}", + {"suspend": True}, + access_token=self.admin_tok, ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {f"user_{self.user1}_suspended": True}) + + channel = self.make_request( + "PUT", + f"/rooms/{self.room2}/send/m.room.encrypted/1", + access_token=self.tok1, + content={}, + ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") def test_suspended_user_cannot_change_profile_data(self) -> None: # set the user as suspended @@ -4026,9 +4205,7 @@ class UserSuspensionTests(unittest.HomeserverTestCase): content={"avatar_url": "mxc://matrix.org/wefh34uihSDRGhw34"}, shorthand=False, ) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") channel2 = self.make_request( "PUT", @@ -4037,9 +4214,7 @@ class UserSuspensionTests(unittest.HomeserverTestCase): content={"displayname": "something offensive"}, shorthand=False, ) - self.assertEqual( - channel2.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel2.json_body["errcode"], "M_USER_SUSPENDED") def test_suspended_user_cannot_redact_messages_other_than_their_own(self) -> None: # first user sends message @@ -4073,9 +4248,7 @@ class UserSuspensionTests(unittest.HomeserverTestCase): content={"reason": "bogus"}, shorthand=False, ) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") # but can redact their own channel = self.make_request( @@ -4086,3 +4259,1226 @@ class UserSuspensionTests(unittest.HomeserverTestCase): shorthand=False, ) self.assertEqual(channel.code, 200) + + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/rooms/{self.room1}/send/m.room.redaction/3456346", + access_token=self.tok1, + content={"reason": "bogus", "redacts": event_id}, + shorthand=False, + ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") + + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/rooms/{self.room1}/send/m.room.redaction/3456346", + access_token=self.tok1, + content={"reason": "bogus", "redacts": event_id2}, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + + def test_suspended_user_cannot_ban_others(self) -> None: + # user to ban joins room user1 created + self.make_request("POST", f"/rooms/{self.room1}/join", access_token=self.tok2) + + # suspend user1 + self.get_success(self.store.set_user_suspended_status(self.user1, True)) + + # user1 tries to ban other user while suspended + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{self.room1}/ban", + access_token=self.tok1, + content={"reason": "spite", "user_id": self.user2}, + shorthand=False, + ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") + + # un-suspend user1 + self.get_success(self.store.set_user_suspended_status(self.user1, False)) + + # ban now goes through + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{self.room1}/ban", + access_token=self.tok1, + content={"reason": "spite", "user_id": self.user2}, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + + +class RoomParticipantTestCase(unittest.HomeserverTestCase): + servlets = [ + login.register_servlets, + room.register_servlets, + profile.register_servlets, + admin.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user1 = self.register_user("thomas", "hackme") + self.tok1 = self.login("thomas", "hackme") + + self.user2 = self.register_user("teresa", "hackme") + self.tok2 = self.login("teresa", "hackme") + + self.room1 = self.helper.create_room_as( + room_creator=self.user1, + tok=self.tok1, + # Allow user2 to send state events into the room. + extra_content={ + "power_level_content_override": { + "state_default": 0, + }, + }, + ) + self.store = hs.get_datastores().main + + @parameterized.expand( + [ + # Should record participation. + param( + is_state=False, + event_type="m.room.message", + event_content={ + "msgtype": "m.text", + "body": "I am engaging in this room", + }, + record_participation=True, + ), + param( + is_state=False, + event_type="m.room.encrypted", + event_content={ + "algorithm": "m.megolm.v1.aes-sha2", + "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", + "device_id": "RJYKSTBOIE", + "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + }, + record_participation=True, + ), + # Should not record participation. + param( + is_state=False, + event_type="m.sticker", + event_content={ + "body": "My great sticker", + "info": {}, + "url": "mxc://unused/mxcurl", + }, + record_participation=False, + ), + # An invalid **state event** with type `m.room.message` + param( + is_state=True, + event_type="m.room.message", + event_content={ + "msgtype": "m.text", + "body": "I am engaging in this room", + }, + record_participation=False, + ), + # An invalid **state event** with type `m.room.encrypted` + # Note: this may become valid in the future with encrypted state, though we + # still may not want to consider it grounds for marking a user as participating. + param( + is_state=True, + event_type="m.room.encrypted", + event_content={ + "algorithm": "m.megolm.v1.aes-sha2", + "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", + "device_id": "RJYKSTBOIE", + "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + }, + record_participation=False, + ), + ] + ) + def test_sending_message_records_participation( + self, + is_state: bool, + event_type: str, + event_content: JsonDict, + record_participation: bool, + ) -> None: + """ + Test that sending an various events into a room causes the user to + appropriately marked or not marked as a participant in that room. + """ + self.helper.join(self.room1, self.user2, tok=self.tok2) + + # user has not sent any messages, so should not be a participant + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertFalse(participant) + + # send an event into the room + if is_state: + # send a state event + self.helper.send_state( + self.room1, + event_type, + body=event_content, + tok=self.tok2, + ) + else: + # send a non-state event + self.helper.send_event( + self.room1, + event_type, + content=event_content, + tok=self.tok2, + ) + + # check whether the user has been marked as a participant + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertEqual(participant, record_participation) + + @parameterized.expand( + [ + param( + event_type="m.room.message", + event_content={ + "msgtype": "m.text", + "body": "I am engaging in this room", + }, + ), + param( + event_type="m.room.encrypted", + event_content={ + "algorithm": "m.megolm.v1.aes-sha2", + "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", + "device_id": "RJYKSTBOIE", + "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + }, + ), + ] + ) + def test_sending_event_and_leaving_does_not_record_participation( + self, + event_type: str, + event_content: JsonDict, + ) -> None: + """ + Test that sending an event into a room that should mark a user as a + participant, but then leaving the room, results in the user no longer + be marked as a participant in that room. + """ + self.helper.join(self.room1, self.user2, tok=self.tok2) + + # user has not sent any messages, so should not be a participant + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertFalse(participant) + + # sending a message should now mark user as participant + self.helper.send_event( + self.room1, + event_type, + content=event_content, + tok=self.tok2, + ) + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertTrue(participant) + + # leave the room + self.helper.leave(self.room1, self.user2, tok=self.tok2) + + # user should no longer be considered a participant + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertFalse(participant) + + +class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets_for_client_rest_resource, + room.register_servlets, + login.register_servlets, + admin.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self.creator = self.register_user("creator", "test") + self.creator_tok = self.login("creator", "test") + + self.bad_user_id = self.register_user("bad", "test") + self.bad_tok = self.login("bad", "test") + + self.room_id = self.helper.create_room_as(self.creator, tok=self.creator_tok) + + self.store = hs.get_datastores().main + self._storage_controllers = hs.get_storage_controllers() + + self.federation_event_handler = self.hs.get_federation_event_handler() + + self.hs.config.experimental.msc4293_enabled = True + + def _check_redactions( + self, + original_events: List[EventBase], + pulled_events: List[JsonDict], + expect_redaction: bool, + reason: Optional[str] = None, + ) -> None: + """ + Checks a set of original events against a second set of the same events, pulled + from the /messages api. If expect_redaction is true, we expect that the second + set of events will be redacted, and the test will fail if that is not the case. + Otherwise, verifies that the events have not been redacted and fails if not. + + Args: + original_events: A list of the original events sent + pulled_events: A list of the same events as the orignal events, fetched + over the /messages api + expect_redaction: Whether or not the pulled_events should be redacted + reason: If the events are expected to be redacted, the expected reason + for the redaction + + """ + if expect_redaction: + redacted_count = 0 + for pulled_event in pulled_events: + for old_event in original_events: + if pulled_event["event_id"] != old_event.event_id: + continue + # we have a matching event, check that it is redacted + event_content = pulled_event["content"] + if event_content: + self.fail(f"Expected event {pulled_event} to be redacted") + redacting_event = pulled_event.get("redacted_because") + if not redacting_event: + self.fail( + f"Expected event {pulled_event} to have a redacting event." + ) + # check that the redacting event records the expected reason, and the + # redact_events flag + content = redacting_event["content"] + self.assertEqual(content["reason"], reason) + self.assertEqual(content["org.matrix.msc4293.redact_events"], True) + redacted_count += 1 + # all provided events should be redacted + self.assertEqual(len(original_events), redacted_count) + else: + unredacted_events = 0 + for pulled_event in pulled_events: + for old_event in original_events: + if pulled_event["event_id"] != old_event.event_id: + continue + # we have a matching event, make sure it is not redacted + redacted_because = pulled_event.get("redacted_because") + if redacted_because: + self.fail("Event should not have been redacted") + self.assertEqual(old_event.content, pulled_event["content"]) + unredacted_events += 1 + # all provided events should not have been redacted + self.assertEqual(unredacted_events, len(original_events)) + + def test_banning_local_member_with_flag_redacts_their_events(self) -> None: + self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) + + # bad user sends some messages + originals = [] + for i in range(5): + event = {"body": f"bothersome noise {i}", "msgtype": "m.text"} + res = self.helper.send_event( + self.room_id, "m.room.message", event, tok=self.bad_tok, expect_code=200 + ) + originals.append(res["event_id"]) + + # grab original events for comparison + original_events = [self.get_success(self.store.get_event(x)) for x in originals] + + # creator bans user with redaction flag set + content = { + "reason": "flooding", + "org.matrix.msc4293.redact_events": True, + } + self.helper.change_membership( + self.room_id, + self.creator, + self.bad_user_id, + "ban", + content, + self.creator_tok, + ) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions( + original_events, + channel.json_body["chunk"], + expect_redaction=True, + reason="flooding", + ) + + def test_banning_remote_member_with_flag_redacts_their_events(self) -> None: + bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + join_result = channel.json_body + + join_event_dict = join_result["event"] + self.add_hashes_and_signatures_from_other_server( + join_event_dict, + RoomVersions.V10, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/send_join/{self.room_id}/x", + content=join_event_dict, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + # the room should show that the bad user is a member + r = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + self.assertEqual(r[("m.room.member", bad_user)].membership, "join") + + auth_ids = [ + r[("m.room.create", "")].event_id, + r[("m.room.power_levels", "")].event_id, + r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, + ] + original_messages = [] + for i in range(5): + remote_message = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": bad_user, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": f"remote bummer{i}"}, + "auth_events": auth_ids, + "prev_events": auth_ids, + } + ), + room_version=RoomVersions.V10, + ) + + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, remote_message + ) + ) + original_messages.append(remote_message) + + # creator bans bad user with redaction flag set + content = { + "reason": "bummer messages", + "org.matrix.msc4293.redact_events": True, + } + res = self.helper.change_membership( + self.room_id, self.creator, bad_user, "ban", content, self.creator_tok + ) + ban_event_id = res["event_id"] + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions( + original_messages, + channel.json_body["chunk"], + expect_redaction=True, + reason="bummer messages", + ) + + # any future messages that are soft-failed are also redacted - send messages referencing + # dag before ban, they should be soft-failed but also redacted + new_original_messages = [] + for i in range(5): + remote_message = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": bad_user, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": f"soft-fail remote bummer{i}"}, + "auth_events": auth_ids, + "prev_events": auth_ids, + } + ), + room_version=RoomVersions.V10, + ) + + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, remote_message + ) + ) + new_original_messages.append(remote_message) + + # pull them from the db to check because they should be soft-failed and thus not available over + # cs-api + for message in new_original_messages: + original = self.get_success(self.store.get_event(message.event_id)) + if not original: + self.fail("Expected to find remote message in DB") + redacted_because = original.unsigned.get("redacted_because") + if not redacted_because: + self.fail("Did not find redacted_because field") + self.assertEqual(redacted_because.event_id, ban_event_id) + + def test_unbanning_remote_user_stops_redaction_action(self) -> None: + bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + join_result = channel.json_body + + join_event_dict = join_result["event"] + self.add_hashes_and_signatures_from_other_server( + join_event_dict, + RoomVersions.V10, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/send_join/{self.room_id}/x", + content=join_event_dict, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + # the room should show that the bad user is a member + r = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + self.assertEqual(r[("m.room.member", bad_user)].membership, "join") + + auth_ids = [ + r[("m.room.create", "")].event_id, + r[("m.room.power_levels", "")].event_id, + r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, + ] + original_messages = [] + for i in range(5): + remote_message = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": bad_user, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": f"annoying messages {i}"}, + "auth_events": auth_ids, + "prev_events": auth_ids, + } + ), + room_version=RoomVersions.V10, + ) + + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, remote_message + ) + ) + original_messages.append(remote_message) + + # creator bans bad user with redaction flag set + content = { + "reason": "this dude sucks", + "org.matrix.msc4293.redact_events": True, + } + self.helper.change_membership( + self.room_id, self.creator, bad_user, "ban", content, self.creator_tok + ) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions( + original_messages, + channel.json_body["chunk"], + True, + reason="this dude sucks", + ) + + # unban user + self.helper.change_membership( + self.room_id, self.creator, bad_user, "unban", {}, self.creator_tok + ) + + # user should be able to join again + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + join_result = channel.json_body + + join_event_dict = join_result["event"] + self.add_hashes_and_signatures_from_other_server( + join_event_dict, + RoomVersions.V10, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/send_join/{self.room_id}/x", + content=join_event_dict, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + # the room should show that the bad user is a member again + new_state = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + self.assertEqual(new_state[("m.room.member", bad_user)].membership, "join") + + new_state = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + auth_ids = [ + new_state[("m.room.create", "")].event_id, + new_state[("m.room.power_levels", "")].event_id, + new_state[("m.room.member", "@remote_bad_user:other.example.com")].event_id, + ] + + # messages after unban and join proceed unredacted + new_original_messages = [] + for i in range(5): + remote_message = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": bad_user, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": f"no longer a bummer {i}"}, + "auth_events": auth_ids, + "prev_events": auth_ids, + } + ), + room_version=RoomVersions.V10, + ) + + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, remote_message + ) + ) + new_original_messages.append(remote_message) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions(new_original_messages, channel.json_body["chunk"], False) + + def test_redaction_flag_ignored_for_user_if_banner_lacks_redaction_power( + self, + ) -> None: + # change power levels so creator can ban but not redact + self.helper.send_state( + self.room_id, + "m.room.power_levels", + {"events_default": 0, "redact": 100, "users": {self.creator: 75}}, + tok=self.creator_tok, + ) + self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) + + # bad user sends some messages + original_ids = [] + for i in range(15): + event = {"body": f"being a menace {i}", "msgtype": "m.text"} + res = self.helper.send_event( + self.room_id, "m.room.message", event, tok=self.bad_tok, expect_code=200 + ) + original_ids.append(res["event_id"]) + + # grab original events before ban + originals = [self.get_success(self.store.get_event(x)) for x in original_ids] + + # creator bans bad user with redaction flag + content = { + "reason": "flooding", + "org.matrix.msc4293.redact_events": True, + } + self.helper.change_membership( + self.room_id, + self.creator, + self.bad_user_id, + "ban", + content, + self.creator_tok, + ) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + # messages are not redacted + self._check_redactions(originals, channel.json_body["chunk"], False) + + def test_kicking_local_member_with_flag_redacts_their_events(self) -> None: + self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) + + # bad user sends some messages + originals = [] + for i in range(5): + event = {"body": f"bothersome noise {i}", "msgtype": "m.text"} + res = self.helper.send_event( + self.room_id, "m.room.message", event, tok=self.bad_tok, expect_code=200 + ) + originals.append(res["event_id"]) + + # grab original events for comparison + original_events = [self.get_success(self.store.get_event(x)) for x in originals] + + # creator kicks user with redaction flag set + content = { + "reason": "flooding", + "org.matrix.msc4293.redact_events": True, + } + self.helper.change_membership( + self.room_id, + self.creator, + self.bad_user_id, + "kick", + content, + self.creator_tok, + ) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions( + original_events, + channel.json_body["chunk"], + expect_redaction=True, + reason="flooding", + ) + + def test_kicking_remote_member_with_flag_redacts_their_events(self) -> None: + bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + join_result = channel.json_body + + join_event_dict = join_result["event"] + self.add_hashes_and_signatures_from_other_server( + join_event_dict, + RoomVersions.V10, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/send_join/{self.room_id}/x", + content=join_event_dict, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + # the room should show that the bad user is a member + r = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + self.assertEqual(r[("m.room.member", bad_user)].membership, "join") + + auth_ids = [ + r[("m.room.create", "")].event_id, + r[("m.room.power_levels", "")].event_id, + r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, + ] + original_messages = [] + for i in range(5): + remote_message = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": bad_user, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": f"remote bummer{i}"}, + "auth_events": auth_ids, + "prev_events": auth_ids, + } + ), + room_version=RoomVersions.V10, + ) + + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, remote_message + ) + ) + original_messages.append(remote_message) + + # creator kicks bad user with redaction flag set + content = { + "reason": "bummer messages", + "org.matrix.msc4293.redact_events": True, + } + res = self.helper.change_membership( + self.room_id, self.creator, bad_user, "kick", content, self.creator_tok + ) + ban_event_id = res["event_id"] + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions( + original_messages, + channel.json_body["chunk"], + expect_redaction=True, + reason="bummer messages", + ) + + # any future messages that are soft-failed are also redacted - send messages referencing + # dag before ban, they should be soft-failed but also redacted + new_original_messages = [] + for i in range(5): + remote_message = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": bad_user, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": f"soft-fail remote bummer{i}"}, + "auth_events": auth_ids, + "prev_events": auth_ids, + } + ), + room_version=RoomVersions.V10, + ) + + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, remote_message + ) + ) + new_original_messages.append(remote_message) + + # pull them from the db to check because they should be soft-failed and thus not available over + # cs-api + for message in new_original_messages: + original = self.get_success(self.store.get_event(message.event_id)) + if not original: + self.fail("Expected to find remote message in DB") + self.assertEqual(original.unsigned["redacted_by"], ban_event_id) + + def test_rejoining_kicked_remote_user_stops_redaction_action(self) -> None: + bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + join_result = channel.json_body + + join_event_dict = join_result["event"] + self.add_hashes_and_signatures_from_other_server( + join_event_dict, + RoomVersions.V10, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/send_join/{self.room_id}/x", + content=join_event_dict, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + # the room should show that the bad user is a member + r = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + self.assertEqual(r[("m.room.member", bad_user)].membership, "join") + + auth_ids = [ + r[("m.room.create", "")].event_id, + r[("m.room.power_levels", "")].event_id, + r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, + ] + original_messages = [] + for i in range(5): + remote_message = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": bad_user, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": f"annoying messages {i}"}, + "auth_events": auth_ids, + "prev_events": auth_ids, + } + ), + room_version=RoomVersions.V10, + ) + + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, remote_message + ) + ) + original_messages.append(remote_message) + + # creator kicks bad user with redaction flag set + content = { + "reason": "this dude sucks", + "org.matrix.msc4293.redact_events": True, + } + self.helper.change_membership( + self.room_id, self.creator, bad_user, "kick", content, self.creator_tok + ) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions( + original_messages, + channel.json_body["chunk"], + True, + reason="this dude sucks", + ) + + # user re-joins after kick + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + join_result = channel.json_body + + join_event_dict = join_result["event"] + self.add_hashes_and_signatures_from_other_server( + join_event_dict, + RoomVersions.V10, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/send_join/{self.room_id}/x", + content=join_event_dict, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + # the room should show that the bad user is a member again + new_state = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + self.assertEqual(new_state[("m.room.member", bad_user)].membership, "join") + + new_state = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + auth_ids = [ + new_state[("m.room.create", "")].event_id, + new_state[("m.room.power_levels", "")].event_id, + new_state[("m.room.member", "@remote_bad_user:other.example.com")].event_id, + ] + + # messages after kick and re-join proceed unredacted + new_original_messages = [] + for i in range(5): + remote_message = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": bad_user, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": f"no longer a bummer {i}"}, + "auth_events": auth_ids, + "prev_events": auth_ids, + } + ), + room_version=RoomVersions.V10, + ) + + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, remote_message + ) + ) + new_original_messages.append(remote_message) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions(new_original_messages, channel.json_body["chunk"], False) + + def test_redaction_flag_ignored_for_user_if_kicker_lacks_redaction_power( + self, + ) -> None: + # change power levels so creator can kick but not redact + self.helper.send_state( + self.room_id, + "m.room.power_levels", + {"events_default": 0, "redact": 100, "users": {self.creator: 75}}, + tok=self.creator_tok, + ) + self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) + + # bad user sends some messages + original_ids = [] + for i in range(15): + event = {"body": f"being a menace {i}", "msgtype": "m.text"} + res = self.helper.send_event( + self.room_id, "m.room.message", event, tok=self.bad_tok, expect_code=200 + ) + original_ids.append(res["event_id"]) + + # grab original events before ban + originals = [self.get_success(self.store.get_event(x)) for x in original_ids] + + # creator kicks bad user with redaction flag + content = { + "reason": "flooding", + "org.matrix.msc4293.redact_events": True, + } + self.helper.change_membership( + self.room_id, + self.creator, + self.bad_user_id, + "kick", + content, + self.creator_tok, + ) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + # messages are not redacted + self._check_redactions(originals, channel.json_body["chunk"], False) + + def test_MSC4293_flag_ignored_in_other_membership_events(self) -> None: + self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) + + # bad user sends some messages + original_ids = [] + for i in range(15): + event = {"body": f"being a menace {i}", "msgtype": "m.text"} + res = self.helper.send_event( + self.room_id, "m.room.message", event, tok=self.bad_tok, expect_code=200 + ) + original_ids.append(res["event_id"]) + + # grab original events before ban + originals = [self.get_success(self.store.get_event(x)) for x in original_ids] + + # bad user leaves on their own with flag + content = { + "org.matrix.msc4293.redact_events": True, + } + self.helper.change_membership( + self.room_id, + self.bad_user_id, + self.bad_user_id, + "leave", + content, + self.bad_tok, + ) + + # their messages are not redacted + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions(originals, channel.json_body["chunk"], False) + + # bad user is invited with flag in invite event + content = { + "org.matrix.msc4293.redact_events": True, + } + self.helper.change_membership( + self.room_id, + self.creator, + self.bad_user_id, + "invite", + content, + self.creator_tok, + ) + + # their messages are still not redacted + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions(originals, channel.json_body["chunk"], False) + + # bad user joins with flag in invite event + content = { + "org.matrix.msc4293.redact_events": True, + } + self.helper.change_membership( + self.room_id, + self.bad_user_id, + self.bad_user_id, + "join", + content, + self.bad_tok, + ) + + # and still their messages are not redacted + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions(originals, channel.json_body["chunk"], False) + + def test_MSC4293_redaction_applied_via_kick_api(self) -> None: + """ + Test that MSC4239 field passed through and applied when using /kick + """ + self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) + + # bad user sends some messages + original_ids = [] + for i in range(15): + event = {"body": f"being a menace {i}", "msgtype": "m.text"} + res = self.helper.send_event( + self.room_id, "m.room.message", event, tok=self.bad_tok, expect_code=200 + ) + original_ids.append(res["event_id"]) + + # grab original events before kick + originals = [self.get_success(self.store.get_event(x)) for x in original_ids] + + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{self.room_id}/kick", + access_token=self.creator_tok, + content={ + "reason": "being annoying", + "org.matrix.msc4293.redact_events": True, + "user_id": self.bad_user_id, + }, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions( + originals, + channel.json_body["chunk"], + expect_redaction=True, + reason="being annoying", + ) + + def test_MSC4293_redaction_applied_via_ban_api(self) -> None: + """ + Test that MSC4239 field passed through and applied when using /ban + """ + self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) + + # bad user sends some messages + original_ids = [] + for i in range(15): + event = {"body": f"being a menace {i}", "msgtype": "m.text"} + res = self.helper.send_event( + self.room_id, "m.room.message", event, tok=self.bad_tok, expect_code=200 + ) + original_ids.append(res["event_id"]) + + # grab original events before ban + originals = [self.get_success(self.store.get_event(x)) for x in original_ids] + + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{self.room_id}/ban", + access_token=self.creator_tok, + content={ + "reason": "being disruptive", + "org.matrix.msc4293.redact_events": True, + "user_id": self.bad_user_id, + }, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + + filter = json.dumps({"types": [EventTypes.Message]}) + channel = self.make_request( + "GET", + f"rooms/{self.room_id}/messages?filter={filter}&limit=50", + access_token=self.creator_tok, + ) + self.assertEqual(channel.code, 200) + self._check_redactions( + originals, + channel.json_body["chunk"], + expect_redaction=True, + reason="being disruptive", + ) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 5ef501c6d5..56533d85f5 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -18,27 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # -from parameterized import parameterized_class - from synapse.api.constants import EduTypes from synapse.rest import admin from synapse.rest.client import login, sendtodevice, sync -from synapse.types import JsonDict from tests.unittest import HomeserverTestCase, override_config -@parameterized_class( - ("sync_endpoint", "experimental_features"), - [ - ("/sync", {}), - ( - "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", - # Enable sliding sync - {"msc3575_enabled": True}, - ), - ], -) class SendToDeviceTestCase(HomeserverTestCase): """ Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. @@ -48,9 +34,6 @@ class SendToDeviceTestCase(HomeserverTestCase): experimental_features: The experimental features homeserver config to use. """ - sync_endpoint: str - experimental_features: JsonDict - servlets = [ admin.register_servlets, login.register_servlets, @@ -58,11 +41,6 @@ class SendToDeviceTestCase(HomeserverTestCase): sync.register_servlets, ] - def default_config(self) -> JsonDict: - config = super().default_config() - config["experimental_features"] = self.experimental_features - return config - def test_user_to_user(self) -> None: """A to-device message from one user to another should get delivered""" @@ -83,7 +61,7 @@ class SendToDeviceTestCase(HomeserverTestCase): self.assertEqual(chan.code, 200, chan.result) # check it appears - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + channel = self.make_request("GET", "/sync", access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) expected_result = { "events": [ @@ -99,7 +77,7 @@ class SendToDeviceTestCase(HomeserverTestCase): # it should re-appear if we do another sync because the to-device message is not # deleted until we acknowledge it by sending a `?since=...` parameter in the # next sync request corresponding to the `next_batch` value from the response. - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + channel = self.make_request("GET", "/sync", access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) self.assertEqual(channel.json_body["to_device"], expected_result) @@ -107,7 +85,7 @@ class SendToDeviceTestCase(HomeserverTestCase): sync_token = channel.json_body["next_batch"] channel = self.make_request( "GET", - f"{self.sync_endpoint}?since={sync_token}", + f"/sync?since={sync_token}", access_token=user2_tok, ) self.assertEqual(channel.code, 200, channel.result) @@ -133,7 +111,7 @@ class SendToDeviceTestCase(HomeserverTestCase): self.assertEqual(chan.code, 200, chan.result) # now sync: we should get two of the three (because burst_count=2) - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + channel = self.make_request("GET", "/sync", access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) msgs = channel.json_body["to_device"]["events"] self.assertEqual(len(msgs), 2) @@ -163,7 +141,7 @@ class SendToDeviceTestCase(HomeserverTestCase): # ... which should arrive channel = self.make_request( "GET", - f"{self.sync_endpoint}?since={sync_token}", + f"/sync?since={sync_token}", access_token=user2_tok, ) self.assertEqual(channel.code, 200, channel.result) @@ -198,7 +176,7 @@ class SendToDeviceTestCase(HomeserverTestCase): ) # now sync: we should get two of the three - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + channel = self.make_request("GET", "/sync", access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) msgs = channel.json_body["to_device"]["events"] self.assertEqual(len(msgs), 2) @@ -233,7 +211,7 @@ class SendToDeviceTestCase(HomeserverTestCase): # ... which should arrive channel = self.make_request( "GET", - f"{self.sync_endpoint}?since={sync_token}", + f"/sync?since={sync_token}", access_token=user2_tok, ) self.assertEqual(channel.code, 200, channel.result) @@ -258,7 +236,7 @@ class SendToDeviceTestCase(HomeserverTestCase): user2_tok = self.login("u2", "pass", "d2") # Do an initial sync - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + channel = self.make_request("GET", "/sync", access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) sync_token = channel.json_body["next_batch"] @@ -275,7 +253,7 @@ class SendToDeviceTestCase(HomeserverTestCase): channel = self.make_request( "GET", - f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + f"/sync?since={sync_token}&timeout=300000", access_token=user2_tok, ) self.assertEqual(channel.code, 200, channel.result) @@ -285,7 +263,7 @@ class SendToDeviceTestCase(HomeserverTestCase): channel = self.make_request( "GET", - f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + f"/sync?since={sync_token}&timeout=300000", access_token=user2_tok, ) self.assertEqual(channel.code, 200, channel.result) diff --git a/tests/rest/client/test_shadow_banned.py b/tests/rest/client/test_shadow_banned.py index 2287f233b4..b990a8600b 100644 --- a/tests/rest/client/test_shadow_banned.py +++ b/tests/rest/client/test_shadow_banned.py @@ -21,7 +21,7 @@ from unittest.mock import Mock, patch -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EduTypes, EventTypes diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index c52a5b2e79..7f3cf5affb 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -22,9 +22,9 @@ import json import logging from typing import List -from parameterized import parameterized, parameterized_class +from parameterized import parameterized -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import ( @@ -702,29 +702,11 @@ class SyncCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 200, channel.json_body) -@parameterized_class( - ("sync_endpoint", "experimental_features"), - [ - ("/sync", {}), - ( - "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", - # Enable sliding sync - {"msc3575_enabled": True}, - ), - ], -) class DeviceListSyncTestCase(unittest.HomeserverTestCase): """ Tests regarding device list (`device_lists`) changes. - - Attributes: - sync_endpoint: The endpoint under test to use for syncing. - experimental_features: The experimental features homeserver config to use. """ - sync_endpoint: str - experimental_features: JsonDict - servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, @@ -733,11 +715,6 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): devices.register_servlets, ] - def default_config(self) -> JsonDict: - config = super().default_config() - config["experimental_features"] = self.experimental_features - return config - def test_receiving_local_device_list_changes(self) -> None: """Tests that a local users that share a room receive each other's device list changes. @@ -767,7 +744,7 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): # Now have Bob initiate an initial sync (in order to get a since token) channel = self.make_request( "GET", - self.sync_endpoint, + "/sync", access_token=bob_access_token, ) self.assertEqual(channel.code, 200, channel.json_body) @@ -777,7 +754,7 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): # which we hope will happen as a result of Alice updating their device list. bob_sync_channel = self.make_request( "GET", - f"{self.sync_endpoint}?since={next_batch_token}&timeout=30000", + f"/sync?since={next_batch_token}&timeout=30000", access_token=bob_access_token, # Start the request, then continue on. await_result=False, @@ -824,7 +801,7 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): # Have Bob initiate an initial sync (in order to get a since token) channel = self.make_request( "GET", - self.sync_endpoint, + "/sync", access_token=bob_access_token, ) self.assertEqual(channel.code, 200, channel.json_body) @@ -834,7 +811,7 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): # which we hope will happen as a result of Alice updating their device list. bob_sync_channel = self.make_request( "GET", - f"{self.sync_endpoint}?since={next_batch_token}&timeout=1000", + f"/sync?since={next_batch_token}&timeout=1000", access_token=bob_access_token, # Start the request, then continue on. await_result=False, @@ -873,9 +850,7 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): ) # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) + channel = self.make_request("GET", "/sync", access_token=alice_access_token) self.assertEqual(channel.code, 200, channel.json_body) next_batch = channel.json_body["next_batch"] @@ -883,7 +858,7 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): # It won't return until something has happened incremental_sync_channel = self.make_request( "GET", - f"{self.sync_endpoint}?since={next_batch}&timeout=30000", + f"/sync?since={next_batch}&timeout=30000", access_token=alice_access_token, await_result=False, ) @@ -913,17 +888,6 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): ) -@parameterized_class( - ("sync_endpoint", "experimental_features"), - [ - ("/sync", {}), - ( - "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", - # Enable sliding sync - {"msc3575_enabled": True}, - ), - ], -) class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): """ Tests regarding device one time keys (`device_one_time_keys_count`) changes. @@ -933,9 +897,6 @@ class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): experimental_features: The experimental features homeserver config to use. """ - sync_endpoint: str - experimental_features: JsonDict - servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, @@ -943,11 +904,6 @@ class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): devices.register_servlets, ] - def default_config(self) -> JsonDict: - config = super().default_config() - config["experimental_features"] = self.experimental_features - return config - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.e2e_keys_handler = hs.get_e2e_keys_handler() @@ -964,9 +920,7 @@ class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): ) # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) + channel = self.make_request("GET", "/sync", access_token=alice_access_token) self.assertEqual(channel.code, 200, channel.json_body) # Check for those one time key counts @@ -1011,9 +965,7 @@ class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): ) # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) + channel = self.make_request("GET", "/sync", access_token=alice_access_token) self.assertEqual(channel.code, 200, channel.json_body) # Check for those one time key counts @@ -1024,17 +976,6 @@ class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): ) -@parameterized_class( - ("sync_endpoint", "experimental_features"), - [ - ("/sync", {}), - ( - "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", - # Enable sliding sync - {"msc3575_enabled": True}, - ), - ], -) class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): """ Tests regarding device one time keys (`device_unused_fallback_key_types`) changes. @@ -1044,9 +985,6 @@ class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): experimental_features: The experimental features homeserver config to use. """ - sync_endpoint: str - experimental_features: JsonDict - servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, @@ -1054,11 +992,6 @@ class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): devices.register_servlets, ] - def default_config(self) -> JsonDict: - config = super().default_config() - config["experimental_features"] = self.experimental_features - return config - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = self.hs.get_datastores().main self.e2e_keys_handler = hs.get_e2e_keys_handler() @@ -1078,9 +1011,7 @@ class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): ) # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) + channel = self.make_request("GET", "/sync", access_token=alice_access_token) self.assertEqual(channel.code, 200, channel.json_body) # Check for those one time key counts @@ -1122,9 +1053,7 @@ class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): self.assertEqual(fallback_res, ["alg1"], fallback_res) # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) + channel = self.make_request("GET", "/sync", access_token=alice_access_token) self.assertEqual(channel.code, 200, channel.json_body) # Check for the unused fallback key types diff --git a/tests/rest/client/test_tags.py b/tests/rest/client/test_tags.py index 5d596409e1..aee2f6affb 100644 --- a/tests/rest/client/test_tags.py +++ b/tests/rest/client/test_tags.py @@ -15,6 +15,7 @@ """Tests REST events for /tags paths.""" from http import HTTPStatus +from urllib import parse as urlparse import synapse.rest.admin from synapse.rest.client import login, room, tags @@ -93,3 +94,68 @@ class RoomTaggingTestCase(unittest.HomeserverTestCase): ) # Check that the request failed with the correct error self.assertEqual(channel.code, HTTPStatus.FORBIDDEN, channel.result) + + def test_put_tag_fails_if_tag_is_too_long(self) -> None: + """ + Test that a user cannot add a tag to a room that is longer than the 255 bytes + allowed by the matrix specification. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + # create a string which is larger than 255 bytes + tag = "X" * 300 + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/{tag}", + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request failed + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + + def test_put_tag_fails_if_tag_is_too_long_with_graphemes(self) -> None: + """ + Test that a user cannot add a tag to a room that contains graphemes which are in total + longer than the 255 bytes allowed by the matrix specification. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + # create a string which is larger than 255 bytes (275) + tag = "👩‍🚒" * 25 + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/" + + urlparse.quote(tag.encode("utf-8")), + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request failed + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + + def test_put_tag_succeeds_with_graphemes(self) -> None: + """ + Test that a user can add a tag to a room that contains graphemes which are in total + less than the 255 bytes allowed by the matrix specification. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + # create a string of acceptable length (220 bytes) + tag = "👩‍🚒" * 20 + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/" + + urlparse.quote(tag.encode("utf-8")), + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request succeeded + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py index d10df1a90f..f14ca8237a 100644 --- a/tests/rest/client/test_third_party_rules.py +++ b/tests/rest/client/test_third_party_rules.py @@ -22,7 +22,7 @@ import threading from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union from unittest.mock import AsyncMock, Mock -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, LoginType, Membership from synapse.api.errors import SynapseError diff --git a/tests/rest/client/test_thread_subscriptions.py b/tests/rest/client/test_thread_subscriptions.py new file mode 100644 index 0000000000..3fbf3c5bfa --- /dev/null +++ b/tests/rest/client/test_thread_subscriptions.py @@ -0,0 +1,351 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from http import HTTPStatus + +from twisted.internet.testing import MemoryReactor + +from synapse.api.errors import Codes +from synapse.rest import admin +from synapse.rest.client import login, profile, room, thread_subscriptions +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock + +from tests import unittest + +PREFIX = "/_matrix/client/unstable/io.element.msc4306/rooms" + + +class ThreadSubscriptionsTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + login.register_servlets, + profile.register_servlets, + room.register_servlets, + thread_subscriptions.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = {"msc4306_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user_id = self.register_user("user", "password") + self.token = self.login("user", "password") + self.other_user_id = self.register_user("other_user", "password") + self.other_token = self.login("other_user", "password") + + # Create a room and send a message to use as a thread root + self.room_id = self.helper.create_room_as(self.user_id, tok=self.token) + self.helper.join(self.room_id, self.other_user_id, tok=self.other_token) + (self.root_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.token + ) + + # Send a message in the thread + self.threaded_events = self.helper.send_messages( + self.room_id, + 2, + content_fn=lambda idx: { + "body": f"Thread message {idx}", + "msgtype": "m.text", + "m.relates_to": { + "rel_type": "m.thread", + "event_id": self.root_event_id, + }, + }, + tok=self.token, + ) + + def test_get_thread_subscription_unsubscribed(self) -> None: + """Test retrieving thread subscription when not subscribed.""" + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_get_thread_subscription_nonexistent_thread(self) -> None: + """Test retrieving subscription settings for a nonexistent thread.""" + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/$nonexistent:example.org/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_get_thread_subscription_no_access(self) -> None: + """Test that a user can't get thread subscription for a thread they can't access.""" + self.register_user("no_access", "password") + no_access_token = self.login("no_access", "password") + + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=no_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_subscribe_manual_then_automatic(self) -> None: + """Test subscribing to a thread, first a manual subscription then an automatic subscription. + The manual subscription wins over the automatic one.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the subscription was saved + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": False}) + + # Now also register an automatic subscription; it should not + # override the manual subscription + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {"automatic": self.threaded_events[0]}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the manual subscription was not overridden + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": False}) + + def test_subscribe_automatic_then_manual(self) -> None: + """Test subscribing to a thread, first an automatic subscription then a manual subscription. + The manual subscription wins over the automatic one.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": self.threaded_events[0], + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.text_body) + + # Assert the subscription was saved + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": True}) + + # Now also register a manual subscription + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the manual subscription was not overridden + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": False}) + + def test_unsubscribe(self) -> None: + """Test subscribing to a thread, then unsubscribing.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": self.threaded_events[0], + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the subscription was saved + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": True}) + + channel = self.make_request( + "DELETE", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_set_thread_subscription_nonexistent_thread(self) -> None: + """Test setting subscription settings for a nonexistent thread.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/$nonexistent:example.org/subscription", + {}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_set_thread_subscription_no_access(self) -> None: + """Test that a user can't set thread subscription for a thread they can't access.""" + self.register_user("no_access2", "password") + no_access_token = self.login("no_access2", "password") + + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {}, + access_token=no_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_invalid_body(self) -> None: + """Test that sending invalid subscription settings is rejected.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + # non-Event ID `automatic` + {"automatic": True}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) + + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + # non-Event ID `automatic` + {"automatic": "$malformedEventId"}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) + + def test_auto_subscribe_cause_event_not_in_thread(self) -> None: + """ + Test making an automatic subscription, where the cause event is not + actually in the thread. + This is an error. + """ + (unrelated_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.token + ) + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {"automatic": unrelated_event_id}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.text_body) + self.assertEqual(channel.json_body["errcode"], Codes.MSC4306_NOT_IN_THREAD) + + def test_auto_resubscription_conflict(self) -> None: + """ + Test that an automatic subscription that conflicts with an unsubscription + is skipped. + """ + # Reuse the test that subscribes and unsubscribes + self.test_unsubscribe() + + # Now no matter which event we present as the cause of an automatic subscription, + # the automatic subscription is skipped. + # This is because the unsubscription happened after all of the events. + for event in self.threaded_events: + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": event, + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.CONFLICT, channel.text_body) + self.assertEqual( + channel.json_body["errcode"], + Codes.MSC4306_CONFLICTING_UNSUBSCRIPTION, + channel.text_body, + ) + + # Check the subscription was not made + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + + # But if a new event is sent after the unsubscription took place, + # that one can be used for an automatic subscription + (later_event_id,) = self.helper.send_messages( + self.room_id, + 1, + content_fn=lambda _: { + "body": "Thread message after unsubscription", + "msgtype": "m.text", + "m.relates_to": { + "rel_type": "m.thread", + "event_id": self.root_event_id, + }, + }, + tok=self.token, + ) + + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": later_event_id, + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.text_body) + + # Check the subscription was made + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": True}) diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index af1eecbb34..5f42acb391 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -90,7 +90,7 @@ class HttpTransactionCacheTestCase(unittest.TestCase): ) -> Generator["defer.Deferred[Any]", object, None]: @defer.inlineCallbacks def cb() -> Generator["defer.Deferred[object]", object, Tuple[int, JsonDict]]: - yield Clock(reactor).sleep(0) + yield defer.ensureDeferred(Clock(reactor).sleep(0)) return 1, {} @defer.inlineCallbacks diff --git a/tests/rest/client/test_typing.py b/tests/rest/client/test_typing.py index 805c49b540..ce2504156c 100644 --- a/tests/rest/client/test_typing.py +++ b/tests/rest/client/test_typing.py @@ -21,7 +21,7 @@ """Tests REST events for /rooms paths.""" -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from synapse.api.constants import EduTypes from synapse.rest.client import room diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index c4b15c5ae7..66fddc5475 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -21,9 +21,9 @@ from typing import Optional from unittest.mock import patch -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor -from synapse.api.constants import EventContentFields, EventTypes, RoomTypes +from synapse.api.constants import EventContentFields, EventTypes, Membership, RoomTypes from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.rest import admin from synapse.rest.client import login, room, room_upgrade_rest_servlet @@ -411,3 +411,24 @@ class UpgradeRoomTest(unittest.HomeserverTestCase): channel = self._upgrade_room(expire_cache=False) self.assertEqual(200, channel.code, channel.result) + + def test_bans(self) -> None: + """ + Test that bans get copied over when upgrading a room. + """ + + users_to_ban = ["@user2:test", "@user3:test", "@user4:test"] + for user in users_to_ban: + self.helper.ban(self.room_id, self.creator, user, tok=self.creator_token) + + channel = self._upgrade_room(self.creator_token) + self.assertEqual(200, channel.code, channel.result) + + for user in users_to_ban: + content = self.helper.get_state( + self.room_id, + event_type=EventTypes.Member, + state_key=user, + tok=self.creator_token, + ) + self.assertEqual(content[EventContentFields.MEMBERSHIP], Membership.BAN) diff --git a/tests/rest/client/utils.py b/tests/rest/client/utils.py index a1c284726a..bb214759d9 100644 --- a/tests/rest/client/utils.py +++ b/tests/rest/client/utils.py @@ -29,23 +29,25 @@ from http import HTTPStatus from typing import ( Any, AnyStr, + Callable, Dict, Iterable, + Literal, Mapping, MutableMapping, Optional, + Sequence, Tuple, overload, ) from urllib.parse import urlencode import attr -from typing_extensions import Literal -from twisted.test.proto_helpers import MemoryReactorClock +from twisted.internet.testing import MemoryReactorClock from twisted.web.server import Site -from synapse.api.constants import Membership, ReceiptTypes +from synapse.api.constants import EventTypes, Membership, ReceiptTypes from synapse.api.errors import Codes from synapse.server import HomeServer from synapse.types import JsonDict @@ -185,7 +187,7 @@ class RestHelper: def join( self, room: str, - user: Optional[str] = None, + user: str, expect_code: int = HTTPStatus.OK, tok: Optional[str] = None, appservice_user_id: Optional[str] = None, @@ -394,6 +396,32 @@ class RestHelper: custom_headers=custom_headers, ) + def send_messages( + self, + room_id: str, + num_events: int, + content_fn: Callable[[int], JsonDict] = lambda idx: { + "msgtype": "m.text", + "body": f"Test event {idx}", + }, + tok: Optional[str] = None, + ) -> Sequence[str]: + """ + Helper to send a handful of sequential events and return their event IDs as a sequence. + """ + event_ids = [] + + for event_index in range(num_events): + response = self.send_event( + room_id, + EventTypes.Message, + content_fn(event_index), + tok=tok, + ) + event_ids.append(response["event_id"]) + + return event_ids + def send_event( self, room_id: str, @@ -548,7 +576,7 @@ class RestHelper: room_id: str, event_type: str, body: Dict[str, Any], - tok: Optional[str], + tok: Optional[str] = None, expect_code: int = HTTPStatus.OK, state_key: str = "", ) -> JsonDict: @@ -716,9 +744,9 @@ class RestHelper: "/login", content={"type": "m.login.token", "token": login_token}, ) - assert ( - channel.code == expected_status - ), f"unexpected status in response: {channel.code}" + assert channel.code == expected_status, ( + f"unexpected status in response: {channel.code}" + ) return channel.json_body def auth_via_oidc( @@ -889,7 +917,7 @@ class RestHelper: "GET", uri, ) - assert channel.code == 302 + assert channel.code == 302, f"Expected 302 for {uri}, got {channel.code}" # hit the redirect url again with the right Host header, which should now issue # a cookie and redirect to the SSO provider. @@ -901,17 +929,18 @@ class RestHelper: location = get_location(channel) parts = urllib.parse.urlsplit(location) + next_uri = urllib.parse.urlunsplit(("", "") + parts[2:]) channel = make_request( self.reactor, self.site, "GET", - urllib.parse.urlunsplit(("", "") + parts[2:]), + next_uri, custom_headers=[ ("Host", parts[1]), ], ) - assert channel.code == 302 + assert channel.code == 302, f"Expected 302 for {next_uri}, got {channel.code}" channel.extract_cookies(cookies) return get_location(channel) diff --git a/tests/rest/key/v2/test_remote_key_resource.py b/tests/rest/key/v2/test_remote_key_resource.py index 21e12b2a2f..3717d70b6b 100644 --- a/tests/rest/key/v2/test_remote_key_resource.py +++ b/tests/rest/key/v2/test_remote_key_resource.py @@ -27,7 +27,7 @@ from canonicaljson import encode_canonical_json from signedjson.sign import sign_json from signedjson.types import SigningKey -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import NoResource, Resource from synapse.crypto.keyring import PerspectivesKeyFetcher @@ -99,7 +99,7 @@ class RemoteKeyResourceTestCase(BaseRemoteKeyResourceTestCase): """ channel = FakeChannel(self.site, self.reactor) # channel is a `FakeChannel` but `HTTPChannel` is expected - req = SynapseRequest(channel, self.site) # type: ignore[arg-type] + req = SynapseRequest(channel, self.site, self.hs.hostname) # type: ignore[arg-type] req.content = BytesIO(b"") req.requestReceived( b"GET", @@ -201,7 +201,7 @@ class EndToEndPerspectivesTests(BaseRemoteKeyResourceTestCase): channel = FakeChannel(self.site, self.reactor) # channel is a `FakeChannel` but `HTTPChannel` is expected - req = SynapseRequest(channel, self.site) # type: ignore[arg-type] + req = SynapseRequest(channel, self.site, self.hs.hostname) # type: ignore[arg-type] req.content = BytesIO(encode_canonical_json(data)) req.requestReceived( diff --git a/tests/rest/media/test_domain_blocking.py b/tests/rest/media/test_domain_blocking.py index 49d81f4b28..3feade4a4b 100644 --- a/tests/rest/media/test_domain_blocking.py +++ b/tests/rest/media/test_domain_blocking.py @@ -20,7 +20,7 @@ # from typing import Dict -from twisted.test.proto_helpers import MemoryReactor +from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource from synapse.media._base import FileInfo @@ -61,6 +61,7 @@ class MediaDomainBlockingTests(unittest.HomeserverTestCase): time_now_ms=clock.time_msec(), upload_name="test.png", filesystem_id=file_id, + sha256=file_id, ) ) diff --git a/tests/rest/media/test_url_preview.py b/tests/rest/media/test_url_preview.py index 103d7662d9..e096780ce2 100644 --- a/tests/rest/media/test_url_preview.py +++ b/tests/rest/media/test_url_preview.py @@ -29,7 +29,7 @@ from twisted.internet._resolver import HostResolution from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.error import DNSLookupError from twisted.internet.interfaces import IAddress, IResolutionReceiver -from twisted.test.proto_helpers import AccumulatingProtocol, MemoryReactor +from twisted.internet.testing import AccumulatingProtocol, MemoryReactor from twisted.web.resource import Resource from synapse.config.oembed import OEmbedEndpointConfig @@ -878,7 +878,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): data = base64.b64encode(SMALL_PNG) end_content = ( - b"" b'' b"" + b'' ) % (data,) channel = self.make_request( diff --git a/tests/rest/synapse/mas/__init__.py b/tests/rest/synapse/mas/__init__.py new file mode 100644 index 0000000000..db2cfe109f --- /dev/null +++ b/tests/rest/synapse/mas/__init__.py @@ -0,0 +1,12 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . diff --git a/tests/rest/synapse/mas/_base.py b/tests/rest/synapse/mas/_base.py new file mode 100644 index 0000000000..19d33807a6 --- /dev/null +++ b/tests/rest/synapse/mas/_base.py @@ -0,0 +1,43 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from twisted.web.resource import Resource + +from synapse.rest.synapse.client import build_synapse_client_resource_tree +from synapse.types import JsonDict + +from tests import unittest + + +class BaseTestCase(unittest.HomeserverTestCase): + SHARED_SECRET = "shared_secret" + + def default_config(self) -> JsonDict: + config = super().default_config() + config["enable_registration"] = False + config["experimental_features"] = { + "msc3861": { + "enabled": True, + "issuer": "https://example.com", + "client_id": "dummy", + "client_auth_method": "client_secret_basic", + "client_secret": "dummy", + "admin_token": self.SHARED_SECRET, + } + } + return config + + def create_resource_dict(self) -> dict[str, Resource]: + base = super().create_resource_dict() + base.update(build_synapse_client_resource_tree(self.hs)) + return base diff --git a/tests/rest/synapse/mas/test_devices.py b/tests/rest/synapse/mas/test_devices.py new file mode 100644 index 0000000000..458878c13c --- /dev/null +++ b/tests/rest/synapse/mas/test_devices.py @@ -0,0 +1,693 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from twisted.internet.testing import MemoryReactor + +from synapse.server import HomeServer +from synapse.types import UserID +from synapse.util import Clock + +from tests.unittest import skip_unless +from tests.utils import HAS_AUTHLIB + +from ._base import BaseTestCase + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasUpsertDeviceResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_upsert_device(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + # This created a new device, hence the 201 status code + self.assertEqual(channel.code, 201, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the device exists + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["device_id"], "DEVICE1") + self.assertIsNone(device["display_name"]) + + def test_update_existing_device(self) -> None: + store = self.hs.get_datastores().main + device_handler = self.hs.get_device_handler() + + # Create an initial device + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id="DEVICE1", + display_name="Old Name", + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + # This updated an existing device, hence the 200 status code + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the device was updated + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "New Name") + + def test_upsert_device_with_display_name(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "Alice's Phone", + }, + ) + + self.assertEqual(channel.code, 201, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the device exists with correct display name + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "Alice's Phone") + + def test_upsert_device_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_upsert_device_missing_device_id(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_upsert_device_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "device_id": "DEVICE1", + }, + ) + + # We get a 404 here as the user doesn't exist + self.assertEqual(channel.code, 404, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasDeleteDeviceResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user and device for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + # Create a device + device_handler = homeserver.get_device_handler() + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id="DEVICE1", + display_name="Test Device", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_delete_device(self) -> None: + store = self.hs.get_datastores().main + + # Verify device exists before deletion + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 204) + + # Verify the device no longer exists + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + self.assertIsNone(device) + + def test_delete_nonexistent_device(self) -> None: + # Deleting a non-existent device should be idempotent + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "NONEXISTENT", + }, + ) + + self.assertEqual(channel.code, 204) + + def test_delete_device_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_device_missing_device_id(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_device_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "device_id": "DEVICE1", + }, + ) + + # Should fail on a non-existent user + self.assertEqual(channel.code, 404, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasUpdateDeviceDisplayNameResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user and device for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + # Create a device + device_handler = homeserver.get_device_handler() + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id="DEVICE1", + display_name="Old Name", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_update_device_display_name(self) -> None: + store = self.hs.get_datastores().main + + # Verify initial display name + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "Old Name") + + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "Updated Name", + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the display name was updated + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "Updated Name") + + def test_update_nonexistent_device(self) -> None: + # Updating a non-existent device should fail + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "NONEXISTENT", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_update_device_display_name_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_update_device_display_name_missing_device_id(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_update_device_display_name_missing_display_name(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_update_device_display_name_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasSyncDevicesResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + # Create some initial devices + device_handler = homeserver.get_device_handler() + for device_id in ["DEVICE1", "DEVICE2", "DEVICE3"]: + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id=device_id, + display_name=f"Device {device_id}", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_sync_devices_no_changes(self) -> None: + # Sync with the same devices that already exist + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE2", "DEVICE3"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify all devices still exist + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", "DEVICE2", "DEVICE3"}) + + def test_sync_devices_add_only(self) -> None: + # Sync with additional devices + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE2", "DEVICE3", "DEVICE4", "DEVICE5"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify new devices were added + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual( + set(devices.keys()), {"DEVICE1", "DEVICE2", "DEVICE3", "DEVICE4", "DEVICE5"} + ) + + def test_sync_devices_delete_only(self) -> None: + # Sync with fewer devices + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify devices were deleted + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1"}) + + def test_sync_devices_add_and_delete(self) -> None: + # Sync with a mix of additions and deletions + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE4", "DEVICE5"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the correct devices exist + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", "DEVICE4", "DEVICE5"}) + + def test_sync_devices_empty_list(self) -> None: + # Sync with empty device list (delete all devices) + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": [], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify all devices were deleted + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(devices, {}) + + def test_sync_devices_for_new_user(self) -> None: + # Test syncing devices for a user that doesn't have any devices yet + bob_user_id = UserID("bob", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=bob_user_id.localpart, + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "bob", + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify devices were created + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(bob_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", "DEVICE2"}) + + def test_sync_devices_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_sync_devices_missing_devices(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_sync_devices_invalid_devices_type(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": "not_a_list", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_sync_devices_nonexistent_user(self) -> None: + # Test syncing devices for a user that doesn't exist + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_sync_devices_duplicate_device_ids(self) -> None: + # Test syncing with duplicate device IDs (sets should handle this) + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the correct devices exist (duplicates should be handled) + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(sorted(devices.keys()), ["DEVICE1", "DEVICE2"]) diff --git a/tests/rest/synapse/mas/test_users.py b/tests/rest/synapse/mas/test_users.py new file mode 100644 index 0000000000..b236aceaf2 --- /dev/null +++ b/tests/rest/synapse/mas/test_users.py @@ -0,0 +1,1399 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from urllib.parse import urlencode + +from twisted.internet.testing import MemoryReactor + +from synapse.appservice import ApplicationService +from synapse.server import HomeServer +from synapse.types import JsonDict, UserID, create_requester +from synapse.util import Clock + +from tests.unittest import skip_unless +from tests.utils import HAS_AUTHLIB + +from ._base import BaseTestCase + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasQueryUserResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token="other_token", + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_query_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + self.get_success( + store.set_profile_avatar_url( + user_id=alice, + new_avatar_url="mxc://example.com/avatar", + ) + ) + + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual( + channel.json_body, + { + "user_id": "@alice:test", + "display_name": "Alice", + "avatar_url": "mxc://example.com/avatar", + "is_suspended": False, + "is_deactivated": False, + }, + ) + + self.get_success( + store.set_user_suspended_status(user_id=str(alice), suspended=True) + ) + + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual( + channel.json_body, + { + "user_id": "@alice:test", + "display_name": "Alice", + "avatar_url": "mxc://example.com/avatar", + "is_suspended": True, + "is_deactivated": False, + }, + ) + + # Deactivate the account, it should clear the display name and avatar + # and mark the user as deactivated + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=True, + requester=create_requester(alice), + ) + ) + + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual( + channel.json_body, + { + "user_id": "@alice:test", + "display_name": None, + "avatar_url": None, + "is_suspended": True, + "is_deactivated": True, + }, + ) + + def test_query_unknown_user(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_query_user_missing_localpart(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/query_user", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasProvisionUserResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token="other_token", + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_provision_user(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_displayname": "Alice", + "set_emails": ["alice@example.com"], + "set_avatar_url": "mxc://example.com/avatar", + }, + ) + + # This created the user, hence the 201 status code + self.assertEqual(channel.code, 201, channel.json_body) + self.assertEqual(channel.json_body, {}) + + alice = UserID("alice", "test") + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, "Alice") + self.assertEqual(profile.avatar_url, "mxc://example.com/avatar") + threepids = self.get_success(store.user_get_threepids(str(alice))) + self.assertEqual(len(threepids), 1) + self.assertEqual(threepids[0].medium, "email") + self.assertEqual(threepids[0].address, "alice@example.com") + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "unset_displayname": True, + "unset_avatar_url": True, + "unset_emails": True, + }, + ) + + # This updated the user, hence the 200 status code + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the profile and threepids were deleted + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, None) + self.assertEqual(profile.avatar_url, None) + threepids = self.get_success(store.user_get_threepids(str(alice))) + self.assertEqual(threepids, []) + + def test_provision_user_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "set_displayname": "Alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_provision_user_empty_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "", + "set_displayname": "Alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_provision_user_invalid_localpart(self) -> None: + # Test with characters that are invalid in localparts + invalid_localparts = [ + "@alice:test", # That's a MXID + "alice@domain.com", + "alice:test", + "alice space", + "alice#hash", + "a" * 1000, # Very long localpart + ] + + for localpart in invalid_localparts: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": localpart, + "set_displayname": "Alice", + }, + ) + # Should be a validation error + self.assertEqual( + channel.code, 400, f"Should fail for localpart: {localpart}" + ) + + def test_provision_user_multiple_emails(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_emails": ["alice@example.com", "alice.alt@example.com"], + }, + ) + + self.assertEqual(channel.code, 201, channel.json_body) + + alice = UserID("alice", "test") + threepids = self.get_success(store.user_get_threepids(str(alice))) + self.assertEqual(len(threepids), 2) + email_addresses = {tp.address for tp in threepids} + self.assertEqual( + email_addresses, {"alice@example.com", "alice.alt@example.com"} + ) + + def test_provision_user_duplicate_emails(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_emails": ["alice@example.com", "alice@example.com"], + }, + ) + + self.assertEqual(channel.code, 201, channel.json_body) + + alice = UserID("alice", "test") + threepids = self.get_success(store.user_get_threepids(str(alice))) + # Should deduplicate + self.assertEqual(len(threepids), 1) + self.assertEqual(threepids[0].address, "alice@example.com") + + def test_provision_user_conflicting_operations(self) -> None: + # Test setting and unsetting the same field + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_displayname": "Alice", + "unset_displayname": True, + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_provision_user_invalid_json_types(self) -> None: + # Test with wrong data types + invalid_contents: list[JsonDict] = [ + {"localpart": "alice", "set_displayname": 123}, # Number instead of string + { + "localpart": "alice", + "set_emails": "not-an-array", + }, # String instead of array + { + "localpart": "alice", + "unset_displayname": "not-a-bool", + }, # String instead of bool + {"localpart": 123}, # Number instead of string for localpart + ] + + for content in invalid_contents: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content=content, + ) + self.assertEqual(channel.code, 400, f"Should fail for content: {content}") + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasIsLocalpartAvailableResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Provision a user + store = homeserver.get_datastores().main + self.get_success(store.register_user("@alice:test")) + + def test_other_token(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=alice", + shorthand=False, + access_token="other_token", + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_is_localpart_available(self) -> None: + # "alice" is not available + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_USER_IN_USE") + + # "bob" is available + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=bob", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + def test_is_localpart_available_invalid_localparts(self) -> None: + # Numeric-only localparts are not allowed + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=0", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_USERNAME") + + # A super-long MXID is not allowed by the spec + super_long = "a" * 1000 + channel = self.make_request( + "GET", + f"/_synapse/mas/is_localpart_available?localpart={super_long}", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_USERNAME") + + def test_is_localpart_available_appservice_exclusive(self) -> None: + # Insert an appservice which has exclusive namespaces + appservice = ApplicationService( + token="i_am_an_app_service", + id="1234", + namespaces={"users": [{"regex": r"@as_user_.*:.+", "exclusive": True}]}, + sender=UserID.from_string("@as_main:test"), + ) + self.hs.get_datastores().main.services_cache = [appservice] + + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=as_main", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_EXCLUSIVE") + + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=as_user_alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_EXCLUSIVE") + + # Sanity-check that "bob" is available + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=bob", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + def test_is_localpart_available_missing_localpart(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_is_localpart_available_empty_localpart(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_is_localpart_available_invalid_characters(self) -> None: + # Test with characters that are invalid in localparts + invalid_localparts = [ + "alice@domain.com", # Contains @ + "alice:test", # Contains : + "alice space", # Contains space + "alice\\backslash", # Contains backslash + "alice#hash", # Contains hash + "alice$dollar", # Contains $ + "alice%percent", # Contains % + "alice&", # Contains & + "alice?question", # Contains ? + "alice[bracket", # Contains [ + "alice]bracket", # Contains ] + "alice{brace", # Contains { + "alice}brace", # Contains } + "alice|pipe", # Contains | + 'alice"quote', # Contains " + "alice'apostrophe", # Contains ' + "alicegreater", # Contains > + "alice\ttab", # Contains tab + "alice\nnewline", # Contains newline + ] + + for localpart in invalid_localparts: + channel = self.make_request( + "GET", + f"/_synapse/mas/is_localpart_available?{urlencode({'localpart': localpart})}", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + # Should return 400 for invalid characters + self.assertEqual( + channel.code, + 400, + f"Should reject localpart with invalid chars: {localpart}", + ) + self.assertEqual( + channel.json_body["errcode"], "M_INVALID_USERNAME", localpart + ) + + def test_is_localpart_available_case_sensitivity(self) -> None: + # Register a user with an uppercase localpart + self.get_success(self.hs.get_datastores().main.register_user("@BOB:test")) + + # It should report as not available, the search should be case-insensitive + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=bob", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_USER_IN_USE") + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasDeleteUserResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Provision a user with a display name + self.get_success( + homeserver.get_registration_handler().register_user( + localpart="alice", + default_display_name="Alice", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token="other_token", + content={"localpart": "alice", "erase": False}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_delete_user_no_erase(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + + # Delete the user + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the user was deleted + self.assertTrue( + self.get_success(store.get_user_deactivated_status(user_id=str(alice))) + ) + # But not erased + self.assertFalse(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_delete_user_erase(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + + # Delete the user + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": True}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the user was deleted + self.assertTrue( + self.get_success(store.get_user_deactivated_status(user_id=str(alice))) + ) + # And erased + self.assertTrue(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_delete_user_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"erase": False}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_user_missing_erase(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_user_invalid_erase_type(self) -> None: + invalid_erase_values = [ + "true", # String instead of bool + 1, # Number instead of bool + "false", # String instead of bool + 0, # Number instead of bool + {}, # Object instead of bool + [], # Array instead of bool + ] + + for erase_value in invalid_erase_values: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": erase_value}, + ) + self.assertEqual( + channel.code, 400, f"Should fail for erase value: {erase_value}" + ) + + def test_delete_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "nonexistent", "erase": False}, + ) + + self.assertEqual(channel.code, 404) + + def test_delete_already_deleted_user(self) -> None: + # First deletion + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + self.assertEqual(channel.code, 200) + + # Second deletion should be idempotent + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + self.assertEqual(channel.code, 200) + + def test_delete_user_erase_already_deleted_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + + # First delete without erase + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + self.assertEqual(channel.code, 200) + + # Verify not erased initially + self.assertFalse(self.get_success(store.is_user_erased(user_id=str(alice)))) + + # Now delete with erase + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": True}, + ) + self.assertEqual(channel.code, 200) + + # Should now be erased + self.assertTrue(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_delete_user_empty_json(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_user_extra_fields(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "erase": False, + "extra_field": "should_be_ignored", + "another_field": 123, + }, + ) + + # Should succeed and ignore extra fields + self.assertEqual(channel.code, 200, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasReactivateUserResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token="other_token", + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_reactivate_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=True, + requester=create_requester(alice), + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the user was reactivated + self.assertFalse( + self.get_success(store.get_user_deactivated_status(user_id=str(alice))) + ) + + def test_reactivate_user_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_reactivate_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "nonexistent"}, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_reactivate_active_user(self) -> None: + # Create an active user + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + # Should be idempotent + self.assertEqual(channel.code, 200, channel.json_body) + + def test_reactivate_erased_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + + # Deactivate with erase + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=True, + requester=create_requester(alice), + ) + ) + + # Verify user is erased + self.assertTrue(self.get_success(store.is_user_erased(user_id=str(alice)))) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + # Should succeed even for erased users + self.assertEqual(channel.code, 200, channel.json_body) + # Shouldn't be erased anymore + self.assertFalse(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_reactivate_user_extra_fields(self) -> None: + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + ) + ) + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=False, + requester=create_requester(alice), + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "extra_field": "should_be_ignored", + "another_field": 123, + }, + ) + + # Should succeed and ignore extra fields + self.assertEqual(channel.code, 200, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasSetDisplayNameResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token="other_token", + content={"localpart": "alice", "displayname": "Bob"}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_set_display_name(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, "Alice") + + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "displayname": "Bob"}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the profile was updated + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, "Bob") + + def test_set_display_name_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"displayname": "Bob"}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_set_display_name_missing_displayname(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_set_display_name_very_long(self) -> None: + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + ) + ) + + long_name = "A" * 1000 + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "displayname": long_name}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_set_display_name_special_characters(self) -> None: + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + ) + ) + + special_names = [ + "Alice 👋", # Emoji + "Alice & Bob", # HTML entities + "Alice\nNewline", # Newline + "Alice\tTab", # Tab + 'Alice"Quote', # Quote + "Alice'Apostrophe", # Apostrophe + "Alice