diff --git a/.dockerignore b/.dockerignore index 0b51345cbd..7b09e18b43 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,7 @@ # ignore everything by default * -# things to include +# Things to include !docker !synapse !rust @@ -12,6 +12,7 @@ !Cargo.toml !build_rust.py +# Things to re-ignore rust/target synapse/*.so diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dc5bcaed14..8617b871e4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,31 +1,35 @@ +# Configure general package update PRs filed by dependabot. +# Security update PRs are generally unaffected by these settings. version: 2 -# As dependabot is currently only run on a weekly basis, we raise the -# open-pull-requests-limit to 10 (from the default of 5) to better ensure we -# don't continuously grow a backlog of updates. + updates: - - # "pip" is the correct setting for poetry, per https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem + - # "pip" is the correct setting for poetry, per + # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem package-ecosystem: "pip" directory: "/" - open-pull-requests-limit: 10 - versioning-strategy: "increase-if-necessary" - schedule: - interval: "weekly" - # Group patch updates to packages together into a single PR, as they rarely - # if ever contain breaking changes that need to be reviewed separately. - # - # Less PRs means a streamlined review process. + # Only bump the minimum and maximum versions of dependencies in our + # lockfile. The maximum version in `pyproject.toml` may be bumped, but the + # minimum may not be. # - # Python packages follow semantic versioning, and tend to only introduce - # breaking changes in major version bumps. Thus, we'll group minor and patch - # versions together. + # We want to avoid bumping the minimum version in `pyproject.toml` unless + # necessary, as it requires our supported Linux distributions to have + # those versions in their repos first. + versioning-strategy: "increase-if-necessary" + # Group together all updates for this ecosystem into a single, big PR. + # + # Doing this prevents us from needing to review a PR per dependency upgrade + # (which can become burdensome). + # + # Splitting up each ecosystem into their own PR (rather than putting ALL + # upgrades into a single PR) allows us to more quicker determine the culprit + # when CI fails. groups: - minor-and-patches: + all-python-dependencies: applies-to: version-updates patterns: + # Update all non-transitive dependencies. - "*" - update-types: - - "minor" - - "patch" + open-pull-requests-limit: 1 # Prevent pulling packages that were recently updated to help mitigate # supply chain attacks. 14 days was taken from the recommendation at # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns @@ -36,58 +40,57 @@ updates: # be pulled in as soon as possible. cooldown: default-days: 14 - - - package-ecosystem: "docker" - directory: "/docker" - open-pull-requests-limit: 10 schedule: - interval: "weekly" - # For container versions, breaking changes are also typically only introduced in major - # package bumps. + interval: "monthly" + time: "08:00" + timezone: "Europe/London" + + - package-ecosystem: "cargo" + directory: "/" + open-pull-requests-limit: 1 groups: - minor-and-patches: + all-rust-dependencies: applies-to: version-updates patterns: - "*" - update-types: - - "minor" - - "patch" + schedule: + interval: "monthly" + time: "08:00" + timezone: "Europe/London" cooldown: default-days: 14 - package-ecosystem: "github-actions" directory: "/" - open-pull-requests-limit: 10 - schedule: - interval: "weekly" - # Similarly for GitHub Actions, breaking changes are typically only introduced in major - # package bumps. + open-pull-requests-limit: 1 groups: - minor-and-patches: + all-github-actions: applies-to: version-updates patterns: - "*" - update-types: - - "minor" - - "patch" + schedule: + interval: "monthly" + time: "08:00" + timezone: "Europe/London" cooldown: default-days: 14 - - package-ecosystem: "cargo" - directory: "/" - open-pull-requests-limit: 10 - versioning-strategy: "lockfile-only" - schedule: - interval: "weekly" - # The Rust ecosystem is special in that breaking changes are often introduced - # in minor version bumps, as packages typically stay pre-1.0 for a long time. - # Thus we specifically keep minor version bumps separate in their own PRs. + - package-ecosystem: "gomod" + directory: "/complement" + open-pull-requests-limit: 1 groups: - patches: + all-go-dependencies: applies-to: version-updates patterns: - "*" - update-types: - - "patch" + schedule: + interval: "monthly" + time: "08:00" + timezone: "Europe/London" cooldown: default-days: 14 + + # Note: "docker" is purposefully excluded from this list. + # + # Our docker containers should be kept up-to-date using container scanning + # technology instead of dependabot. \ No newline at end of file diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index a98ba23f43..cb43c1a106 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -50,7 +50,7 @@ jobs: shell: bash - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 90cfd9becf..127f6bf224 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -54,7 +54,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tailscale - uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 + uses: tailscale/github-action@d1b6cd204f8dceda5b3eaad7f1f767be390056cd # v4.2.0 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} audience: ${{ secrets.TS_AUDIENCE }} @@ -87,7 +87,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: push: true labels: | @@ -151,7 +151,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tailscale - uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 + uses: tailscale/github-action@d1b6cd204f8dceda5b3eaad7f1f767be390056cd # v4.2.0 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} audience: ${{ secrets.TS_AUDIENCE }} @@ -183,7 +183,7 @@ jobs: password: ${{ steps.import-secrets.outputs.OCI_PASSWORD }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Install Cosign uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 31ac11bfcd..5312b72217 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -23,7 +23,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} components: clippy, rustfmt diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index fe324e3dbb..5ddf921e9c 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -80,7 +80,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -154,7 +154,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 75ceb0f533..bfc5d05c34 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -67,7 +67,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Set up docker layer caching uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/schema_diff.yml b/.github/workflows/schema_diff.yml index eaa25b53fa..e511ba64fc 100644 --- a/.github/workflows/schema_diff.yml +++ b/.github/workflows/schema_diff.yml @@ -99,7 +99,7 @@ jobs: # # For same-repo PRs, we can do this directly and we are done. - name: Post sticky PR comment - uses: marocchino/sticky-pull-request-comment@3d7b8546315c63df45a03981d50a43ec19237f80 # v3 + uses: marocchino/sticky-pull-request-comment@28ad303d6acb4ccfc50f20c0044188e7e294b8c4 # v3 # Only run for same-repo PRs if: always() && github.event.pull_request.head.repo.full_name == github.repository with: diff --git a/.github/workflows/schema_diff_comment.yml b/.github/workflows/schema_diff_comment.yml index 4aefc3a123..1da02e019a 100644 --- a/.github/workflows/schema_diff_comment.yml +++ b/.github/workflows/schema_diff_comment.yml @@ -64,7 +64,7 @@ jobs: echo "number=$pr_number" >> "$GITHUB_OUTPUT" - name: Post sticky PR comment - uses: marocchino/sticky-pull-request-comment@3d7b8546315c63df45a03981d50a43ec19237f80 # v3 + uses: marocchino/sticky-pull-request-comment@28ad303d6acb4ccfc50f20c0044188e7e294b8c4 # v3 with: header: schema-diff number_force: ${{ steps.verify.outputs.number }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 90b7f9021a..14a4fcdc4e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -108,7 +108,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -174,7 +174,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -237,7 +237,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: components: clippy toolchain: ${{ env.RUST_VERSION }} @@ -256,7 +256,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_NIGHTLY_VERSION }} components: clippy @@ -274,7 +274,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -310,7 +310,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: # We use nightly so that we can use some unstable options that we use in # `.rustfmt.toml`. @@ -429,7 +429,7 @@ jobs: postgres:${{ matrix.job.postgres-version }} - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -473,7 +473,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -483,7 +483,7 @@ jobs: - run: | sudo apt-get -qq update sudo apt-get -qq install build-essential libffi-dev python3-dev \ - libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev + libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -535,7 +535,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Install libs necessary for PyPy to build binary wheels for dependencies - - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev + - run: sudo apt-get -qq install xmlsec1 libxslt-dev - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.python-version }} @@ -588,7 +588,7 @@ jobs: run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -729,7 +729,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -750,7 +750,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_NIGHTLY_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 9836f743c7..9c3d87abaf 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -45,7 +45,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -73,7 +73,7 @@ jobs: - run: sudo apt-get -qq install xmlsec1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -118,7 +118,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/CHANGES.md b/CHANGES.md index c8ea637bd3..d42c769e4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,60 @@ +# Synapse 1.162.0 (2026-09-29) + +No significant changes since 1.162.0rc1. + + + + +# Synapse 1.162.0rc1 (2026-09-22) + +## Features + +- Raise default room version to "12". Contributed by @jason-famedly @famedly. ([\#20130](https://github.com/element-hq/synapse/issues/20130)) +- Limit the number of end-to-end encryption one-time keys stored per device to 500 per algorithm, rejecting uploads which would exceed the limit with a `400 Bad Request`. ([\#20162](https://github.com/element-hq/synapse/issues/20162)) +- Add support for configuring a `username` for Redis connections, for Redis 6+ ACL authentication. ([\#20187](https://github.com/element-hq/synapse/issues/20187)) +- Add a rate limit on the client profile lookup endpoints, configurable via `rc_profile`. ([\#20218](https://github.com/element-hq/synapse/issues/20218)) + +## Bugfixes + +- Fix flawed [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) partial implementation introduced in Synapse v1.136.0 for invites/knocks: client-side API's like `/sync` use [stripped state events](https://spec.matrix.org/v1.18/client-server-api/#stripped-state) and we now send full PDUs on the federation-side. To allow some time for the ecosystem to adapt and support MSC4311, Synapse will only apply strict validation when receiving invites/knocks after 2027-06-01. ([\#19723](https://github.com/element-hq/synapse/issues/19723)) +- Allow the third party rules callback `check_event_allowed()` to work with [MSC4291](https://github.com/matrix-org/matrix-spec-proposals/pull/4291) rooms. Contributed by @famedly. ([\#19768](https://github.com/element-hq/synapse/issues/19768)) +- Improve server concurrency by opening local media thumbnails asynchronously. Contributed by @guillemo12. ([\#20100](https://github.com/element-hq/synapse/issues/20100)) +- Drop incoming federation device list updates from non-compliant (grandfathered historical) user IDs, so that such user IDs are not forwarded to clients outside the context of an event, as per [the spec](https://spec.matrix.org/v1.14/appendices/#historical-user-ids). ([\#20115](https://github.com/element-hq/synapse/issues/20115)) +- Stop treating unset display names and avatar URLs as profile fields with a `null` value. ([\#20145](https://github.com/element-hq/synapse/issues/20145)) +- Return `allowed_room_ids` in the client [`GET /_matrix/client/v1/rooms/{roomId}/hierarchy`](https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1roomsroomidhierarchy) response, as required since Matrix 1.15. ([\#20154](https://github.com/element-hq/synapse/issues/20154)) +- Fix state events being omitted from the [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) `state_after` sync response when the client's `since` token falls inside an event persistence batch, as could happen on worker deployments. ([\#20171](https://github.com/element-hq/synapse/issues/20171)) +- Return the stable `M_UNKNOWN_DEVICE` error code, added in Matrix 1.17, instead of its unstable [MSC4326](https://github.com/matrix-org/matrix-spec-proposals/pull/4326)-prefixed identifier. ([\#20181](https://github.com/element-hq/synapse/issues/20181)) +- Fix slow recursive `/relations` requests in large rooms by joining events inside the recursive query. ([\#20182](https://github.com/element-hq/synapse/issues/20182)) +- Fix a bug introduced in Synapse 1.135.0 where fetching an unset `displayname` or `avatar_url` via `GET /_matrix/client/v3/profile/{userId}/{field}` returned the field with a `null` value instead of an empty object. ([\#20200](https://github.com/element-hq/synapse/issues/20200)) +- Add support for un-soft-failing [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) when room state changes, making federation support more reliable. ([\#20204](https://github.com/element-hq/synapse/issues/20204)) + +## Improved Documentation + +- Document the paths that can be handled on workers with stabilised delegated authentication. ([\#20209](https://github.com/element-hq/synapse/issues/20209)) +- Document the endpoint for getting a single delayed event is workerisable. ([\#20210](https://github.com/element-hq/synapse/issues/20210)) +- Fix small warnings in documentation building tooling output. ([\#20217](https://github.com/element-hq/synapse/issues/20217)) +- Replace the stale minimum `poetry` version in the contributing docs with a pointer to the minimum defined in `pyproject.toml`. ([\#20219](https://github.com/element-hq/synapse/issues/20219)) +- Added docs regarding firewall configuration. Contributed by @HarisDotParis. ([\#20225](https://github.com/element-hq/synapse/issues/20225)) + +## Internal Changes + +- Port the logcontext machinery (`LoggingContext`, `ContextResourceUsage` and the current-context storage) to Rust. ([\#19979](https://github.com/element-hq/synapse/issues/19979)) +- Refactor the Rust code to have a single place to store per-homeserver state. ([\#20011](https://github.com/element-hq/synapse/issues/20011)) +- Add a `synapse_storage_stream_current_position` metric, reporting each stream's current position as each worker process sees it. ([\#20097](https://github.com/element-hq/synapse/issues/20097)) +- Add HTTP serving functions for future [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) work. ([\#20133](https://github.com/element-hq/synapse/issues/20133)) +- Add a cache for looking up individual pieces of current room state. ([\#20160](https://github.com/element-hq/synapse/issues/20160)) +- Run the in-repo Complement test suite in CI, even when the standard Complement suite fails. ([\#20161](https://github.com/element-hq/synapse/issues/20161)) +- Refactor the federation transmission code to delineate transaction preparation and completion. ([\#20166](https://github.com/element-hq/synapse/issues/20166)) +- Add a cache to state resolution keyed off the conflicted events. ([\#20185](https://github.com/element-hq/synapse/issues/20185)) +- Improve the rendering of the set inequality errors produced by `assertEqual` in the tests. ([\#20193](https://github.com/element-hq/synapse/issues/20193)) +- Fix `/room_summary` returning stale `join_rules`. Contributed by @famedly. ([\#20205](https://github.com/element-hq/synapse/issues/20205)) +- Fix the `Schema Diff` CI check failing to post a comment on pull requests from forks. ([\#20207](https://github.com/element-hq/synapse/issues/20207)) +- Move documentation building python dependencies into `pyproject.toml` under a new `docs` dependency group. ([\#20216](https://github.com/element-hq/synapse/issues/20216)) +- Align comments in `.dockerignore` with what we have in the private Synapse Pro branch. ([\#20224](https://github.com/element-hq/synapse/issues/20224)) + + + + # Synapse 1.161.0 (2026-09-15) No significant changes since 1.161.0rc1. diff --git a/Cargo.lock b/Cargo.lock index 13228a5542..a609527b59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,6 +46,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bitflags" version = "2.9.1" @@ -54,11 +60,11 @@ checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "blake2" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -70,6 +76,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -111,9 +126,21 @@ checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core 0.10.1", + "rand_core", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "core-foundation" version = "0.10.1" @@ -167,15 +194,44 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", - "subtle", + "block-buffer 0.10.4", + "crypto-common 0.1.6", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -323,22 +379,10 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasi 0.14.2+wasi-0.2.4", -] - [[package]] name = "getrandom" version = "0.4.3" @@ -348,8 +392,8 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi 6.0.0", - "rand_core 0.10.1", + "r-efi", + "rand_core", "wasm-bindgen", ] @@ -384,7 +428,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "headers-core", "http", @@ -459,6 +503,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.6.0" @@ -502,7 +555,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -514,7 +567,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2 0.6.5", "tokio", "tower-service", "tracing", @@ -701,9 +754,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" dependencies = [ "either", ] @@ -732,9 +785,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" @@ -774,13 +827,13 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mio" -version = "1.0.4" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", + "wasi", + "windows-sys 0.61.2", ] [[package]] @@ -823,15 +876,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "proc-macro2" version = "1.0.95" @@ -951,7 +995,7 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand", "rand_pcg", "ring", "rustc-hash", @@ -987,28 +1031,12 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" -dependencies = [ - "rand_chacha", - "rand_core 0.9.3", -] - [[package]] name = "rand" version = "0.10.2" @@ -1017,26 +1045,7 @@ checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.3", + "rand_core", ] [[package]] @@ -1051,7 +1060,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -1089,7 +1098,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -1314,18 +1323,18 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -1358,12 +1367,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1406,7 +1415,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.23.1", "blake2", "bytes", "futures", @@ -1417,6 +1426,7 @@ dependencies = [ "icu_segmenter", "itertools", "lazy_static", + "libc", "log", "mime", "once_cell", @@ -1506,15 +1516,15 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", "mio", "pin-project-lite", - "socket2 0.6.0", + "socket2 0.6.5", "windows-sys 0.61.2", ] @@ -1619,11 +1629,11 @@ checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "ulid" -version = "1.2.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +checksum = "947dde63b6d514cc5e044edad4e0ca7261afd1099d16c83d942cb2b2f348689c" dependencies = [ - "rand 0.9.4", + "rand", "web-time", ] @@ -1677,15 +1687,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -1887,15 +1888,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - [[package]] name = "writeable" version = "0.6.1" @@ -1926,26 +1918,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "zerocopy" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", -] - [[package]] name = "zerofrom" version = "0.1.6" diff --git a/README.rst b/README.rst index 4db5d461a8..599dafefb7 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ below for more useful documentation links. Enterprise quality support for Synapse including SLAs is available as part of an `Element Server Suite (ESS) `_ subscription. -If you are an existing ESS subscriber then you can raise a `support request `_ +If you are an existing ESS subscriber then you can raise a `support request `_ and access the `Element product documentation `_. 🤝 Community support diff --git a/changelog.d/19070.doc b/changelog.d/19070.doc new file mode 100644 index 0000000000..0be0703a60 --- /dev/null +++ b/changelog.d/19070.doc @@ -0,0 +1 @@ +Update ESS support link to point to the new customer portal domain. \ No newline at end of file diff --git a/changelog.d/19301.misc b/changelog.d/19301.misc new file mode 100644 index 0000000000..a8e802bdf8 --- /dev/null +++ b/changelog.d/19301.misc @@ -0,0 +1 @@ +Switch to `beautifulsoup4` from `lxml` for URL previews. Contributed by @clokep. \ No newline at end of file diff --git a/changelog.d/19768.bugfix b/changelog.d/19768.bugfix deleted file mode 100644 index 8314491ced..0000000000 --- a/changelog.d/19768.bugfix +++ /dev/null @@ -1 +0,0 @@ -Allow the Third Party Rules `check_event_allowed()` callback to work with [MSC4291](https://github.com/matrix-org/matrix-spec-proposals/pull/4291) rooms. Contributed by @famedly. diff --git a/changelog.d/20095.misc b/changelog.d/20095.misc new file mode 100644 index 0000000000..7e0d0c58c7 --- /dev/null +++ b/changelog.d/20095.misc @@ -0,0 +1 @@ +Add a `synapse_notifier_wait_for_stream_token_timeouts` metric, counting the times a request gave up waiting for a worker to catch up to a stream token, labelled by the lagging stream. diff --git a/changelog.d/20108.bugfix b/changelog.d/20108.bugfix new file mode 100644 index 0000000000..6a842b22ea --- /dev/null +++ b/changelog.d/20108.bugfix @@ -0,0 +1 @@ +Fix a bug where read receipts could be permanently skipped and never sent to application services if more than 100 read receipts arrived in a single stream update. diff --git a/changelog.d/20115.bugfix b/changelog.d/20115.bugfix deleted file mode 100644 index 4cb368a1b3..0000000000 --- a/changelog.d/20115.bugfix +++ /dev/null @@ -1 +0,0 @@ -Drop incoming federation device list updates from non-compliant (grandfathered historical) user IDs, so that such user IDs are not forwarded to clients outside the context of an event, as per [the spec](https://spec.matrix.org/v1.14/appendices/#historical-user-ids). diff --git a/changelog.d/20133.misc b/changelog.d/20133.misc deleted file mode 100644 index 9590db6934..0000000000 --- a/changelog.d/20133.misc +++ /dev/null @@ -1 +0,0 @@ -Add HTTP serving functions for future [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) work. diff --git a/changelog.d/20137.feature b/changelog.d/20137.feature new file mode 100644 index 0000000000..025f9b50cf --- /dev/null +++ b/changelog.d/20137.feature @@ -0,0 +1 @@ +Advertise support for Matrix v1.14. Contributed by @famedly. diff --git a/changelog.d/20145.bugfix b/changelog.d/20145.bugfix deleted file mode 100644 index e2ce9d52db..0000000000 --- a/changelog.d/20145.bugfix +++ /dev/null @@ -1 +0,0 @@ -Stop treating unset display names and avatar URLs as profile fields with a `null` value. \ No newline at end of file diff --git a/changelog.d/20161.misc b/changelog.d/20161.misc deleted file mode 100644 index 4351f779c2..0000000000 --- a/changelog.d/20161.misc +++ /dev/null @@ -1 +0,0 @@ -Run the in-repo Complement test suite in CI, even when the standard Complement suite fails. \ No newline at end of file diff --git a/changelog.d/20162.feature b/changelog.d/20162.feature deleted file mode 100644 index a197d13664..0000000000 --- a/changelog.d/20162.feature +++ /dev/null @@ -1 +0,0 @@ -Limit the number of end-to-end encryption one-time keys stored per device to 500 per algorithm, rejecting uploads which would exceed the limit with a `400 Bad Request`. diff --git a/changelog.d/20166.misc b/changelog.d/20166.misc deleted file mode 100644 index 716e910f8f..0000000000 --- a/changelog.d/20166.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor the federation transmission code to delineate transaction preparation and completion. \ No newline at end of file diff --git a/changelog.d/20171.bugfix b/changelog.d/20171.bugfix deleted file mode 100644 index ab54bf8df4..0000000000 --- a/changelog.d/20171.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix state events being omitted from the [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) `state_after` sync response when the client's `since` token falls inside an event persistence batch, as could happen on worker deployments. diff --git a/changelog.d/20181.bugfix b/changelog.d/20181.bugfix deleted file mode 100644 index 477902b97f..0000000000 --- a/changelog.d/20181.bugfix +++ /dev/null @@ -1 +0,0 @@ -Return the stable `M_UNKNOWN_DEVICE` error code, added in Matrix 1.17, instead of its unstable [MSC4326](https://github.com/matrix-org/matrix-spec-proposals/pull/4326)-prefixed identifier. diff --git a/changelog.d/20182.bugfix b/changelog.d/20182.bugfix deleted file mode 100644 index 637c3efb96..0000000000 --- a/changelog.d/20182.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix slow recursive `/relations` requests in large rooms by joining events inside the recursive query. diff --git a/changelog.d/20187.feature b/changelog.d/20187.feature deleted file mode 100644 index d5c53f1510..0000000000 --- a/changelog.d/20187.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for configuring a `username` for Redis connections, for Redis 6+ ACL authentication. diff --git a/changelog.d/20193.misc b/changelog.d/20193.misc deleted file mode 100644 index 03ab2b0f7e..0000000000 --- a/changelog.d/20193.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the rendering of the set inequality errors produced by `assertEqual` in the tests. \ No newline at end of file diff --git a/changelog.d/20200.bugfix b/changelog.d/20200.bugfix deleted file mode 100644 index 41eadbcf6e..0000000000 --- a/changelog.d/20200.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.135.0 where fetching an unset `displayname` or `avatar_url` via `GET /_matrix/client/v3/profile/{userId}/{field}` returned the field with a `null` value instead of an empty object. diff --git a/changelog.d/20203.bugfix b/changelog.d/20203.bugfix new file mode 100644 index 0000000000..125afaad56 --- /dev/null +++ b/changelog.d/20203.bugfix @@ -0,0 +1 @@ +Tell clients to remove local profile information for users after leaving the last room shared with them. \ No newline at end of file diff --git a/changelog.d/20204.bugfix b/changelog.d/20204.bugfix deleted file mode 100644 index 783bedee14..0000000000 --- a/changelog.d/20204.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add support for un-soft-failing [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) when room state changes, making federation support more reliable. \ No newline at end of file diff --git a/changelog.d/20205.misc b/changelog.d/20205.misc deleted file mode 100644 index 5a5d03b44a..0000000000 --- a/changelog.d/20205.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `/room_summary` returning stale `join_rules`. Contributed by @famedly. diff --git a/changelog.d/20207.misc b/changelog.d/20207.misc deleted file mode 100644 index 3709ad46f1..0000000000 --- a/changelog.d/20207.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the `Schema Diff` CI check failing to post a comment on pull requests from forks. diff --git a/changelog.d/20209.doc b/changelog.d/20209.doc deleted file mode 100644 index cfee70c67c..0000000000 --- a/changelog.d/20209.doc +++ /dev/null @@ -1 +0,0 @@ -Document the paths that can be handled on workers with stabilised delegated authentication. diff --git a/changelog.d/20210.doc b/changelog.d/20210.doc deleted file mode 100644 index 3b0f828036..0000000000 --- a/changelog.d/20210.doc +++ /dev/null @@ -1 +0,0 @@ -Document the endpoint for getting a single delayed event is workerisable. diff --git a/changelog.d/20211.bugfix b/changelog.d/20211.bugfix new file mode 100644 index 0000000000..363ac18bd0 --- /dev/null +++ b/changelog.d/20211.bugfix @@ -0,0 +1 @@ +Do not include `og:image` in responses to `/preview_url` when the media was automatically quarantined by hash matching. diff --git a/changelog.d/20216.misc b/changelog.d/20216.misc deleted file mode 100644 index 130a5b1339..0000000000 --- a/changelog.d/20216.misc +++ /dev/null @@ -1 +0,0 @@ -Move documentation building python dependencies into `pyproject.toml` under a new `docs` dependency group. \ No newline at end of file diff --git a/changelog.d/20217.doc b/changelog.d/20217.doc deleted file mode 100644 index a7d4590cfa..0000000000 --- a/changelog.d/20217.doc +++ /dev/null @@ -1 +0,0 @@ -Fix small warnings in documentation building tooling output. \ No newline at end of file diff --git a/changelog.d/20227.feature b/changelog.d/20227.feature new file mode 100644 index 0000000000..e50010aed0 --- /dev/null +++ b/changelog.d/20227.feature @@ -0,0 +1 @@ +Add push rules for MSC4075: MatrixRTC invites and notifications. diff --git a/changelog.d/20230.feature b/changelog.d/20230.feature new file mode 100644 index 0000000000..769361fa33 --- /dev/null +++ b/changelog.d/20230.feature @@ -0,0 +1 @@ +Make the task scheduler concurrency configurable via `task_scheduler.max_concurrent_tasks`, defaulting to 5. diff --git a/changelog.d/20231.bugfix b/changelog.d/20231.bugfix new file mode 100644 index 0000000000..cc9b403e3e --- /dev/null +++ b/changelog.d/20231.bugfix @@ -0,0 +1 @@ +Requires `require_auth_for_profile_requests` to be enabled to enable `limit_profile_requests_to_users_who_share_rooms`. diff --git a/changelog.d/20232.bugfix b/changelog.d/20232.bugfix new file mode 100644 index 0000000000..c7f4645dc2 --- /dev/null +++ b/changelog.d/20232.bugfix @@ -0,0 +1 @@ +Prevent [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) being sticky when they were redacted prior to persistence. \ No newline at end of file diff --git a/changelog.d/20233.bugfix b/changelog.d/20233.bugfix new file mode 100644 index 0000000000..c9d5a6ee8e --- /dev/null +++ b/changelog.d/20233.bugfix @@ -0,0 +1 @@ +Send all of a room's [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) down legacy `/sync` when a user joins it. diff --git a/changelog.d/20237.bugfix b/changelog.d/20237.bugfix new file mode 100644 index 0000000000..3d6c6bf027 --- /dev/null +++ b/changelog.d/20237.bugfix @@ -0,0 +1 @@ +Fix a bug where push badge counts could ignore notifications in one room based on another room's summarised counts. diff --git a/changelog.d/20241.bugfix b/changelog.d/20241.bugfix new file mode 100644 index 0000000000..3478b28dd3 --- /dev/null +++ b/changelog.d/20241.bugfix @@ -0,0 +1 @@ +Reject user creation via the Admin API when Synapse is delegating authentication to MAS. diff --git a/changelog.d/20246.bugfix b/changelog.d/20246.bugfix new file mode 100644 index 0000000000..e2095ba861 --- /dev/null +++ b/changelog.d/20246.bugfix @@ -0,0 +1 @@ +Fix spurious `Closing scope ... which is not the currently-active one` errors being logged when writing large responses with tracing enabled. diff --git a/changelog.d/20247.feature b/changelog.d/20247.feature new file mode 100644 index 0000000000..4dbffdc07e --- /dev/null +++ b/changelog.d/20247.feature @@ -0,0 +1 @@ +Cancel a user's scheduled [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140) delayed events when their account is deactivated. diff --git a/changelog.d/20252.misc b/changelog.d/20252.misc new file mode 100644 index 0000000000..a8e1359f34 --- /dev/null +++ b/changelog.d/20252.misc @@ -0,0 +1 @@ +Avoid taking the GIL on Tokio worker threads when completing Rust futures. diff --git a/changelog.d/20253.misc b/changelog.d/20253.misc new file mode 100644 index 0000000000..54bf15e0e9 --- /dev/null +++ b/changelog.d/20253.misc @@ -0,0 +1 @@ +Bump `cryptography` from `46.0.7` to `50.0.1` to resolve `CVE-2026-69248`, `CVE-2026-69249` and `CVE-2026-69247`, none of which affected Synapse. Also bump `pyOpenSSL` from `26.0.0` to `26.4.0` for compatibility with newer `cryptography` versions. \ No newline at end of file diff --git a/changelog.d/20255.misc b/changelog.d/20255.misc new file mode 100644 index 0000000000..5be8bda1ce --- /dev/null +++ b/changelog.d/20255.misc @@ -0,0 +1 @@ +Switch dependabot version upgrades to monthly "big bang" upgrades in a single PR per ecosystem to reduce team maintenance burden. Security update PRs remain individual and active. \ No newline at end of file diff --git a/changelog.d/20256.removal b/changelog.d/20256.removal new file mode 100644 index 0000000000..e258976f5f --- /dev/null +++ b/changelog.d/20256.removal @@ -0,0 +1 @@ +Remove the ability to send state events that are sticky, as this feature was removed from [MSC4354: Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354). \ No newline at end of file diff --git a/changelog.d/20258.misc b/changelog.d/20258.misc new file mode 100644 index 0000000000..d74f377f2d --- /dev/null +++ b/changelog.d/20258.misc @@ -0,0 +1,2 @@ +Add type annotations for `RestServlet.PATTERNS` to prevent misuse. + diff --git a/changelog.d/20259.misc b/changelog.d/20259.misc new file mode 100644 index 0000000000..5a015cf0bc --- /dev/null +++ b/changelog.d/20259.misc @@ -0,0 +1 @@ +Bump `Twisted` in `poetry.lock` from `25.5.0` to `26.4.0`. \ No newline at end of file diff --git a/changelog.d/20262.docker b/changelog.d/20262.docker new file mode 100644 index 0000000000..be2a5142b7 --- /dev/null +++ b/changelog.d/20262.docker @@ -0,0 +1 @@ +Add worker support to the endpoint for retrieving a single [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140) delayed event. diff --git a/changelog.d/20263.doc b/changelog.d/20263.doc new file mode 100644 index 0000000000..7521d48941 --- /dev/null +++ b/changelog.d/20263.doc @@ -0,0 +1 @@ +Clarify the `--exists-ok` help text in `register_new_matrix_user`. diff --git a/changelog.d/20267.misc b/changelog.d/20267.misc new file mode 100644 index 0000000000..e6321a6acc --- /dev/null +++ b/changelog.d/20267.misc @@ -0,0 +1 @@ +Stop taking the GIL on Tokio worker threads when calling into Python from Rust. diff --git a/changelog.d/20270.misc b/changelog.d/20270.misc new file mode 100644 index 0000000000..8269e81946 --- /dev/null +++ b/changelog.d/20270.misc @@ -0,0 +1 @@ +Keep in-repo Complement go dependencies up to date with dependabot. \ No newline at end of file diff --git a/changelog.d/20272.bugfix b/changelog.d/20272.bugfix new file mode 100644 index 0000000000..6b545eefcc --- /dev/null +++ b/changelog.d/20272.bugfix @@ -0,0 +1 @@ +Prevent `trial` tests from hanging when running with Twisted 26.4.0 against Postgres. \ No newline at end of file diff --git a/changelog.d/20274.bugfix b/changelog.d/20274.bugfix new file mode 100644 index 0000000000..88d371cd7a --- /dev/null +++ b/changelog.d/20274.bugfix @@ -0,0 +1 @@ +Fix a bug where sending a `null` or non-object `auth` to an endpoint requiring user-interactive authentication returned a 500 error. Contributed by @ankit373. diff --git a/changelog.d/20283.misc b/changelog.d/20283.misc new file mode 100644 index 0000000000..4de827995b --- /dev/null +++ b/changelog.d/20283.misc @@ -0,0 +1 @@ +Ignore suspended users failing to join new room when using the `new_room_user_id` option with the admin API to delete a room. diff --git a/changelog.d/20285.misc b/changelog.d/20285.misc new file mode 100644 index 0000000000..3eb3f49f99 --- /dev/null +++ b/changelog.d/20285.misc @@ -0,0 +1 @@ +Prune `user_ips` less frequently (less stress on the database). diff --git a/changelog.d/20286.feature b/changelog.d/20286.feature new file mode 100644 index 0000000000..6331fe13f9 --- /dev/null +++ b/changelog.d/20286.feature @@ -0,0 +1 @@ +Advertise support for Matrix v1.15. diff --git a/changelog.d/20289.bugfix b/changelog.d/20289.bugfix new file mode 100644 index 0000000000..74b7d1f0aa --- /dev/null +++ b/changelog.d/20289.bugfix @@ -0,0 +1 @@ +Fix bug where membership events would not be returned in `/sync` when using lazy-loaded members and using an old token for `since`. diff --git a/complement/go.mod b/complement/go.mod index cea013573b..055409a612 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -5,11 +5,11 @@ go 1.25.0 require ( github.com/matrix-org/complement v0.0.0-20251120181401-44111a2a8a9d github.com/matrix-org/gomatrixserverlib v0.0.0-20250813150445-9f5070a65744 - github.com/tidwall/gjson v1.18.0 + github.com/tidwall/gjson v1.19.0 ) require ( - github.com/docker/docker v28.3.3+incompatible + github.com/docker/docker v28.5.2+incompatible github.com/docker/go-connections v0.5.0 // indirect github.com/hashicorp/go-set/v3 v3.0.0 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect @@ -30,7 +30,6 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 // indirect github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 // indirect diff --git a/complement/go.sum b/complement/go.sum index 4858bc78c8..79ddab0e81 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -18,8 +18,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -31,8 +31,6 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -46,8 +44,6 @@ github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslC github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/go-set/v3 v3.0.0 h1:CaJBQvQCOWoftrBcDt7Nwgo0kdpmrKxar/x2o6pV9JA= github.com/hashicorp/go-set/v3 v3.0.0/go.mod h1:IEghM2MpE5IaNvL+D7X480dfNtxjRXZ6VMpK3C8s2ok= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/matrix-org/complement v0.0.0-20251120181401-44111a2a8a9d h1:s2Xc9GB2E/pXdElP18h8+04Y3SmhaII7xh2YmCM7oZc= github.com/matrix-org/complement v0.0.0-20251120181401-44111a2a8a9d/go.mod h1:HioTV089DHLBfljH9QLGifJRE4Avnyk08BXXhCwd4gs= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= @@ -88,8 +84,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= -github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= +github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -97,7 +93,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= @@ -126,18 +121,15 @@ golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= @@ -156,8 +148,6 @@ golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= diff --git a/debian/changelog b/debian/changelog index cebc8490cb..528e6b1d67 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +matrix-synapse-py3 (1.162.0) stable; urgency=medium + + * New synapse release 1.162.0. + + -- Synapse Packaging team Tue, 29 Sep 2026 16:15:16 +0000 + +matrix-synapse-py3 (1.162.0~rc1) stable; urgency=medium + + * New synapse release 1.162.0rc1. + + -- Synapse Packaging team Tue, 22 Sep 2026 18:51:00 +0000 + matrix-synapse-py3 (1.161.0) stable; urgency=medium * New synapse release 1.161.0. diff --git a/demo/start.sh b/demo/start.sh index 471e881f46..5e4e1c10b9 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -151,6 +151,9 @@ for port in 8080 8081 8082; do rc_user_directory: per_second: 1000 burst_count: 1000 + rc_profile: + per_second: 1000 + burst_count: 1000 RC ) echo "${ratelimiting}" >> "$port.config" diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index c5d785fc38..1f97ef12fd 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -202,7 +202,7 @@ WORKERS_CONFIG: dict[str, dict[str, Any]] = { "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload", "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/device_signing/upload$", "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$", - "^/_matrix/client/unstable/org.matrix.msc4140/delayed_events(/.*/restart)?$", + "^/_matrix/client/unstable/org.matrix.msc4140/delayed_events(/[^/]+(/restart)?)?$", ], "shared_extra_conf": {}, "worker_extra_conf": "", diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index f967dda6f2..c3f6a236c2 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -72,7 +72,7 @@ pipx install poetry but see poetry's [installation instructions](https://python-poetry.org/docs/#installation) for other installation methods. -Developing Synapse requires Poetry version 1.3.2 or later. +The minimum poetry version is specified by the `requires-poetry` line in [pyproject.toml](https://github.com/element-hq/synapse/blob/develop/pyproject.toml). Next, open a terminal and install dependencies as follows: diff --git a/docs/log_contexts.md b/docs/log_contexts.md index 76710e10e0..bb2dede66d 100644 --- a/docs/log_contexts.md +++ b/docs/log_contexts.md @@ -2,9 +2,15 @@ To help track the processing of individual requests, synapse uses a '`log context`' to track which request it is handling at any given -moment. This is done via a thread-local variable; a `logging.Filter` is -then used to fish the information back out of the thread-local variable -and add it to each log record. +moment. The "current" log context is stored in the Rust extension +(`synapse.synapse_rust.logcontext`), which resolves it from the current +tokio task (if we are running inside one) and otherwise from the current OS +thread; a `logging.Filter` is then used to fish the information back out and +add it to each log record. Storing it in Rust means a single source of truth is +visible from both Python (the reactor and its thread pool) and Rust (tokio +tasks), so log records emitted from either — including from Rust code polled on +a worker thread — are attributed to the right request. See +[the Rust side](#the-rust-side) below. Logcontexts are also used for CPU and database accounting, so that we can track which requests were responsible for high CPU use or database @@ -18,7 +24,7 @@ Asynchronous functions make the whole thing complicated, so this document descri how it all works, and how to write code which follows the rules. In this document, "awaitable" refers to any object which can be `await`ed. In the context of -Synapse, that normally means either a coroutine or a Twisted +Synapse, that normally means either a coroutine or a Twisted [`Deferred`](https://twistedmatrix.com/documents/current/api/twisted.internet.defer.Deferred.html). ## Logcontexts without asynchronous code @@ -268,7 +274,7 @@ async def main(): await d # Good: This will be logged against the "main" logcontext logger.debug("phew") -``` +``` **Solution 2:** We could also fix this by surrounding the call to `d.callback` with a `PreserveLoggingContext`, which will reset the logcontext to the sentinel before calling @@ -534,9 +540,9 @@ def reset_listener_queue(): So, both ends of the awaitable chain have now dropped their references, and the awaitable chain is now orphaned, and will be garbage-collected at -some point. Note that `await_something_interesting` is a coroutine, +some point. Note that `await_something_interesting` is a coroutine, which Python implements as a generator function. When Python -garbage-collects generator functions, it gives them a chance to +garbage-collects generator functions, it gives them a chance to clean up by making the `await` (or `yield`) raise a `GeneratorExit` exception. In our case, that means that the `__exit__` handler of `PreserveLoggingContext` will carefully restore the request context, but @@ -550,6 +556,30 @@ actually happen too much. Unfortunately, when it does happen, it will lead to leaked logcontexts which are incredibly hard to track down. +## The Rust side + +The "current" logcontext is stored in the Rust extension rather than in a +Python thread-local, so that it is visible from both Python and Rust. +`current_context()` and `set_current_context()` are imported from +`synapse.logging.context` as usual — the Rust storage is an implementation +detail that Python code does not need to care about. + +`set_current_context` only ever runs on a Python thread, i.e. the reactor or one +of its thread pools, where it does the `getrusage` CPU accounting. It is never +called from a tokio worker thread. Therefore, CPU accounting is not captured +when executing purely Rust work. + +What Rust code *does* need to be aware of is that when spawning a future onto +the tokio runtime, the current logcontext must be captured and carried along, so +that log records emitted while the future is polled (including any `log::` +records from dependencies, and any Python invoked back from Rust) are attributed +correctly. Don't use a bare `tokio::spawn`. Instead use +`LogContextHandle::capture(py)` plus `LogContextHandle::scope`, which capture +the caller's logcontext from the Python side and record it on the spawned task; +this is what `create_deferred` does. `current_context()` returns the task's +captured context first, so `LoggingContextFilter` — and therefore `pyo3-log` — +sees the right context on worker threads without any per-log-record work. + ## Debugging logcontext issues Debugging logcontext issues can be tricky as leaking or losing a logcontext will surface @@ -564,3 +594,18 @@ loggers: synapse.logging.context.debug: level: DEBUG ``` + +Note that some of these traces (`LoggingContext(...).__enter__`/`__exit__`) are +emitted from Rust via `pyo3-log`, which caches logger levels for performance. +Configuring the logger in the logging config (as above) works — the cache is +flushed whenever the config is (re)loaded, including on `SIGHUP` — but enabling +the logger *at runtime* (e.g. `setLevel(logging.DEBUG)` from the manhole) will +not surface the Rust-emitted traces until you also flush the cache: + +```python +import logging +from synapse.synapse_rust import reset_logging_config + +logging.getLogger("synapse.logging.context.debug").setLevel(logging.DEBUG) +reset_logging_config() +``` diff --git a/docs/setup/installation.md b/docs/setup/installation.md index c887f9900c..4e7822f49e 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -312,7 +312,7 @@ Installing prerequisites on CentOS or Fedora Linux: ```sh sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \ - libwebp-devel libxml2-devel libxslt-devel libpq-devel \ + libwebp-devel libxslt-devel libpq-devel \ python3-virtualenv libffi-devel openssl-devel python3-devel sudo dnf group install "Development Tools" ``` @@ -638,10 +638,6 @@ This is critical from a security perspective to stop arbitrary Matrix users spidering 'internal' URLs on your network. At the very least we recommend that your loopback and RFC1918 IP addresses are blacklisted. -This also requires the optional `lxml` python dependency to be installed. This -in turn requires the `libxml2` library to be available - on Debian/Ubuntu this -means `apt-get install libxml2-dev`, or equivalent for your OS. - ### Backups Don't forget to take [backups](../usage/administration/backups.md) of your new server! diff --git a/docs/setup/security.md b/docs/setup/security.md index 2c21b494e5..237476a1a6 100644 --- a/docs/setup/security.md +++ b/docs/setup/security.md @@ -39,3 +39,19 @@ mentioned in MXIDs hosted on that server. Following this advice ensures that even if an XSS is found in Synapse, the impact to other applications will be minimal. + +## Firewall configuration + +If your Matrix server is secured by a firewall, firewall rules for two different purposes are required. + +Client access to a Synapse server runs via the client port which is TCP port 443 by default. The client port has to be allowed inbound. + +Federation to exchange messages with other Matrix servers runs via TCP port 8448 by default or alternatively via regular HTTPS TCP port 443 (usually if run with [delegation](../delegate.md) and/or behind a [reverse proxy](../reverse_proxy.md)). So to allow federation in a firewall, make sure to allow the federation port of your Synapse server inbound. + +If you have outbound access limited by a firewall allowlist, make sure both TCP ports 443 **and** 8448 are allowed outbound. + +Direction | Port | Protocol | Description +--------- | ---------------- | -------- | ----------- +Inbound | 443 | TCP | Client access +Inbound | 8448 and/or 443 | TCP | Federation inbound, depending on the federation port used +Outbound | 8448 **and** 443 | TCP | Federation outbound, both ports are usually used diff --git a/docs/upgrade.md b/docs/upgrade.md index 9eac9c5ebc..1febde5555 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -118,6 +118,23 @@ stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.163.0 + +## `limit_profile_requests_to_users_who_share_rooms` now requires `require_auth_for_profile_requests` + +The shared-room check enabled by +[`limit_profile_requests_to_users_who_share_rooms`](usage/configuration/config_documentation.md#limit_profile_requests_to_users_who_share_rooms) +is only applied to authenticated requests, and profile requests are only +authenticated when +[`require_auth_for_profile_requests`](usage/configuration/config_documentation.md#require_auth_for_profile_requests) +is enabled. Enabling the former without the latter therefore never restricted +any profile lookup. + +Synapse will now refuse to start with that combination. If you have +`limit_profile_requests_to_users_who_share_rooms: true` in your configuration, +either also set `require_auth_for_profile_requests: true`, or remove the option to +keep the behaviour you have today. + # Upgrading to v1.161.0 ## Deprecation of `matrix_rtc.livekit_service_url` diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 74ac85256f..d875337c0e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -311,7 +311,8 @@ presence: --- ### `require_auth_for_profile_requests` -*(boolean)* Whether to require authentication to retrieve profile data (avatars, display names) of other users through the client API. Note that profile data is also available via the federation API, unless `allow_profile_lookup_over_federation` is set to false. Defaults to `false`. +*(boolean)* Whether to require authentication to retrieve profile data (avatars, display names) of other users through the client API. Note that profile data is also available via the federation API, unless `allow_profile_lookup_over_federation` is set to false. +This option must be enabled in order to use `limit_profile_requests_to_users_who_share_rooms`. Defaults to `false`. Example configuration: ```yaml @@ -320,7 +321,8 @@ require_auth_for_profile_requests: true --- ### `limit_profile_requests_to_users_who_share_rooms` -*(boolean)* Use this option to require a user to share a room with another user in order to retrieve their profile information. Only checked on Client-Server requests. Profile requests from other servers should be checked by the requesting server. Defaults to `false`. +*(boolean)* Use this option to require a user to share a room with another user in order to retrieve their profile information. Only checked on Client-Server requests. Profile requests from other servers should be checked by the requesting server. +The check can only be applied to authenticated requests, so this option requires `require_auth_for_profile_requests` to be enabled as well. Synapse will refuse to start if this option is enabled without it. Defaults to `false`. Example configuration: ```yaml @@ -375,8 +377,9 @@ For example, for room version 1, `default_room_version` should be set to "1". _Changed in Synapse 1.76:_ the default version room version was increased from [9](https://spec.matrix.org/v1.5/rooms/v9/) to [10](https://spec.matrix.org/v1.5/rooms/v10/). _Changed in Synapse 1.157:_ the default version room version was increased from [10](https://spec.matrix.org/v1.12/rooms/v10/) to [11](https://spec.matrix.org/v1.12/rooms/v11/). +_Changed in Synapse 1.162:_ the default room version was increased from [11](https://spec.matrix.org/v1.16/rooms/v11/) to [12](https://spec.matrix.org/v1.16/rooms/v12/) -Defaults to `"11"`. +Defaults to `"12"`. Example configuration: ```yaml @@ -2115,6 +2118,27 @@ rc_user_directory: burst_count: 200.0 ``` --- +### `rc_profile` + +*(object)* This option allows admins to ratelimit profile lookups by clients. + +Requests are limited per user when the request is authenticated, otherwise per client IP address. + +_Added in Synapse 1.162.0._ + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_profile: + per_second: 1.0 + burst_count: 500.0 +``` +--- ### `federation_rr_transactions_per_room_per_second` *(integer)* Sets outgoing federation transaction frequency for sending read-receipts, per-room. @@ -4619,6 +4643,20 @@ Example configuration: run_background_tasks_on: worker1 ``` --- +### `task_scheduler` + +*(object)* Configuration for the task scheduler. + +This setting has the following sub-options: + +* `max_concurrent_tasks` (integer): The maximum number of tasks that can run concurrently in the task scheduler. Setting this too high may swamp the database connection pool. Defaults to `5`. + +Example configuration: +```yaml +task_scheduler: + max_concurrent_tasks: 5 +``` +--- ### `update_user_directory_from_worker` *(string|null)* The [worker](../../workers.md#updating-the-user-directory) that is used to update the user directory. If not provided this defaults to the main process. diff --git a/docs/workers.md b/docs/workers.md index 5587ba9c6d..6984c1337e 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -289,7 +289,7 @@ information. ^/_matrix/client/(r0|v3|unstable)/user_directory/search$ # Unstable MSC4140 support - ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events(/.*/restart)?$ + ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events(/[^/]+(/restart)?)?$ # Stabilised Delegated Authentication support (`matrix_authentication_service.enabled: true`) ^/_synapse/mas/ @@ -309,9 +309,6 @@ Additionally, the following REST endpoints can be handled for GET requests: # Admin API requests ^/_synapse/admin/v2/users/[^/]+$ - # Unstable MSC4140 support - ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events/[^/]+$ - Pagination requests can also be handled, but all requests for a given room must be routed to the same instance. Additionally, care must be taken to ensure that the purge history admin API is not used while pagination requests diff --git a/flake.nix b/flake.nix index 4ff6518aed..47a86102b9 100644 --- a/flake.nix +++ b/flake.nix @@ -100,7 +100,6 @@ libjpeg libpqxx libwebp - libxml2 libxslt sqlite diff --git a/poetry.lock b/poetry.lock index 38c00e1af0..a19babbcd9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -149,6 +149,30 @@ files = [ tests = ["pytest (>=3.2.1,!=3.3.0)"] typecheck = ["mypy"] +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +description = "Screen-scraping library" +optional = true +python-versions = ">=3.7.0" +groups = ["main"] +markers = "extra == \"url-preview\" or extra == \"all\"" +files = [ + {file = "beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9"}, + {file = "beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7"}, +] + +[package.dependencies] +soupsieve = ">=1.6.1" +typing-extensions = ">=4.0.0" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "bleach" version = "6.4.0" @@ -454,76 +478,66 @@ files = [ [[package]] name = "cryptography" -version = "46.0.7" +version = "50.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false -python-versions = "!=3.9.0,!=3.9.1,>=3.8" +python-versions = "!=3.9.0,!=3.9.1,>=3.9" groups = ["main", "dev"] files = [ - {file = "cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b"}, - {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85"}, - {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e"}, - {file = "cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457"}, - {file = "cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b"}, - {file = "cryptography-46.0.7-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b7b412817be92117ec5ed95f880defe9cf18a832e8cafacf0a22337dc1981b4d"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:60627cf07e0d9274338521205899337c5d18249db56865f943cbe753aa96f40f"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:80406c3065e2c55d7f49a9550fe0c49b3f12e5bfff5dedb727e319e1afb9bf99"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:c5b1ccd1239f48b7151a65bc6dd54bcfcc15e028c8ac126d3fada09db0e07ef1"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2"}, - {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e"}, - {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65814c60f8cc400c63131584e3e1fad01235edba2614b61fbfbfa954082db0ee"}, - {file = "cryptography-46.0.7-cp314-cp314t-win32.whl", hash = "sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298"}, - {file = "cryptography-46.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb"}, - {file = "cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0"}, - {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85"}, - {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e"}, - {file = "cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246"}, - {file = "cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4"}, - {file = "cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5"}, + {file = "cryptography-50.0.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986"}, + {file = "cryptography-50.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f"}, + {file = "cryptography-50.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef"}, + {file = "cryptography-50.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8"}, + {file = "cryptography-50.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45"}, + {file = "cryptography-50.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad"}, + {file = "cryptography-50.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49"}, + {file = "cryptography-50.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f"}, + {file = "cryptography-50.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527"}, + {file = "cryptography-50.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a"}, + {file = "cryptography-50.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959"}, + {file = "cryptography-50.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b"}, + {file = "cryptography-50.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648"}, + {file = "cryptography-50.0.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f"}, + {file = "cryptography-50.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a"}, + {file = "cryptography-50.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367"}, + {file = "cryptography-50.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5"}, + {file = "cryptography-50.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9"}, + {file = "cryptography-50.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0"}, + {file = "cryptography-50.0.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc"}, + {file = "cryptography-50.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17"}, + {file = "cryptography-50.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6"}, + {file = "cryptography-50.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3"}, + {file = "cryptography-50.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6"}, + {file = "cryptography-50.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149"}, + {file = "cryptography-50.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf"}, + {file = "cryptography-50.0.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0"}, + {file = "cryptography-50.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23"}, + {file = "cryptography-50.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733"}, + {file = "cryptography-50.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88"}, + {file = "cryptography-50.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054"}, + {file = "cryptography-50.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5"}, + {file = "cryptography-50.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361"}, + {file = "cryptography-50.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71"}, + {file = "cryptography-50.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80"}, + {file = "cryptography-50.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239"}, + {file = "cryptography-50.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558"}, + {file = "cryptography-50.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e"}, + {file = "cryptography-50.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2"}, + {file = "cryptography-50.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94"}, + {file = "cryptography-50.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f"}, + {file = "cryptography-50.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671"}, + {file = "cryptography-50.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e"}, + {file = "cryptography-50.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6"}, + {file = "cryptography-50.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b"}, + {file = "cryptography-50.0.1.tar.gz", hash = "sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20"}, ] [package.dependencies] -cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11.0\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] -docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox[uv] (>=2024.4.15)"] -pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] -sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==46.0.7)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] -test-randomorder = ["pytest-randomly"] [[package]] name = "defusedxml" @@ -1233,164 +1247,14 @@ files = [ {file = "librt-0.8.1.tar.gz", hash = "sha256:be46a14693955b3bd96014ccbdb8339ee8c9346fbe11c1b78901b55125f14c73"}, ] -[[package]] -name = "lxml" -version = "6.1.0" -description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" -files = [ - {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, - {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, - {file = "lxml-6.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1bc4cc83fb7f66ffb16f74d6dd0162e144333fc36ebcce32246f80c8735b2551"}, - {file = "lxml-6.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:20cf4d0651987c906a2f5cba4e3a8d6ba4bfdf973cfe2a96c0d6053888ea2ecd"}, - {file = "lxml-6.1.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffb34ea45a82dd637c2c97ae1bbb920850c1e59bcae79ce1c15af531d83e7215"}, - {file = "lxml-6.1.0-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1d9b99e5b2597e4f5aed2484fef835256fa1b68a19e4265c97628ef4bf8bcf4"}, - {file = "lxml-6.1.0-cp310-cp310-manylinux_2_28_i686.whl", hash = "sha256:d43aa26dcda363f21e79afa0668f5029ed7394b3bb8c92a6927a3d34e8b610ea"}, - {file = "lxml-6.1.0-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:6262b87f9e5c1e5fe501d6c153247289af42eb44ad7660b9b3de17baaf92d6f6"}, - {file = "lxml-6.1.0-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d1392c569c032f78a11a25d1de1c43fff13294c793b39e19d84fade3045cbbc3"}, - {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:045e387d1f4f42a418380930fa3f45c73c9b392faf67e495e58902e68e8f44a7"}, - {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:9f93d5b8b07f73e8c77e3c6556a3db269918390c804b5e5fcdd4858232cc8f16"}, - {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:de550d129f18d8ab819651ffe4f38b1b713c7e116707de3c0c6400d0ef34fbc1"}, - {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c08da09dc003c9e8c70e06b53a11db6fb3b250c21c4236b03c7d7b443c318e7a"}, - {file = "lxml-6.1.0-cp310-cp310-win32.whl", hash = "sha256:37448bf9c7d7adfc5254763901e2bbd6bb876228dfc1fc7f66e58c06368a7544"}, - {file = "lxml-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:2593a0a6621545b9095b71ad74ed4226eba438a7d9fc3712a99bdb15508cf93a"}, - {file = "lxml-6.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:e80807d72f96b96ad5588cb85c75616e4f2795a7737d4630784c51497beb7776"}, - {file = "lxml-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cec05be8c876f92a5aa07b01d60bbb4d11cfbdd654cad0561c0d7b5c043a61b9"}, - {file = "lxml-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9c03e048b6ce8e77b09c734e931584894ecd58d08296804ca2d0b184c933ce50"}, - {file = "lxml-6.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:942454ff253da14218f972b23dc72fa4edf6c943f37edd19cd697618b626fac5"}, - {file = "lxml-6.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d036ee7b99d5148072ac7c9b847193decdfeac633db350363f7bce4fff108f0e"}, - {file = "lxml-6.1.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ae5d8d5427f3cc317e7950f2da7ad276df0cfa37b8de2f5658959e618ea8512"}, - {file = "lxml-6.1.0-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:363e47283bde87051b821826e71dde47f107e08614e1aa312ba0c5711e77738c"}, - {file = "lxml-6.1.0-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:f504d861d9f2a8f94020130adac88d66de93841707a23a86244263d1e54682f5"}, - {file = "lxml-6.1.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:23a5dc68e08ed13331d61815c08f260f46b4a60fdd1640bbeb82cf89a9d90289"}, - {file = "lxml-6.1.0-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f15401d8d3dbf239e23c818afc10c7207f7b95f9a307e092122b6f86dd43209a"}, - {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fcf3da95e93349e0647d48d4b36a12783105bcc74cb0c416952f9988410846a3"}, - {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:0d082495c5fcf426e425a6e28daaba1fcb6d8f854a4ff01effb1f1f381203eb9"}, - {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e3c4f84b24a1fcba435157d111c4b755099c6ff00a3daee1ad281817de75ed11"}, - {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:976a6b39b1b13e8c354ad8d3f261f3a4ac6609518af91bdb5094760a08f132c4"}, - {file = "lxml-6.1.0-cp311-cp311-win32.whl", hash = "sha256:857efde87d365706590847b916baff69c0bc9252dc5af030e378c9800c0b10e3"}, - {file = "lxml-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:183bfb45a493081943be7ea2b5adfc2b611e1cf377cefa8b8a8be404f45ef9a7"}, - {file = "lxml-6.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:19f4164243fc206d12ed3d866e80e74f5bc3627966520da1a5f97e42c32a3f39"}, - {file = "lxml-6.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d2f17a16cd8751e8eb233a7e41aecdf8e511712e00088bf9be455f604cd0d28d"}, - {file = "lxml-6.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f0cea5b1d3e6e77d71bd2b9972eb2446221a69dc52bb0b9c3c6f6e5700592d93"}, - {file = "lxml-6.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc46da94826188ed45cb53bd8e3fc076ae22675aea2087843d4735627f867c6d"}, - {file = "lxml-6.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9147d8e386ec3b82c3b15d88927f734f565b0aaadef7def562b853adca45784a"}, - {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5715e0e28736a070f3f34a7ccc09e2fdcba0e3060abbcf61a1a5718ff6d6b105"}, - {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4937460dc5df0cdd2f06a86c285c28afda06aefa3af949f9477d3e8df430c485"}, - {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc783ee3147e60a25aa0445ea82b3e8aabb83b240f2b95d32cb75587ff781814"}, - {file = "lxml-6.1.0-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:40d9189f80075f2e1f88db21ef815a2b17b28adf8e50aaf5c789bfe737027f32"}, - {file = "lxml-6.1.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:05b9b8787e35bec69e68daf4952b2e6dfcfb0db7ecf1a06f8cdfbbac4eb71aad"}, - {file = "lxml-6.1.0-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0f0f08beb0182e3e9a86fae124b3c47a7b41b7b69b225e1377db983802404e54"}, - {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73becf6d8c81d4c76b1014dbd3584cb26d904492dcf73ca85dc8bff08dcd6d2d"}, - {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1ae225f66e5938f4fa29d37e009a3bb3b13032ac57eb4eb42afa44f6e4054e69"}, - {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:690022c7fae793b0489aa68a658822cea83e0d5933781811cabbf5ea3bcfe73d"}, - {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:63aeafc26aac0be8aff14af7871249e87ea1319be92090bfd632ec68e03b16a5"}, - {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:264c605ab9c0e4aa1a679636f4582c4d3313700009fac3ec9c3412ed0d8f3e1d"}, - {file = "lxml-6.1.0-cp312-cp312-win32.whl", hash = "sha256:56971379bc5ee8037c5a0f09fa88f66cdb7d37c3e38af3e45cf539f41131ac1f"}, - {file = "lxml-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bba078de0031c219e5dd06cf3e6bf8fb8e6e64a77819b358f53bb132e3e03366"}, - {file = "lxml-6.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3592631e652afa34999a088f98ba7dfc7d6aff0d535c410bea77a71743f3819"}, - {file = "lxml-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a0092f2b107b69601adf562a57c956fbb596e05e3e6651cabd3054113b007e45"}, - {file = "lxml-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fc7140d7a7386e6b545d41b7358f4d02b656d4053f5fa6859f92f4b9c2572c4d"}, - {file = "lxml-6.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:419c58fc92cc3a2c3fa5f78c63dbf5da70c1fa9c1b25f25727ecee89a96c7de2"}, - {file = "lxml-6.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:37fabd1452852636cf38ecdcc9dd5ca4bba7a35d6c53fa09725deeb894a87491"}, - {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2853c8b2170cc6cd54a6b4d50d2c1a8a7aeca201f23804b4898525c7a152cfc"}, - {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e369cbd690e788c8d15e56222d91a09c6a417f49cbc543040cba0fe2e25a79e"}, - {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e69aa6805905807186eb00e66c6d97a935c928275182eb02ee40ba00da9623b2"}, - {file = "lxml-6.1.0-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:4bd1bdb8a9e0e2dd229de19b5f8aebac80e916921b4b2c6ef8a52bc131d0c1f9"}, - {file = "lxml-6.1.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:cbd7b79cdcb4986ad78a2662625882747f09db5e4cd7b2ae178a88c9c51b3dfe"}, - {file = "lxml-6.1.0-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:43e4d297f11080ec9d64a4b1ad7ac02b4484c9f0e2179d9c4ef78e886e747b88"}, - {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cc16682cc987a3da00aa56a3aa3075b08edb10d9b1e476938cfdbee8f3b67181"}, - {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:d6d8efe71429635f0559579092bb5e60560d7b9115ee38c4adbea35632e7fa24"}, - {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e39ab3a28af7784e206d8606ec0e4bcad0190f63a492bca95e94e5a4aef7f6e"}, - {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9eb667bf50856c4a58145f8ca2d5e5be160191e79eb9e30855a476191b3c3495"}, - {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7f4a77d6f7edf9230cee3e1f7f6764722a41604ee5681844f18db9a81ea0ec33"}, - {file = "lxml-6.1.0-cp313-cp313-win32.whl", hash = "sha256:28902146ffbe5222df411c5d19e5352490122e14447e98cd118907ee3fd6ee62"}, - {file = "lxml-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:4a1503c56e4e2b38dc76f2f2da7bae69670c0f1933e27cfa34b2fa5876410b16"}, - {file = "lxml-6.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:e0af85773850417d994d019741239b901b22c6680206f46a34766926e466141d"}, - {file = "lxml-6.1.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ab863fd37458fed6456525f297d21239d987800c46e67da5ef04fc6b3dd93ac8"}, - {file = "lxml-6.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6fd8b1df8254ff4fd93fd31da1fc15770bde23ac045be9bb1f87425702f61cc9"}, - {file = "lxml-6.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:47024feaae386a92a146af0d2aeed65229bf6fff738e6a11dda6b0015fb8fd03"}, - {file = "lxml-6.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3f00972f84450204cd5d93a5395965e348956aaceaadec693a22ec743f8ae3eb"}, - {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97faa0860e13b05b15a51fb4986421ef7a30f0b3334061c416e0981e9450ca4c"}, - {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:972a6451204798675407beaad97b868d0c733d9a74dafefc63120b81b8c2de28"}, - {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fe022f20bc4569ec66b63b3fb275a3d628d9d32da6326b2982584104db6d3086"}, - {file = "lxml-6.1.0-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:75c4c7c619a744f972f4451bf5adf6d0fb00992a1ffc9fd78e13b0bc817cc99f"}, - {file = "lxml-6.1.0-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3648f20d25102a22b6061c688beb3a805099ea4beb0a01ce62975d926944d292"}, - {file = "lxml-6.1.0-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:77b9f99b17cbf14026d1e618035077060fc7195dd940d025149f3e2e830fbfcb"}, - {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32662519149fd7a9db354175aa5e417d83485a8039b8aaa62f873ceee7ea4cad"}, - {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:73d658216fc173cf2c939e90e07b941c5e12736b0bf6a99e7af95459cfe8eabb"}, - {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ac4db068889f8772a4a698c5980ec302771bb545e10c4b095d4c8be26749616f"}, - {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:45e9dfbd1b661eb64ba0d4dbe762bd210c42d86dd1e5bd2bdf89d634231beb43"}, - {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:89e8d73d09ac696a5ba42ec69787913d53284f12092f651506779314f10ba585"}, - {file = "lxml-6.1.0-cp314-cp314-win32.whl", hash = "sha256:ebe33f4ec1b2de38ceb225a1749a2965855bffeef435ba93cd2d5d540783bf2f"}, - {file = "lxml-6.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:398443df51c538bd578529aa7e5f7afc6c292644174b47961f3bf87fe5741120"}, - {file = "lxml-6.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:8c8984e1d8c4b3949e419158fda14d921ff703a9ed8a47236c6eb7a2b6cb4946"}, - {file = "lxml-6.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1081dd10bc6fa437db2500e13993abf7cc30716d0a2f40e65abb935f02ec559c"}, - {file = "lxml-6.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dabecc48db5f42ba348d1f5d5afdc54c6c4cc758e676926c7cd327045749517d"}, - {file = "lxml-6.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e3dd5fe19c9e0ac818a9c7f132a5e43c1339ec1cbbfecb1a938bd3a47875b7c9"}, - {file = "lxml-6.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e7b0a4ca6dcc007a4cef00a761bba2dea959de4bd2df98f926b33c92ca5dfb9"}, - {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d27bbe326c6b539c64b42638b18bc6003a8d88f76213a97ac9ed4f885efeab7"}, - {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4e425db0c5445ef0ad56b0eec54f89b88b2d884656e536a90b2f52aecb4ca86"}, - {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b89b098105b8599dc57adac95d1813409ac476d3c948a498775d3d0c6124bfb"}, - {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:c4a699432846df86cc3de502ee85f445ebad748a1c6021d445f3e514d2cd4b1c"}, - {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:30e7b2ed63b6c8e97cca8af048589a788ab5c9c905f36d9cf1c2bb549f450d2f"}, - {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:022981127642fe19866d2907d76241bb07ed21749601f727d5d5dd1ce5d1b773"}, - {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:23cad0cc86046d4222f7f418910e46b89971c5a45d3c8abfad0f64b7b05e4a9b"}, - {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:21c3302068f50d1e8728c67c87ba92aa87043abee517aa2576cca1855326b405"}, - {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:be10838781cb3be19251e276910cd508fe127e27c3242e50521521a0f3781690"}, - {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2173a7bffe97667bbf0767f8a99e587740a8c56fdf3befac4b09cb29a80276fd"}, - {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c6854e9cf99c84beb004eecd7d3a3868ef1109bf2b1df92d7bc11e96a36c2180"}, - {file = "lxml-6.1.0-cp314-cp314t-win32.whl", hash = "sha256:00750d63ef0031a05331b9223463b1c7c02b9004cef2346a5b2877f0f9494dd2"}, - {file = "lxml-6.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:80410c3a7e3c617af04de17caa9f9f20adaa817093293d69eae7d7d0522836f5"}, - {file = "lxml-6.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:26dd9f57ee3bd41e7d35b4c98a2ffd89ed11591649f421f0ec19f67d50ec67ac"}, - {file = "lxml-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b6c2f225662bc5ad416bdd06f72ca301b31b39ce4261f0e0097017fc2891b940"}, - {file = "lxml-6.1.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a86f06f059e22a0d574990ee2df24ede03f7f3c68c1336293eee9536c4c776cd"}, - {file = "lxml-6.1.0-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:468479e52ecf3ec23799c863336d02c05fc2f7ffd1a1424eeeb9a28d4eb69d13"}, - {file = "lxml-6.1.0-cp38-cp38-manylinux_2_28_i686.whl", hash = "sha256:a02ca8fe48815bddcfca3248efe54451abb9dbf2f7d1c5744c8aa4142d476919"}, - {file = "lxml-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:bb40648d96157f9081886defe13eac99253e663be969ff938a9289eff6e47b72"}, - {file = "lxml-6.1.0-cp38-cp38-win32.whl", hash = "sha256:1dd6a1c3ad4cb674f44525d9957f3e9c209bb6dd9213245195167a281fcc2bdc"}, - {file = "lxml-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:4e2c54d6b47361d0f1d3bc8d4e082ad87201e56ccdcca4d3b9ee3644ff595ec8"}, - {file = "lxml-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:920354904d1cb86577d4b3cfe2830c2dbe81d6f4449e57ada428f1609b5985f7"}, - {file = "lxml-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c871299c595ee004d186f61840f0bfc4941aa3f17c8ba4a565ead7e4f4f820ee"}, - {file = "lxml-6.1.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d0d799ff958655781296ec870d5e2448e75150da2b3d07f13ff5b0c2c35beefd"}, - {file = "lxml-6.1.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ba11752e346bd804ea312ec2eea2532dfa8b8d3261d81a32ef9e6ab16256280"}, - {file = "lxml-6.1.0-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26c5272c6a4bf4cf32d3f5a7890c942b0e04438691157d341616d02cca74d4bd"}, - {file = "lxml-6.1.0-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c53fa3a5a52122d590e847a57ccf955557b9634a7f99ff5a35131321b0a85317"}, - {file = "lxml-6.1.0-cp39-cp39-manylinux_2_28_i686.whl", hash = "sha256:76b958b4ea3104483c20f74866d55aa056546e15ebe83dd7aecd63698f43b755"}, - {file = "lxml-6.1.0-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:8c11b984b5ce6add4dccc7144c7be5d364d298f15b0c6a57da1991baedc750ce"}, - {file = "lxml-6.1.0-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d3829a6e6fd550a219564912d4002c537f65da4c6ae4e093cc34462f4fa027ad"}, - {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:52b0ac6903cf74ebf997eb8c682d2fbac7d1ab7e4c552413eec55868a9b73f39"}, - {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:29f5c00cb7d752bce2c70ebd2d31b0a42f9499ffdd3ecb2f31a5b73ee43031ad"}, - {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:c748ebcb6877de89f48ab90ca96642ac458fff5dec291a2b9337cd4d0934e383"}, - {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:08950a23f296b3f83521577274e3d3b0f3d739bf2e68d01a752e4288bc50d286"}, - {file = "lxml-6.1.0-cp39-cp39-win32.whl", hash = "sha256:11a873c77a181b4fef9c2e357d08ed399542c2af1390101da66720a19c7c9618"}, - {file = "lxml-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:81ff55c70b67d19d52b6fd118a114c0a4c97d799cd3089ff9bd9e2ff4b414ee2"}, - {file = "lxml-6.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:481d6e2104285d9add34f41b42b247b76b61c5b5c26c303c2e9707bbf8bd9a64"}, - {file = "lxml-6.1.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:546b66c0dd1bb8d9fa89d7123e5fa19a8aff3a1f2141eb22df96112afb17b842"}, - {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5cfa1a34df366d9dc0d5eaf420f4cf2bb1e1bebe1066d1c2fc28c179f8a4004c"}, - {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db88156fcf544cdbf0d95588051515cfdfd4c876fc66444eb98bceb5d6db76de"}, - {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07f98f5496f96bf724b1e3c933c107f0cbf2745db18c03d2e13a291c3afd2635"}, - {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4642e04449a1e164b5ff71ffd901ddb772dfabf5c9adf1b7be5dffe1212bc037"}, - {file = "lxml-6.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7da13bb6fbadfafb474e0226a30570a3445cfd47c86296f2446dafbd77079ace"}, - {file = "lxml-6.1.0.tar.gz", hash = "sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13"}, -] - -[package.extras] -cssselect = ["cssselect (>=0.7)"] -html-clean = ["lxml_html_clean"] -html5 = ["html5lib"] -htmlsoup = ["BeautifulSoup4"] - [[package]] name = "lxml-stubs" version = "0.5.1" description = "Type annotations for the lxml package" -optional = false +optional = true python-versions = "*" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "lxml-stubs-0.5.1.tar.gz", hash = "sha256:e0ec2aa1ce92d91278b719091ce4515c12adc1d564359dfaf81efa7d4feab79d"}, {file = "lxml_stubs-0.5.1-py3-none-any.whl", hash = "sha256:1f689e5dbc4b9247cb09ae820c7d34daeb1fdbd1db06123814b856dae7787272"}, @@ -2302,14 +2166,14 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyjwt" -version = "2.13.0" +version = "2.15.1" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728"}, - {file = "pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423"}, + {file = "pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193"}, + {file = "pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8"}, ] [package.dependencies] @@ -2392,19 +2256,19 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=7.4.0)", "pytest-cov (>=2.10.1)", " [[package]] name = "pyopenssl" -version = "26.0.0" +version = "26.4.0" description = "Python wrapper module around the OpenSSL library" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pyopenssl-26.0.0-py3-none-any.whl", hash = "sha256:df94d28498848b98cc1c0ffb8ef1e71e40210d3b0a8064c9d29571ed2904bf81"}, - {file = "pyopenssl-26.0.0.tar.gz", hash = "sha256:f293934e52936f2e3413b89c6ce36df66a0b34ae1ea3a053b8c5020ff2f513fc"}, + {file = "pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c"}, + {file = "pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7"}, ] [package.dependencies] -cryptography = ">=46.0.0,<47" -typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and python_version >= \"3.8\""} +cryptography = ">=49.0.0,<51" +typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\""} [package.extras] docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] @@ -3107,6 +2971,19 @@ files = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] +[[package]] +name = "soupsieve" +version = "2.10" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"url-preview\" or extra == \"all\"" +files = [ + {file = "soupsieve-2.10-py3-none-any.whl", hash = "sha256:8596eb8967d744174820280fa62b4542a2e955bfaccca73ed8a13c6eb8e9b502"}, + {file = "soupsieve-2.10.tar.gz", hash = "sha256:49e9380d7d2905463583bafe285e818c7366a9ed7b3aee221c1ac79c905d8bc0"}, +] + [[package]] name = "sqlglot" version = "30.2.1" @@ -3252,23 +3129,23 @@ markers = {main = "python_version < \"3.11\""} [[package]] name = "tornado" -version = "6.5.7" +version = "6.5.9" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = true python-versions = ">=3.9" groups = ["main"] markers = "extra == \"opentracing\" or extra == \"all\"" files = [ - {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163"}, - {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100"}, - {file = "tornado-6.5.7-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d759e71906ee783f8867b93bf26a265743da4c1e2f4a018464c1ba019862972"}, - {file = "tornado-6.5.7-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a46347a18f23fb92b396beebe0fb78f61dda0cc302445202c16203d8a18848b"}, - {file = "tornado-6.5.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7778b30bef919231265e91c69963ce0f49a1e9c07ac900bbe75b19ce2575ba92"}, - {file = "tornado-6.5.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e726f0c75da7726eec023aa62751ff8878bd2737e34fbdd33b1ae5897d2200f5"}, - {file = "tornado-6.5.7-cp39-abi3-win32.whl", hash = "sha256:f8de3bf12d3efdd0cbe7c8887868198f8a91415e3f29fcf258d9b8eb7b1d9ae4"}, - {file = "tornado-6.5.7-cp39-abi3-win_amd64.whl", hash = "sha256:de942f843533a039ef9fa3d9c88c7cd8a7c94553fb5ad0154270989b3d99a2c4"}, - {file = "tornado-6.5.7-cp39-abi3-win_arm64.whl", hash = "sha256:ff934fce95643af5f11efdae618eaa73d469dc588641e5c8d19295a0c65c4796"}, - {file = "tornado-6.5.7.tar.gz", hash = "sha256:66c513a76cda70d53907bc27cf1447557699c2e95aa48ba27a442ff61c3ddfc2"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:dea3bee433b73d6312d993ce89ad5f2b8d3ed9fde56e79ba5342b9edb78d342a"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:22bed49cdb55292d1d58f46008ba2cc3020e5b84995ef6ae749676c8b5309514"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cc0876a58eee9143476f24901b1b8eb4d52443283ade2b6bca650dd1da04f84"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc0d703cc7ec92b47ebfe236c97cd765c1f2e45744a2e156e6677e54c948274d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:93f9ab0226ef625b94cacb7981a5af104f1735ea02f92282abe5c389109fb31d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5801bbf94dfde8d21087b6aa0b17f3a5dab6110dfe44e7e4492df5de393ac15b"}, + {file = "tornado-6.5.9-cp39-abi3-win32.whl", hash = "sha256:dca9f377e80efe5dd4d52a2776b67492a81a39b2d931de169b6b36d0c1da444c"}, + {file = "tornado-6.5.9-cp39-abi3-win_amd64.whl", hash = "sha256:f810730ac71eadf009e6e5cb6d534d096887310f5649cff19d7e0d19c0bf4ee7"}, + {file = "tornado-6.5.9-cp39-abi3-win_arm64.whl", hash = "sha256:b5f4d92798337260c6d9f9f0e400c7539dd5e55cf93479a3f8508c9aa09c2b3b"}, + {file = "tornado-6.5.9.tar.gz", hash = "sha256:4d868544ebdf2fc155239a74397f65ebfea9b4d3635296c96b5dfb0701c354f5"}, ] [[package]] @@ -3344,14 +3221,14 @@ keyring = ["keyring (>=21.2.0)"] [[package]] name = "twisted" -version = "25.5.0" +version = "26.4.0" description = "An asynchronous networking framework written in Python" optional = false -python-versions = ">=3.8.0" +python-versions = ">=3.9.12" groups = ["main"] files = [ - {file = "twisted-25.5.0-py3-none-any.whl", hash = "sha256:8559f654d01a54a8c3efe66d533d43f383531ebf8d81d9f9ab4769d91ca15df7"}, - {file = "twisted-25.5.0.tar.gz", hash = "sha256:1deb272358cb6be1e3e8fc6f9c8b36f78eb0fa7c2233d2dbe11ec6fee04ea316"}, + {file = "twisted-26.4.0-py3-none-any.whl", hash = "sha256:dc25ea0ebf6511c24f03232ee9f4afa54b291c5d897990e3a39cc4d14a1ef4c0"}, + {file = "twisted-26.4.0.tar.gz", hash = "sha256:dbfd0fe1ee409d0243fdd7a6a6ff14f4948cec1fd78e0376291f805e1501fae9"}, ] [package.dependencies] @@ -3361,26 +3238,26 @@ constantly = ">=15.1" hyperlink = ">=17.1.1" idna = {version = ">=2.4", optional = true, markers = "extra == \"tls\""} incremental = ">=24.7.0" -pyopenssl = {version = ">=21.0.0", optional = true, markers = "extra == \"tls\""} +pyopenssl = {version = ">=25.2.0", optional = true, markers = "extra == \"tls\""} service-identity = {version = ">=18.1.0", optional = true, markers = "extra == \"tls\""} typing-extensions = ">=4.2.0" zope-interface = ">=5" [package.extras] -all-non-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] -conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)"] -dev = ["coverage (>=7.5,<8.0)", "cython-test-exception-raiser (>=1.0.2,<2)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "pydoctor (>=24.11.1,<24.12.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "python-subunit (>=1.4,<2.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)"] -dev-release = ["pydoctor (>=24.11.1,<24.12.0)", "pydoctor (>=24.11.1,<24.12.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"] -gtk-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pygobject", "pygobject", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] +all-non-platform = ["appdirs (>=1.4.0)", "bcrypt (>=3.2.1)", "cryptography (>=38)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyopenssl (>=25.2.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "wsproto"] +conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.2.1)", "cryptography (>=38)"] +dev = ["coverage (>=7.5,<8.0)", "cython-test-exception-raiser (>=1.0.2,<2)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "pydoctor (>=25.4.0,<25.5.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "python-subunit (>=1.4,<2.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)"] +dev-release = ["pydoctor (>=25.4.0,<25.5.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)"] +gtk-platform = ["appdirs (>=1.4.0)", "bcrypt (>=3.2.1)", "cryptography (>=38)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pygobject (<3.52.1) ; python_version < \"3.10\"", "pygobject ; python_version >= \"3.10\"", "pyhamcrest (>=2)", "pyopenssl (>=25.2.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "wsproto"] http2 = ["h2 (>=3.2,<5.0)", "priority (>=1.1.0,<2.0)"] -macos-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] -mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "coverage (>=7.5,<8.0)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (==1.10.1)", "mypy-zope (==1.0.6)", "priority (>=1.1.0,<2.0)", "pydoctor (>=24.11.1,<24.12.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools", "wsproto"] -osx-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] +macos-platform = ["appdirs (>=1.4.0)", "bcrypt (>=3.2.1)", "cryptography (>=38)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyobjc-core (>=12) ; python_version >= \"3.10\"", "pyobjc-framework-cfnetwork (>=12) ; python_version >= \"3.10\"", "pyobjc-framework-cocoa (>=12) ; python_version >= \"3.10\"", "pyopenssl (>=25.2.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "wsproto"] +mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.2.1)", "coverage (>=7.5,<8.0)", "cryptography (>=38)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (==1.19.1)", "mypy-zope (==1.0.14)", "priority (>=1.1.0,<2.0)", "pydoctor (>=25.4.0,<25.5.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=25.2.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools", "wsproto"] +osx-platform = ["appdirs (>=1.4.0)", "bcrypt (>=3.2.1)", "cryptography (>=38)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyobjc-core (>=12) ; python_version >= \"3.10\"", "pyobjc-framework-cfnetwork (>=12) ; python_version >= \"3.10\"", "pyobjc-framework-cocoa (>=12) ; python_version >= \"3.10\"", "pyopenssl (>=25.2.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "wsproto"] serial = ["pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\""] test = ["cython-test-exception-raiser (>=1.0.2,<2)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"] -tls = ["idna (>=2.4)", "pyopenssl (>=21.0.0)", "service-identity (>=18.1.0)"] +tls = ["idna (>=2.4)", "pyopenssl (>=25.2.0)", "service-identity (>=18.1.0)"] websocket = ["wsproto"] -windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "twisted-iocpsupport (>=1.0.2)", "wsproto", "wsproto"] +windows-platform = ["appdirs (>=1.4.0)", "bcrypt (>=3.2.1)", "cryptography (>=38)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyopenssl (>=25.2.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "wsproto"] [[package]] name = "txredisapi" @@ -3615,14 +3492,14 @@ files = [ [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, + {file = "urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3"}, + {file = "urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63"}, ] [package.extras] @@ -3768,7 +3645,7 @@ docs = ["Sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner (>=6.4)"] [extras] -all = ["authlib", "defusedxml", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pympler", "pysaml2", "pytz", "sentry-sdk", "thrift", "tornado", "txredisapi"] +all = ["authlib", "beautifulsoup4", "defusedxml", "hiredis", "jaeger-client", "lxml-stubs", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pympler", "pysaml2", "pytz", "sentry-sdk", "thrift", "tornado", "txredisapi"] cache-memory = ["pympler"] jwt = ["authlib"] matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] @@ -3776,12 +3653,12 @@ oidc = ["authlib"] opentracing = ["jaeger-client", "opentracing", "thrift", "tornado"] postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"] redis = ["hiredis", "txredisapi"] -saml2 = ["defusedxml", "pysaml2", "pytz"] +saml2 = ["defusedxml", "lxml-stubs", "pysaml2", "pytz"] sentry = ["sentry-sdk"] test = ["idna", "parameterized"] -url-preview = ["lxml"] +url-preview = ["beautifulsoup4"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "3e9a70d3ec7ee5edd32ea5e86a0b56e7633e9ed1b59589890a4df8df25a135d5" +content-hash = "b8ee2c063150549de2946f70cd557c427e16a9101052915ef23eb6dd9d8f77eb" diff --git a/pyproject.toml b/pyproject.toml index e1965b71fd..d4819426ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.161.0" +version = "1.162.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ @@ -128,6 +128,7 @@ postgres = [ ] saml2 = [ "pysaml2>=4.5.0", + "lxml-stubs>=0.4.0", # Transitive dependencies from pysaml2 # These dependencies aren't directly required by Synapse. @@ -137,7 +138,7 @@ saml2 = [ "pytz>=2018.3", # via pysaml2 ] oidc = ["authlib>=0.15.1"] -url-preview = ["lxml>=4.6.3"] +url-preview = ["beautifulsoup4>=4.13.0"] sentry = ["sentry-sdk>=0.7.2"] opentracing = [ "jaeger-client>=4.2.0", @@ -179,10 +180,11 @@ all = [ "psycopg2cffi-compat==1.1;platform_python_implementation == 'PyPy'", # saml2 "pysaml2>=4.5.0", + "lxml-stubs>=0.4.0", # oidc and jwt "authlib>=0.15.1", # url-preview - "lxml>=4.6.3", + "beautifulsoup4>=4.13.0", # sentry "sentry-sdk>=0.7.2", # opentracing @@ -267,7 +269,6 @@ dev = [ "ruff==0.14.6", # Typechecking - "lxml-stubs>=0.4.0", "mypy", "mypy-zope", "types-bleach>=4.1.0", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index f6b12c4c1b..2152ad560e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -24,11 +24,12 @@ name = "synapse.synapse_rust" [dependencies] async-trait = "0.1.89" anyhow = "1.0.63" -base64 = "0.22.1" +base64 = "0.23.1" bytes = "1.6.0" headers = "0.4.0" http = "1.1.0" lazy_static = "1.4.0" +libc = "0.2.174" log = "0.4.17" mime = "0.3.17" pyo3 = { version = "0.29.0", features = [ @@ -43,7 +44,7 @@ pyo3 = { version = "0.29.0", features = [ pyo3-log = "0.13.4" pythonize = { version = "0.29.0", features = ["arbitrary_precision"] } regex = "1.6.0" -sha2 = "0.10.8" +sha2 = "0.11.0" serde = { version = "1.0.144", features = ["derive", "rc"] } serde_json = { version = "1.0.85", features = [ "raw_value", @@ -52,7 +53,7 @@ serde_json = { version = "1.0.85", features = [ # precision integers). "arbitrary_precision", ] } -ulid = "1.1.2" +ulid = "3.0.0" icu_segmenter = "2.0.0" reqwest = { version = "0.12.15", default-features = false, features = [ "http2", @@ -63,9 +64,9 @@ http-body-util = "0.1.3" futures = "0.3.31" tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } once_cell = "1.18.0" -itertools = "0.14.0" +itertools = "0.15.0" [build-dependencies] -blake2 = "0.10.4" +blake2 = "0.11.0" hex = "0.4.3" rustc_version = "0.4.1" diff --git a/rust/benches/evaluator.rs b/rust/benches/evaluator.rs index 96169fd45d..e59a6d2f6a 100644 --- a/rust/benches/evaluator.rs +++ b/rust/benches/evaluator.rs @@ -215,6 +215,7 @@ fn bench_eval_message(b: &mut Bencher) { false, false, false, + false, ); b.iter(|| eval.run(&rules, Some("bob"), Some("person"), None)); diff --git a/rust/clippy.toml b/rust/clippy.toml new file mode 100644 index 0000000000..bf2f715c8e --- /dev/null +++ b/rust/clippy.toml @@ -0,0 +1,10 @@ +# Blocking work spawned onto the tokio threadpool is leaked if it is still +# running when the per-homeserver `RustRuntime` is shut down (see +# `rust/src/runtime.rs`). We should avoid spawning blocking work until we figure +# out how to cancel/stop such work on shutdown. +disallowed-methods = [ + { path = "tokio::task::spawn_blocking", reason = "leaks past RustRuntime shutdown; see rust/src/runtime.rs" }, + { path = "tokio::runtime::Runtime::spawn_blocking", reason = "leaks past RustRuntime shutdown; see rust/src/runtime.rs" }, + { path = "tokio::runtime::Handle::spawn_blocking", reason = "leaks past RustRuntime shutdown; see rust/src/runtime.rs" }, + { path = "tokio::task::block_in_place", reason = "blocks a tokio worker thread; see rust/src/runtime.rs" }, +] diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index db00197107..aba29046db 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -48,6 +48,7 @@ pub struct AuthConfig { #[derive(FromPyObject, Clone)] pub struct ServerConfig { pub msc4140_enabled: bool, + pub public_baseurl: String, pub include_profile_updates_in_sync: bool, } diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 3da5defb83..73721bfe89 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -25,7 +25,8 @@ use pyo3::{ }; use tokio::sync::oneshot; -use crate::tokio_runtime::runtime; +use crate::logging::context::with_logcontext; +use crate::runtime::RustRuntime; create_exception!( synapse.synapse_rust.http_client, @@ -71,10 +72,16 @@ fn logging_context_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { /// Creates a twisted deferred from the given future, spawning the task on the /// tokio runtime. /// -/// Does not handle deferred cancellation or contextvars. +/// Does not handle contextvars. +/// +/// TODO: propagate deferred cancellation to the tokio task (via +/// `JoinHandle::abort`). Until then a cancelled request leaves its task +/// running, so the task can outlive the request's logcontext — +/// `run_python_awaitable` defends against the resulting finished-context case, +/// but the work itself is wasted. pub fn create_deferred<'py, F, O>( py: Python<'py>, - reactor: &Bound<'py, PyAny>, + runtime: &RustRuntime, fut: F, ) -> PyResult> where @@ -85,16 +92,23 @@ where let deferred_callback = deferred.getattr("callback")?.unbind(); let deferred_errback = deferred.getattr("errback")?.unbind(); - let rt = runtime(reactor)?; - let handle = rt.handle()?; - let task = handle.spawn(fut); + // Capture the caller's logcontext here (on the reactor thread) and record + // it in the spawned task. Log records emitted while the future is polled + // (on the tokio threads) are then attributed to the context that was + // current when the caller invoked us. See `crate::logging::context`. + let logcontext = crate::logging::context::LogContextHandle::capture(py); - // Unbind the reactor so that we can pass it to the task - let reactor = reactor.clone().unbind(); + let handle = runtime.tokio_handle()?; + let task = handle.spawn(logcontext.scope(fut)); + + // Keep the runtime state (and, through it, the reactor) alive while the + // task is in flight. + let runtime = runtime.clone(); handle.spawn(async move { let res = task.await; - Python::attach(move |py| { + // Once done pass the result to the Twisted reactor thread for handling. + runtime.dispatch_to_twisted(move |py| { // Flatten the panic into standard python error let res = match res { Ok(r) => r, @@ -104,23 +118,19 @@ where }, }; - // Re-bind the reactor - let reactor = reactor.bind(py); - // Send the result to the deferred, via `.callback(..)` or `.errback(..)` - match res { - Ok(obj) => { - reactor - .call_method("callFromThread", (deferred_callback, obj), None) - .expect("callFromThread should not fail"); // There's nothing we can really do with errors here - } - Err(err) => { - reactor - .call_method("callFromThread", (deferred_errback, err), None) - .expect("callFromThread should not fail"); // There's nothing we can really do with errors here - } + let fired = match res { + Ok(obj) => deferred_callback.call1(py, (obj,)), + Err(err) => deferred_errback.call1(py, (err,)), + }; + + if let Err(err) = fired { + // There is nowhere to propagate this to. The closure runs from + // the dispatch reader's `doRead`, and an exception out of that + // makes Twisted drop the reader. Log it instead. + log::error!("Failed to fire a deferred from a Rust future: {err}"); } - }); + }) }); // Make the deferred follow the Synapse logcontext rules @@ -137,7 +147,7 @@ where /// the Twisted reactor and runs to completion regardless of whether the returned Rust /// future is ever polled; awaiting it only observes the result. pub(crate) async fn run_python_awaitable( - reactor: Py, + runtime: &RustRuntime, make_awaitable: F, ) -> PyResult> where @@ -148,98 +158,106 @@ where // Shared between the success and error callbacks (only one ever fires). let sender = Arc::new(Mutex::new(Some(tx))); - Python::attach(|py| -> PyResult<()> { - // Create some deferred success/error callback functions that we will use to get - // the result from Python to Rust. - let success_sender = Arc::clone(&sender); - let on_success = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let value = args.get_item(0)?.unbind(); - if let Some(tx) = success_sender - .lock() - .map_err(|err| { - anyhow::anyhow!("Failed to acquire lock on `success_sender`: {:#}", err) - })? - .take() - { - let _ = tx.send(Ok(value)); - } - Ok(args.py().None()) - }, - )? - .unbind(); + // Capture the logcontext of the calling tokio task (if any). We restore it + // on the reactor thread before driving the awaitable, so Python code + // invoked from Rust (e.g. `DatabasePool.runInteraction`) runs in the same + // logcontext that was current when Python originally called into Rust. Its + // logging and DB-metrics accounting are then attributed to the right + // request. If we are not inside a scoped task, this is `None` and the + // awaitable runs in the sentinel. + let logcontext = crate::logging::context::LogContextHandle::task_current(); - let error_sender = Arc::clone(&sender); - let on_error = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let err = failure_to_pyerr(&args.get_item(0)?); - if let Some(tx) = error_sender - .lock() - .map_err(|err| { - anyhow::anyhow!("Failed to acquire lock on `error_sender`: {:#}", err) - })? - .take() - { - let _ = tx.send(Err(err)); - } - Ok(args.py().None()) - }, - )? - .unbind(); + runtime.dispatch_to_twisted(move |py| { + let started = (|| -> PyResult<()> { + // Create some deferred success/error callback functions that we will use to get + // the result from Python to Rust. + let success_sender = Arc::clone(&sender); + let on_success = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let value = args.get_item(0)?.unbind(); + if let Some(tx) = success_sender + .lock() + .map_err(|err| { + anyhow::anyhow!("Failed to acquire lock on `success_sender`: {:#}", err) + })? + .take() + { + let _ = tx.send(Ok(value)); + } + Ok(args.py().None()) + }, + )? + .unbind(); - // Wrap `make_awaitable` as a Python callable so we can hand it to - // `run_in_background`, which calls it (in the active logcontext) to produce - // the awaitable it then drives. - let awaitable_factory = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let py = args.py(); - Ok(make_awaitable(py)?.unbind()) - }, - )? - .unbind(); + let error_sender = Arc::clone(&sender); + let on_error = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let err = failure_to_pyerr(&args.get_item(0)?); + if let Some(tx) = error_sender + .lock() + .map_err(|err| { + anyhow::anyhow!("Failed to acquire lock on `error_sender`: {:#}", err) + })? + .take() + { + let _ = tx.send(Err(err)); + } + Ok(args.py().None()) + }, + )? + .unbind(); - // Create a function that we will run with the Twisted reactor that will drive - // the Python awaitable. - let starter = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let py = args.py(); + // Wrap `make_awaitable` as a Python callable so we can hand it to + // `run_in_background`, which calls it (in the active logcontext) to produce + // the awaitable it then drives. + let awaitable_factory = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + Ok(make_awaitable(py)?.unbind()) + }, + )? + .unbind(); - // We fire-and-forget using `run_in_background`. Re-using - // `run_in_background` also makes sure the awaitable gets run with the - // current logcontext while following the logcontext rules. - // - // FIXME: Currently runs in the sentinel logcontext because we don't manage it here - let deferred = logging_context_module(py)?.call_method1( - intern!(py, "run_in_background"), - (awaitable_factory.bind(py),), - ); + // Drive the awaitable in the captured logcontext. Restored here + // as we're on the reactor thread (the only thread where the + // context's `main_thread` check passes). + let context = match &logcontext { + Some(handle) => handle.logging_context().map(|ctx| ctx.clone_ref(py)), + // Called from outside any scoped task: the sentinel. + None => None, + }; - let deferred = deferred?; + // Kick off the awaitable, fire-and-forget, via + // `run_in_background`. It calls the factory in the current + // logcontext and follows the logcontext rules from there. In + // particular, it arranges for the reactor to be back at the + // sentinel when the awaitable later completes. + with_logcontext(py, context, || { + let deferred = run_in_background(py, awaitable_factory.bind(py))?; deferred.call_method1( intern!(py, "addCallbacks"), (on_success.bind(py), on_error.bind(py)), )?; - Ok(py.None()) - }, - )?; + Ok(()) + }) + })(); - reactor - .bind(py) - .call_method1(intern!(py, "callFromThread"), (starter,))?; - - Ok(()) + if let Err(err) = started { + // We failed before the awaitable's callbacks were attached, so + // nothing else will ever resolve the channel. Do it ourselves. + if let Some(tx) = sender.lock().ok().and_then(|mut slot| slot.take()) { + let _ = tx.send(Err(err)); + } + } })?; match rx.await { @@ -270,6 +288,26 @@ fn failure_to_pyerr(failure: &Bound<'_, PyAny>) -> PyErr { } } +/// A reference to `synapse.logging.context.run_in_background`. +static RUN_IN_BACKGROUND: OnceCell> = OnceCell::new(); + +/// Call `synapse.logging.context.run_in_background(f)`, which calls `f` in the +/// current logcontext and drives the awaitable it returns to completion, +/// following the logcontext rules. Returns the resulting `Deferred`. +fn run_in_background<'py>(py: Python<'py>, f: &Bound<'py, PyAny>) -> PyResult> { + let run_in_background = RUN_IN_BACKGROUND.get_or_try_init(|| { + logging_context_module(py)? + .getattr("run_in_background") + .map(Into::into) + })?; + + run_in_background + .call1(py, (f,))? + .extract(py) + .map_err(Into::into) +} + +/// A reference to `synapse.logging.context.make_deferred_yieldable`. static MAKE_DEFERRED_YIELDABLE: OnceCell> = OnceCell::new(); /// Given a deferred, make it follow the Synapse logcontext rules diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs index 5df019adff..d268e89837 100644 --- a/rust/src/handlers/mod.rs +++ b/rust/src/handlers/mod.rs @@ -21,7 +21,7 @@ use pyo3::{ Bound, PyResult, Python, }; -use crate::config::SynapseHomeServerConfig; +use crate::homeserver::HomeServer; use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; use crate::storage::store::Store; @@ -36,19 +36,15 @@ struct RustHandlers { impl RustHandlers { #[new] #[pyo3(signature = (homeserver))] - pub fn py_new(py: Python<'_>, homeserver: &Bound<'_, PyAny>) -> PyResult { - let config: SynapseHomeServerConfig = homeserver.getattr("config")?.extract()?; + pub fn py_new(py: Python<'_>, homeserver: HomeServer) -> PyResult { + let config = homeserver.config(py)?; - // The Twisted reactor, used both to drive our Tokio runtime and to - // marshal database work back onto the reactor thread. - let reactor = homeserver.call_method0("get_reactor")?.unbind(); + // The per-homeserver Rust state, which gives us the tokio runtime + // and the Twisted reactor. + let runtime = homeserver.get_rust_runtime(py)?; - // hs.get_datastores().main.db_pool - let db_pool_py = homeserver - .call_method0("get_datastores")? - .getattr("main")? - .getattr("db_pool")?; - let db_pool = PythonDatabasePoolWrapper::new(&db_pool_py, reactor.clone_ref(py))?; + let db_pool = + PythonDatabasePoolWrapper::new(&homeserver.main_database_pool(py)?, runtime.clone())?; // Store is shared across all of the handlers so let's use an `Arc` let store = Arc::new(Store { @@ -64,7 +60,7 @@ impl RustHandlers { versions::VersionsHandler { global_unstable_feature_map: Arc::clone(&global_unstable_feature_map), store: Arc::clone(&store), - reactor: reactor.clone_ref(py), + runtime: runtime.clone(), }, )?; diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index ab4bd9a160..e3419d5020 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -21,6 +21,7 @@ use serde::Serialize; use crate::config::{types::RoomCreationPreset, SynapseHomeServerConfig}; use crate::deferred::create_deferred; +use crate::runtime::RustRuntime; use crate::storage::store::{PerUserExperimentalFeature, Store}; /// `GET /_matrix/client/versions` response @@ -45,9 +46,9 @@ impl<'py> IntoPyObject<'py> for VersionsResponse { pub struct VersionsHandler { pub global_unstable_feature_map: Arc, pub store: Arc, - /// The Twisted reactor, used to bridge our `async` response back into a - /// Twisted deferred that Python can `await`. - pub reactor: Py, + /// The per-homeserver Rust state, used to bridge our `async` response + /// back into a Twisted deferred that Python can `await`. + pub runtime: RustRuntime, } #[pymethods] @@ -63,7 +64,7 @@ impl VersionsHandler { let store = Arc::clone(&self.store); let global_unstable_feature_map = Arc::clone(&self.global_unstable_feature_map); - create_deferred(py, self.reactor.bind(py), async move { + create_deferred(py, &self.runtime, async move { build_versions_response(&store, &global_unstable_feature_map, user_id.as_deref()) .await .map_err(|err| { @@ -155,6 +156,9 @@ async fn build_versions_response( "v1.10".to_string(), "v1.11".to_string(), "v1.12".to_string(), + "v1.13".to_string(), + "v1.14".to_string(), + "v1.15".to_string(), ]), unstable_features: unstable_feature_map, }) diff --git a/rust/src/homeserver.rs b/rust/src/homeserver.rs new file mode 100644 index 0000000000..44f8ba905e --- /dev/null +++ b/rust/src/homeserver.rs @@ -0,0 +1,102 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! A typed wrapper around the Python `HomeServer`. + +use pyo3::{intern, prelude::*, types::PyDict}; + +use crate::config::SynapseHomeServerConfig; +use crate::reactor::Reactor; +use crate::runtime::RustRuntime; + +/// The Python `HomeServer`, as seen from Rust. +/// +/// This is a wrapper around a foreign Python object. No validation is +/// performed, on the assumption the Python side is correctly typed. +pub struct HomeServer(Py); + +impl<'a, 'py> FromPyObject<'a, 'py> for HomeServer { + type Error = PyErr; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> PyResult { + Ok(HomeServer(obj.to_owned().unbind())) + } +} + +impl HomeServer { + /// Fetch the Twisted reactor in use by this HomeServer. + pub fn get_reactor(&self, py: Python<'_>) -> PyResult { + self.0 + .bind(py) + .call_method0(intern!(py, "get_reactor"))? + .extract() + } + + /// Register a system event trigger with the HomeServer so it can be cleanly + /// removed when the HomeServer is shutdown. + pub fn register_sync_shutdown_handler( + &self, + py: Python<'_>, + callable: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let kwargs = PyDict::new(py); + kwargs.set_item(intern!(py, "phase"), intern!(py, "after"))?; + kwargs.set_item(intern!(py, "eventType"), intern!(py, "shutdown"))?; + kwargs.set_item(intern!(py, "shutdown_func"), callable)?; + self.0.bind(py).call_method( + intern!(py, "register_sync_shutdown_handler"), + (), + Some(&kwargs), + )?; + + Ok(()) + } + + /// The per-homeserver Rust state (`hs.get_rust_runtime()`), which gives + /// access to the tokio runtime and the reactor. + pub fn get_rust_runtime(&self, py: Python<'_>) -> PyResult { + Ok(self + .0 + .bind(py) + .call_method0(intern!(py, "get_rust_runtime"))? + .cast::()? + .get() + .clone()) + } + + /// The Rust-side view of `hs.config`. + pub fn config(&self, py: Python<'_>) -> PyResult { + self.0.bind(py).getattr(intern!(py, "config"))?.extract() + } + + /// The Synapse `Clock` (`hs.get_clock()`). + // TODO: give the clock a typed wrapper of its own. + pub fn get_clock(&self, py: Python<'_>) -> PyResult> { + Ok(self + .0 + .bind(py) + .call_method0(intern!(py, "get_clock"))? + .unbind()) + } + + /// The main database pool (`hs.get_datastores().main.db_pool`). + pub fn main_database_pool<'py>(&self, py: Python<'py>) -> PyResult> { + self.0 + .bind(py) + .call_method0(intern!(py, "get_datastores"))? + .getattr(intern!(py, "main"))? + .getattr(intern!(py, "db_pool")) + } +} diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index aaa1066a67..802f93b2d0 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -21,7 +21,7 @@ use reqwest::RequestBuilder; use crate::deferred::create_deferred; use crate::errors::HttpResponseException; -use crate::tokio_runtime::runtime; +use crate::runtime::RustRuntime; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -42,21 +42,18 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> #[pyclass] struct HttpClient { client: reqwest::Client, - reactor: Py, + runtime: RustRuntime, } #[pymethods] impl HttpClient { #[new] - #[pyo3(signature = (reactor, user_agent, http2_only = false))] + #[pyo3(signature = (runtime, user_agent, http2_only = false))] pub fn py_new( - reactor: Bound, + runtime: &RustRuntime, user_agent: &str, http2_only: bool, ) -> PyResult { - // Make sure the runtime gets installed - let _ = runtime(&reactor)?; - let mut builder = reqwest::Client::builder().user_agent(user_agent); if http2_only { @@ -69,7 +66,7 @@ impl HttpClient { Ok(HttpClient { client, - reactor: reactor.unbind(), + runtime: runtime.clone(), }) } @@ -107,7 +104,7 @@ impl HttpClient { builder: RequestBuilder, response_limit: usize, ) -> PyResult> { - create_deferred(py, self.reactor.bind(py), async move { + create_deferred(py, &self.runtime, async move { let response = builder.send().await.context("sending request")?; let status = response.status(); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 28783afbba..af1162709c 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -12,18 +12,22 @@ pub mod duration; pub mod errors; pub mod events; pub mod handlers; +pub mod homeserver; pub mod http; pub mod http_client; pub mod identifier; pub mod json; +pub mod logging; pub mod matrix_const; pub mod msc4388_rendezvous; pub mod push; +pub mod reactor; pub mod rendezvous; pub mod room_versions; +pub mod runtime; pub mod segmenter; pub mod storage; -pub mod tokio_runtime; +pub mod twisted_dispatch; pub mod types; lazy_static! { @@ -70,12 +74,14 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?; acl::register_module(py, m)?; + logging::context::register_module(py, m)?; deferred::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; handlers::register_module(py, m)?; http_client::register_module(py, m)?; rendezvous::register_module(py, m)?; + runtime::register_module(py, m)?; msc4388_rendezvous::register_module(py, m)?; segmenter::register_module(py, m)?; room_versions::register_module(py, m)?; diff --git a/rust/src/logging/context.rs b/rust/src/logging/context.rs new file mode 100644 index 0000000000..d01223e7b7 --- /dev/null +++ b/rust/src/logging/context.rs @@ -0,0 +1,1143 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Rust storage for the Synapse "current logcontext". +//! +//! The storage lives in Rust rather than in a Python `threading.local` because +//! a Python thread-local is invisible to the Rust tokio threads. +//! +//! There are two pieces of storage, and [`current_context`] combines them so +//! that it answers correctly from both Python and Rust: +//! +//! 1. [`THREAD_LOCAL_CONTEXT`], one value per OS thread. Used by the Python +//! reactor thread and any reactor-managed threadpool threads. +//! 2. [`TASK_LOCAL_CONTEXT`], one value per tokio task, set when the task is +//! spawned. It stays with the task as the task moves between worker threads +//! across `.await` points. +//! +//! Both hold an `Option>`, where `None` means "no context", +//! i.e. the sentinel. The `_Sentinel` object itself stays pure Python +//! (`synapse.logging.context.SENTINEL_CONTEXT`): the wrappers there convert +//! between it and `None`, so Rust code never sees the sentinel object. +//! +//! [`current_context`] returns the task-local value when called from inside a +//! tokio task, and the thread-local value otherwise. `LoggingContextFilter` +//! (and therefore `pyo3-log`) calls [`current_context`] for each log record, so +//! records emitted while a task runs are attributed to that task's context +//! without having to call into Python. +//! +//! [`set_current_context`] also does the CPU accounting. It reads the thread +//! rusage via libc, calls `stop` on the old context and `start` on the new one, +//! and writes the new context with [`swap_current_context`]. It is only ever +//! called on reactor or threadpool threads, never on tokio worker threads, so +//! it always writes the thread-local. Therefore, CPU accounting is not captured +//! when executing purely Rust work. +//! +//! The task-local is written exactly once, at spawn time, by +//! [`LogContextHandle::scope`]. [`swap_current_context`] checks that invariant +//! rather than trusting it. + +use std::{cell::RefCell, future::Future}; + +use log::{debug, error, log_enabled, Level}; +use pyo3::call::PyCallArgs; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{intern, PyTraverseError, PyVisit}; + +/// Name of the opt-in logger for debugging when the logcontext switches. +/// +/// Used as the `debug!` `target:` for the messages emitted below, and exported +/// to Python via [`register_module`] so that `synapse.logging.context` builds +/// exactly this logger (see `logcontext_debug_logger` there). Sharing one +/// constant keeps the two names in sync. The messages only appear when this +/// logger is explicitly configured: `ExplicitlyConfiguredLogger` on the Python +/// side implements that, and pyo3-log honours its `isEnabledFor`, so the +/// opt-in applies to messages from Rust too. +pub const DEBUG_LOGGER_NAME: &str = "synapse.logging.context.debug"; + +thread_local! { + /// The current logcontext for this OS thread, a [`LoggingContext`] + /// (possibly a Python subclass instance), or `None` for the sentinel. + static THREAD_LOCAL_CONTEXT: RefCell>> = const { RefCell::new(None) }; +} + +tokio::task_local! { + /// The logcontext captured for the current tokio task, set by + /// [`LogContextHandle::scope`] when the task is spawned. Only present + /// inside such a task. Readable during any poll of the task, whichever + /// worker thread runs it. + static TASK_LOCAL_CONTEXT: LogContextHandle; +} + +/// A captured logcontext that can be cloned and read without the GIL. Holds a +/// [`LoggingContext`] (possibly a Python subclass instance), or `None` for the +/// sentinel. +#[derive(Clone)] +pub struct LogContextHandle { + context: std::sync::Arc>>, +} + +impl LogContextHandle { + /// Capture the calling thread's current logcontext. + pub fn capture(py: Python<'_>) -> Self { + LogContextHandle { + context: std::sync::Arc::new(current_context(py)), + } + } + + /// Create a handle to the logcontext of the current tokio task, if we are + /// running inside one that was spawned through [`LogContextHandle::scope`]. + pub fn task_current() -> Option { + TASK_LOCAL_CONTEXT.try_with(|c| c.clone()).ok() + } + + /// Run `fut` with this logcontext active for the duration of the task. + pub fn scope(self, fut: F) -> impl Future + where + F: Future, + { + TASK_LOCAL_CONTEXT.scope(self, fut) + } + + /// The captured [`LoggingContext`], or `None` if the sentinel was captured. + pub fn logging_context(&self) -> Option<&Py> { + self.context.as_ref().as_ref() + } +} + +/// Tracks the resources used by a log context. +/// +/// The attributes, operators and `repr` format are relied on by Python callers +/// (`Measure`, request and background-process metrics, the task scheduler, +/// etc). Implemented in Rust so that [`set_current_context`] can update the +/// counters without allocating a Python object each time. +#[pyclass(skip_from_py_object, get_all, set_all)] +#[derive(Clone, Default)] +pub struct ContextResourceUsage { + /// System CPU time, in seconds. + pub ru_stime: f64, + /// User CPU time, in seconds. + pub ru_utime: f64, + /// Number of database transactions done. + pub db_txn_count: i64, + /// Time spent doing database transactions (excluding scheduling), in seconds. + pub db_txn_duration_sec: f64, + /// Time spent waiting for a database connection, in seconds. + pub db_sched_duration_sec: f64, + /// Number of events requested from the database. + pub evt_db_fetch_count: i64, +} + +impl ContextResourceUsage { + /// Add the stats from `other` to `self`, mutating in place. Equivalent to + /// `self += other`. + fn add_assign(&mut self, other: &ContextResourceUsage) { + self.ru_utime += other.ru_utime; + self.ru_stime += other.ru_stime; + self.db_txn_count += other.db_txn_count; + self.db_txn_duration_sec += other.db_txn_duration_sec; + self.db_sched_duration_sec += other.db_sched_duration_sec; + self.evt_db_fetch_count += other.evt_db_fetch_count; + } + + /// Subtract the stats from `other` from `self`, mutating in place. + /// Equivalent to `self -= other`. + fn sub_assign(&mut self, other: &ContextResourceUsage) { + self.ru_utime -= other.ru_utime; + self.ru_stime -= other.ru_stime; + self.db_txn_count -= other.db_txn_count; + self.db_txn_duration_sec -= other.db_txn_duration_sec; + self.db_sched_duration_sec -= other.db_sched_duration_sec; + self.evt_db_fetch_count -= other.evt_db_fetch_count; + } +} + +#[pymethods] +impl ContextResourceUsage { + /// If `copy_from` is given, copy its stats; otherwise start at zero. + #[new] + #[pyo3(signature = (copy_from=None))] + fn new(copy_from: Option<&ContextResourceUsage>) -> Self { + copy_from.cloned().unwrap_or_default() + } + + /// Return a copy of this object. + fn copy(&self) -> ContextResourceUsage { + self.clone() + } + + /// Reset all stats to zero. + fn reset(&mut self) { + *self = ContextResourceUsage::default(); + } + + fn __repr__(&self) -> String { + // `Measure` logs this string in its "Failed to save metrics!" warning, + // and log scrapers may match on it, so keep the format stable. + format!( + "", + self.ru_stime, + self.ru_utime, + self.db_txn_count, + self.db_txn_duration_sec, + self.db_sched_duration_sec, + self.evt_db_fetch_count, + ) + } + + /// `self += other`, mutating in place. + fn __iadd__(&mut self, other: &ContextResourceUsage) { + self.add_assign(other); + } + + /// `self -= other`, mutating in place. + fn __isub__(&mut self, other: &ContextResourceUsage) { + self.sub_assign(other); + } + + /// `self + other`, returning a new object. + fn __add__(&self, other: &ContextResourceUsage) -> ContextResourceUsage { + let mut res = self.clone(); + res.add_assign(other); + res + } + + /// `self - other`, returning a new object. + fn __sub__(&self, other: &ContextResourceUsage) -> ContextResourceUsage { + let mut res = self.clone(); + res.sub_assign(other); + res + } +} + +/// Call `logcontext_error(msg)` in `synapse.logging.context`. +/// +/// Looked up at call time rather than cached, so tests can patch it. +fn logcontext_error(py: Python<'_>, msg: String) -> PyResult<()> { + let module = py.import("synapse.logging.context")?; + module.getattr("logcontext_error")?.call1((msg,))?; + Ok(()) +} + +extern "C" { + /// CPython's thread identifier (`pythread.h`) — the exact value + /// `threading.get_ident()` returns. Part of the stable ABI, but not bound + /// by pyo3-ffi. + fn PyThread_get_thread_ident() -> std::os::raw::c_ulong; +} + +/// This thread's `threading.get_ident()` value, read without calling into +/// Python. +/// +/// Note that `get_ident` is *not* an OS-level tid. On Linux it returns the same +/// value either side of a `fork()` call. Synapse forks in exactly one place, so +/// contexts created before the fork still pass the `main_thread` check after +/// it. +fn get_thread_id() -> u64 { + // SAFETY: no preconditions; returns an identifier for the calling thread. + (unsafe { PyThread_get_thread_ident() }) as u64 +} + +/// Propagate a usage update to the parent context, if there is a (truthy, i.e. +/// non-sentinel) one. +/// +/// Handles the parent being a [`LoggingContext`] subclass (a Python object), +/// but has a fast-path for the common case that the parent is a base +/// [`LoggingContext`]. +fn forward_to_parent<'py, N>( + parent: &Option>, + py: Python<'py>, + method: &str, + args: impl PyCallArgs<'py>, + native: N, +) -> PyResult<()> +where + N: FnOnce(&Bound<'py, LoggingContext>) -> PyResult<()>, +{ + if let Some(parent) = parent { + let parent = parent.bind(py); + + // Check if the parent is a base LoggingContext, and if so call the + // native Rust method rather than going through Python. + if let Ok(base) = parent.cast_exact::() { + return native(base); + } + + // Otherwise call the method on the parent, if it is truthy. This + // handles Python subclasses of `LoggingContext`. + if parent.is_truthy()? { + parent.call_method1(method, args)?; + } + } + Ok(()) +} + +/// The current thread's CPU usage as `(ru_utime, ru_stime)` in seconds, read +/// via `getrusage(RUSAGE_THREAD)`. +/// +/// Returns `None` where per-thread rusage isn't available, which we take to +/// be any non-Linux target: `RUSAGE_THREAD` is Linux-only, so e.g. macOS gets +/// no per-context CPU accounting. +#[cfg(target_os = "linux")] +fn get_thread_rusage() -> Option<(f64, f64)> { + fn timeval_to_secs(tv: libc::timeval) -> f64 { + tv.tv_sec as f64 + tv.tv_usec as f64 / 1_000_000.0 + } + + // SAFETY: `getrusage` only writes into `usage`, and we only read it once it + // reports success. + let mut usage = std::mem::MaybeUninit::::uninit(); + let ret = unsafe { libc::getrusage(libc::RUSAGE_THREAD, usage.as_mut_ptr()) }; + if ret != 0 { + return None; + } + let usage = unsafe { usage.assume_init() }; + Some(( + timeval_to_secs(usage.ru_utime), + timeval_to_secs(usage.ru_stime), + )) +} + +#[cfg(not(target_os = "linux"))] +fn get_thread_rusage() -> Option<(f64, f64)> { + None +} + +/// The `(user, system)` CPU seconds elapsed between `start` and `current`. +/// +/// Guards against the clock going backwards (clamping to zero and logging, as +/// the accounting must never go negative). +fn cputime_delta(current: (f64, f64), start: (f64, f64)) -> (f64, f64) { + let mut utime_delta = current.0 - start.0; + let mut stime_delta = current.1 - start.1; + + // sanity check + if utime_delta < 0.0 { + error!("utime went backwards! {} < {}", current.0, start.0); + utime_delta = 0.0; + } + if stime_delta < 0.0 { + error!("stime went backwards! {} < {}", current.1, start.1); + stime_delta = 0.0; + } + + (utime_delta, stime_delta) +} + +/// Additional context for log formatting, tracking which request a unit of work +/// belongs to and accounting CPU/DB usage against it. Contexts are scoped +/// within a `with` block. +/// +/// The attributes and methods are relied on by Python callers and subclasses +/// (notably `BackgroundProcessLoggingContext`). +/// +/// Construction is split between `__new__`, which allocates a blank instance, +/// and `__init__`, which does the real initialisation, to support subclasses. +#[pyclass(subclass)] +pub struct LoggingContext { + /// Name for the context, used in logging. Stored as a Python string. + /// `LoggingContextFilter` calls `str(context)` on every log record, and + /// returning the stored `Py` only bumps a reference count where a + /// `String` field would allocate a fresh `str` per record. + #[pyo3(get, set)] + name: Py, + /// The homeserver name this context is associated with. Stored as a Python + /// string for the same reason as `name` (read per log record). + #[pyo3(get, set)] + server_name: Py, + /// The `threading.get_ident()` value of the thread this context was + /// created on (see [`get_thread_id`] for why it is not a real OS tid). + /// Activity on any other thread is an error. Settable only so tests can + /// simulate activity on the wrong thread. + #[pyo3(get)] + main_thread: u64, + /// Whether `__exit__` has run. Re-activating a finished context is an error. + #[pyo3(get, set)] + finished: bool, + /// The thread CPU usage `(ru_utime, ru_stime)` in seconds captured when + /// this context became active, or `None` if it is not currently active. + /// Kept as a plain `(f64, f64)` rather than a Python `struct_rusage` so + /// that switching contexts allocates no Python object. Nothing outside + /// this module reads it, so it is not exposed to Python. + usage_start: Option<(f64, f64)>, + /// A short human-readable tag (e.g. the sync type). Initialised to `""` and + /// treated as a `str` by everything in-tree, but `Option` so that assigning + /// `None` (which the sentinel's `tag` reports, and which out-of-tree + /// callers may assign) is allowed. + #[pyo3(get, set)] + tag: Option, + /// The resources used by this context so far. + resource_usage: Py, + /// The context that was current when this one was created. Restored on + /// exit. + #[pyo3(get, set)] + previous_context: Option>, + /// The parent context, if any. + #[pyo3(get, set)] + parent_context: Option>, + /// The `ContextRequest` this work belongs to, if any. + #[pyo3(get, set)] + request: Option>, + /// The opentracing scope associated with this context, if any. + #[pyo3(get, set)] + scope: Option>, +} + +#[pymethods] +impl LoggingContext { + /// Allocate a blank context. The real initialisation happens in `__init__`, + /// the same as for a normal Python class. Extra positional/keyword + /// arguments are accepted and ignored. `type.__call__` feeds the same + /// arguments to both `__new__` and `__init__`, so a subclass constructor's + /// arguments must not be rejected here. + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + fn __new__( + py: Python<'_>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult { + Ok(LoggingContext { + name: intern!(py, "").clone().unbind(), + server_name: intern!(py, "").clone().unbind(), + main_thread: 0, + finished: false, + usage_start: None, + tag: Some(String::new()), + resource_usage: Py::new(py, ContextResourceUsage::default())?, + previous_context: None, + parent_context: None, + request: None, + scope: None, + }) + } + + #[pyo3(signature = (*, name, server_name, parent_context=None, request=None))] + fn __init__( + &mut self, + py: Python<'_>, + name: Bound<'_, PyString>, + server_name: Bound<'_, PyString>, + parent_context: Option>, + request: Option>, + ) -> PyResult<()> { + self.previous_context = current_context(py); + + // The resource-usage tracker was already allocated (zeroed) by `__new__`, + // which `type.__call__` runs immediately before this. + + self.usage_start = None; + + self.name = name.unbind(); + self.server_name = server_name.unbind(); + self.main_thread = get_thread_id(); + self.request = None; + self.tag = Some(String::new()); + self.scope = None; + + // keep track of whether we have hit the __exit__ block for this context + self.finished = false; + + // Inherit some fields from the parent context (read before we move it + // into `self`, so no borrow of `self.parent_context` is held). + if let Some(parent) = &parent_context { + let parent = parent.bind(py); + // which request this corresponds to + self.request = parent.getattr("request")?.extract()?; + // we also track the current scope + self.scope = parent.getattr("scope")?.extract()?; + } + + if let Some(request) = request { + // the request param overrides the request from the parent context + self.request = Some(request); + } + + self.parent_context = parent_context; + + Ok(()) + } + + /// Returns the stored name object itself, without copying the string. + /// This runs for every log record via `LoggingContextFilter`. + fn __str__(&self, py: Python<'_>) -> Py { + self.name.clone_ref(py) + } + + /// Enter this logging context, making it the current context. + fn __enter__<'py>(slf: Bound<'py, Self>) -> PyResult> { + let py = slf.py(); + // Clone the reference rather than holding a borrow across the call. + // `set_current_context` re-borrows `slf` (borrow_mut in `start_inner`). + let previous = slf + .borrow() + .previous_context + .as_ref() + .map(|p| p.clone_ref(py)); + + if log_enabled!(target: DEBUG_LOGGER_NAME, Level::Debug) { + // Only build the name string when the opt-in debug logger is + // enabled. This runs on every context entry. + debug!( + target: DEBUG_LOGGER_NAME, + "LoggingContext({}).__enter__", + slf.borrow().name_string(py) + ); + } + + let old_context = set_current_context(py, Some(slf.clone().unbind()))?; + + if !are_contexts_identical(&previous, &old_context) { + let previous_repr = context_repr(py, &previous)?; + let old_repr = context_repr(py, &old_context)?; + logcontext_error( + py, + format!("Expected previous context {previous_repr}, found {old_repr}"), + )?; + } + + Ok(slf) + } + + /// Restore the previous logging context. Returns `None` (does not suppress + /// exceptions). + fn __exit__( + slf: Bound<'_, Self>, + _exc_type: Bound<'_, PyAny>, + _exc_value: Bound<'_, PyAny>, + _traceback: Bound<'_, PyAny>, + ) -> PyResult<()> { + let py = slf.py(); + + let previous = slf + .borrow() + .previous_context + .as_ref() + .map(|p| p.clone_ref(py)); + + if log_enabled!(target: DEBUG_LOGGER_NAME, Level::Debug) { + // The `str()` of the previous context, or `"sentinel"` for + // `None`. Only built (along with the name) when the opt-in debug + // logger is enabled. This runs on every context exit. + let previous_str = match &previous { + Some(p) => p.bind(py).str()?.extract::()?, + None => "sentinel".to_owned(), + }; + debug!( + target: DEBUG_LOGGER_NAME, + "LoggingContext({}).__exit__ --> {previous_str}", + slf.borrow().name_string(py) + ); + } + + let current = set_current_context(py, previous)?; + + let restored_self = current.as_ref().is_some_and(|c| c.bind(py).is(&slf)); + if !restored_self { + // Error path: the name string is only built for the message. + let name = slf.borrow().name_string(py); + match ¤t { + None => logcontext_error(py, format!("Expected logging context {name} was lost"))?, + Some(current) => { + let current_str: String = current.bind(py).str()?.extract()?; + logcontext_error( + py, + format!("Expected logging context {name} but found {current_str}"), + )?; + } + } + } + + slf.borrow_mut().finished = true; + + Ok(()) + } + + /// Record that this logcontext is currently running. + /// + /// This should not be called directly from Python, use + /// `set_current_context`. + /// + /// `rusage` is the thread CPU usage `(ru_utime, ru_stime)` at the point of + /// switching to this context (`None` if the platform doesn't track it). + /// + /// Note that the this takes a `Bound<..>` rather than `&self` as it can + /// call into Python, which may re-enter the context and try mutable borrow + /// it. + fn start(slf: &Bound<'_, Self>, rusage: Option<(f64, f64)>) -> PyResult<()> { + let py = slf.py(); + let main_thread = slf.borrow().main_thread; + + if get_thread_id() != main_thread { + let name = slf.borrow().name_string(py); + logcontext_error(py, format!("Started logcontext {name} on different thread"))?; + return Ok(()); + } + + if slf.borrow().finished { + let name = slf.borrow().name_string(py); + logcontext_error(py, format!("Re-starting finished log context {name}"))?; + return Ok(()); + } + + if slf.borrow().usage_start.is_some() { + let name = slf.borrow().name_string(py); + logcontext_error(py, format!("Re-starting already-active log context {name}"))?; + return Ok(()); + } + + // If we haven't already started, record the thread resource usage so far. + slf.borrow_mut().usage_start = rusage; + + Ok(()) + } + + /// Record that this logcontext is no longer running. + /// + /// This should not be called directly from Python: use + /// `set_current_context`. + /// + /// Note that the this takes a `Bound<..>` rather than `&self` as it can + /// call into Python, which may re-enter the context and try mutable borrow + /// it. + fn stop(slf: &Bound<'_, Self>, rusage: Option<(f64, f64)>) -> PyResult<()> { + // `usage_start` must be cleared even if `stop_inner` fails, so don't + // use `?` here. + let result = Self::stop_inner(slf, rusage); + + slf.borrow_mut().usage_start = None; + + result + } + + /// Get a *copy* of the resources used by this logcontext so far. + /// + /// Should only be called from the main thread (it will return None otherwise). + fn get_resource_usage(slf: Bound<'_, Self>) -> PyResult { + let py = slf.py(); + + let mut res = slf.borrow().resource_usage.borrow(py).clone(); + + let (usage_start, main_thread) = { + let this = slf.borrow(); + (this.usage_start, this.main_thread) + }; + + // If we are on the correct thread and we're currently running then we can + // include resource usage so far. + if let Some(start) = usage_start { + if get_thread_id() == main_thread { + if let Some(current) = get_thread_rusage() { + let (utime_delta, stime_delta) = cputime_delta(current, start); + res.ru_utime += utime_delta; + res.ru_stime += stime_delta; + } + } + } + + Ok(res) + } + + /// Update the CPU time usage of this context (and any parents, recursively). + fn add_cputime(&self, py: Python<'_>, utime_delta: f64, stime_delta: f64) -> PyResult<()> { + { + let mut usage = self.resource_usage.borrow_mut(py); + usage.ru_utime += utime_delta; + usage.ru_stime += stime_delta; + } + forward_to_parent( + &self.parent_context, + py, + "add_cputime", + (utime_delta, stime_delta), + |p| p.borrow().add_cputime(py, utime_delta, stime_delta), + ) + } + + /// Record the use of a database transaction and how long it took. + fn add_database_transaction(&self, py: Python<'_>, duration_sec: f64) -> PyResult<()> { + if duration_sec < 0.0 { + return Err(PyValueError::new_err( + "DB txn time can only be zero or non-negative", + )); + } + { + let mut usage = self.resource_usage.borrow_mut(py); + usage.db_txn_count += 1; + usage.db_txn_duration_sec += duration_sec; + } + forward_to_parent( + &self.parent_context, + py, + "add_database_transaction", + (duration_sec,), + |p| p.borrow().add_database_transaction(py, duration_sec), + ) + } + + /// Record a use of the database pool (the time taken to get a connection). + fn add_database_scheduled(&self, py: Python<'_>, sched_sec: f64) -> PyResult<()> { + if sched_sec < 0.0 { + return Err(PyValueError::new_err( + "DB scheduling time can only be zero or non-negative", + )); + } + { + let mut usage = self.resource_usage.borrow_mut(py); + usage.db_sched_duration_sec += sched_sec; + } + forward_to_parent( + &self.parent_context, + py, + "add_database_scheduled", + (sched_sec,), + |p| p.borrow().add_database_scheduled(py, sched_sec), + ) + } + + /// Record a number of events being fetched from the db. + fn record_event_fetch(&self, py: Python<'_>, event_count: i64) -> PyResult<()> { + { + let mut usage = self.resource_usage.borrow_mut(py); + usage.evt_db_fetch_count += event_count; + } + forward_to_parent( + &self.parent_context, + py, + "record_event_fetch", + (event_count,), + |p| p.borrow().record_event_fetch(py, event_count), + ) + } + + /// Traverse referenced Python objects for the cyclic garbage collector. + /// `scope` and the context can reference each other, forming a real cycle. + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + if let Some(previous_context) = &self.previous_context { + visit.call(previous_context)?; + } + if let Some(parent_context) = &self.parent_context { + visit.call(parent_context)?; + } + if let Some(request) = &self.request { + visit.call(request)?; + } + if let Some(scope) = &self.scope { + visit.call(scope)?; + } + Ok(()) + } + + fn __clear__(&mut self) { + self.previous_context = None; + self.parent_context = None; + self.request = None; + self.scope = None; + } +} + +impl LoggingContext { + /// The context name as an owned Rust string. + /// + /// This copies the string data, so it is for error/debug paths only. + /// Switching contexts should not allocate in the common case. + fn name_string(&self, py: Python<'_>) -> String { + self.name.bind(py).to_string_lossy().into_owned() + } + + /// Inner implementation of [`Self::stop`], which does not clear + /// `usage_start`. + /// + /// Note that the this takes a `Bound<..>` rather than `&self` as it can + /// call into Python, which may re-enter the context and try mutable borrow + /// it. + fn stop_inner(slf: &Bound<'_, Self>, rusage: Option<(f64, f64)>) -> PyResult<()> { + let py = slf.py(); + let main_thread = slf.borrow().main_thread; + + if get_thread_id() != main_thread { + let name = slf.borrow().name_string(py); + logcontext_error(py, format!("Stopped logcontext {name} on different thread"))?; + return Ok(()); + } + + // No rusage means this platform doesn't track per-thread CPU, so + // there is nothing to account. + let Some(current) = rusage else { + return Ok(()); + }; + + // Record the cpu used since we started. + let Some(start) = slf.borrow().usage_start else { + let name = slf.borrow().name_string(py); + logcontext_error( + py, + format!("Called stop on logcontext {name} without recording a start rusage"), + )?; + return Ok(()); + }; + + let (utime_delta, stime_delta) = cputime_delta(current, start); + slf.borrow().add_cputime(py, utime_delta, stime_delta)?; + Ok(()) + } +} + +/// Which way a [`switch_context`] dispatch goes. +#[derive(Clone, Copy)] +enum SwitchDirection { + Start, + Stop, +} + +/// Call `stop` or `start` on a context as the current context changes, +/// respecting subclass overrides. +/// +/// `None` (the sentinel) is a no-op. +/// +/// Handles the context being a Python subclass of [`LoggingContext`], but has +/// a fast-path for the common case that it is a base [`LoggingContext`]. +fn switch_context( + py: Python<'_>, + ctx: Option<&Py>, + direction: SwitchDirection, + rusage: Option<(f64, f64)>, +) -> PyResult<()> { + let Some(ctx) = ctx else { + return Ok(()); + }; + let ctx = ctx.bind(py); + if ctx.as_any().is_exact_instance_of::() { + match direction { + SwitchDirection::Start => LoggingContext::start(ctx, rusage), + SwitchDirection::Stop => LoggingContext::stop(ctx, rusage), + } + } else { + let method = match direction { + SwitchDirection::Start => intern!(py, "start"), + SwitchDirection::Stop => intern!(py, "stop"), + }; + ctx.call_method1(method, (rusage,))?; + Ok(()) + } +} + +/// Whether two stored values are the same context, or both the sentinel. +/// `LoggingContext` has no `__eq__`, so contexts compare by identity. +fn are_contexts_identical(a: &Option>, b: &Option>) -> bool { + match (a, b) { + (None, None) => true, + (Some(a), Some(b)) => a.is(b), + _ => false, + } +} + +/// `repr()` of a stored context, for error messages only. `None` renders as +/// `"None"`. +fn context_repr(py: Python<'_>, slot: &Option>) -> PyResult { + match slot { + Some(ctx) => Ok(ctx.bind(py).repr()?.extract()?), + None => Ok("None".to_owned()), + } +} + +/// Set the current logging context, returning the context that was previously +/// current. `None` means the sentinel, in both directions. +/// +/// `context` must be a [`LoggingContext`] (or subclass) or `None`. +/// +/// This is not the Python-facing API. +/// `synapse.logging.context.set_current_context` wraps this, converting between +/// `SENTINEL_CONTEXT` and `None`. +/// +/// Reads the thread rusage once via libc, calls `stop` on the old context and +/// `start` on the new one. +#[pyfunction] +#[pyo3(signature = (context))] +pub fn set_current_context( + py: Python<'_>, + context: Option>, +) -> PyResult>> { + let current = current_context(py); + + if !are_contexts_identical(¤t, &context) { + let rusage = get_thread_rusage(); + switch_context(py, current.as_ref(), SwitchDirection::Stop, rusage)?; + // We already hold `current`, so ignore the previous value the swap + // returns. The clone_ref keeps a reference for the `start` below. + let new_ref = context.as_ref().map(|ctx| ctx.clone_ref(py)); + swap_current_context(context); + switch_context(py, new_ref.as_ref(), SwitchDirection::Start, rusage)?; + } + + Ok(current) +} + +/// Run `f` with `context` as the current logcontext (`None` meaning the +/// sentinel), restoring the previously-current context afterwards. This is the +/// Rust equivalent of Python's `with PreserveLoggingContext(context):`. +/// +/// The restore runs whether or not `f` fails. If both `f` and the restore fail, +/// `f`'s error is reported. +pub(crate) fn with_logcontext( + py: Python<'_>, + context: Option>, + f: impl FnOnce() -> PyResult, +) -> PyResult { + let previous = set_current_context(py, context)?; + let result = f(); + // Run before checking success/failure of `f()`. + let restored = set_current_context(py, previous); + + let value = result?; + restored?; + Ok(value) +} + +/// Get the current logging context, or `None` for the sentinel. +/// +/// Returns the tokio task-local value when called from inside a task that has +/// one (even if what it captured was `None`/the sentinel) and this OS thread's +/// value otherwise. Logging emitted while a task runs is therefore attributed +/// to the context that was current when the task was spawned. +/// +/// This is not the Python-facing API. +/// `synapse.logging.context.current_context` wraps this and returns +/// `SENTINEL_CONTEXT` instead of `None`. +#[pyfunction] +pub fn current_context(py: Python<'_>) -> Option> { + if let Some(handle) = LogContextHandle::task_current() { + return handle.logging_context().map(|ctx| ctx.clone_ref(py)); + } + + THREAD_LOCAL_CONTEXT.with(|slot| slot.borrow().as_ref().map(|ctx| ctx.clone_ref(py))) +} + +/// Replace this OS thread's current logging context, returning the previous one +/// (`None` is the sentinel, in both directions). +/// +/// This is the raw write only. It does no resource-usage accounting and no +/// thread checks; [`set_current_context`] wraps it with the `getrusage` +/// `stop`/`start` bookkeeping. +/// +/// It never touches the tokio task-local. The current context is only ever +/// changed on reactor/threadpool threads, while the task-local is written +/// exactly once, at spawn time, by [`LogContextHandle::scope`]. If this is +/// nonetheless called while a task-local is in scope, it logs an error, leaves +/// the thread-local unchanged and returns `None`. +fn swap_current_context(context: Option>) -> Option> { + // A task-local in scope would shadow the write (`current_context` prefers + // it), so refuse rather than misattribute later reads. + if TASK_LOCAL_CONTEXT.try_with(|_| ()).is_ok() { + error!( + "swap_current_context called during a tokio-scoped poll; the switch is \ + invisible to current_context() and will misattribute logs and metrics" + ); + return None; + } + + THREAD_LOCAL_CONTEXT.with(|slot| std::mem::replace(&mut *slot.borrow_mut(), context)) +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module: Bound<'_, PyModule> = PyModule::new(py, "logcontext")?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_function(wrap_pyfunction!(current_context, &child_module)?)?; + child_module.add_function(wrap_pyfunction!(set_current_context, &child_module)?)?; + child_module.add("DEBUG_LOGGER_NAME", DEBUG_LOGGER_NAME)?; + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import logcontext` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.logcontext", child_module)?; + + Ok(()) +} + +/// Helpers for Rust unit tests elsewhere in the crate that need a logcontext. +#[cfg(test)] +pub(crate) mod testing { + use pyo3::types::PyString; + + use super::*; + + /// A minimal `LoggingContext` for these tests, built directly (bypassing + /// `__init__`, which would capture the current context and thread id). + pub(crate) fn test_context(py: Python<'_>, name: &str) -> Py { + Py::new( + py, + LoggingContext { + name: PyString::new(py, name).unbind(), + server_name: PyString::new(py, "test_server").unbind(), + main_thread: get_thread_id(), + finished: false, + usage_start: None, + tag: Some(String::new()), + resource_usage: Py::new(py, ContextResourceUsage::default()) + .expect("failed to allocate ContextResourceUsage"), + previous_context: None, + parent_context: None, + request: None, + scope: None, + }, + ) + .expect("failed to allocate LoggingContext") + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::testing::test_context; + use super::*; + + #[test] + fn thread_local_defaults_to_sentinel() { + Python::initialize(); + Python::attach(|py| { + // Nothing has been set on this fresh test thread, so the current + // context is `None`: the sentinel. + assert!(current_context(py).is_none()); + }); + } + + #[test] + fn swap_returns_previous_and_updates_thread_local() { + Python::initialize(); + Python::attach(|py| { + let a = test_context(py, "A"); + let b = test_context(py, "B"); + + // Swapping in A returns the previous value (`None` / the sentinel) + // and makes A current. + let prev = swap_current_context(Some(a.clone_ref(py))); + assert!(prev.is_none()); + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(a.bind(py))); + + // Swapping in B returns A. + let prev = swap_current_context(Some(b.clone_ref(py))); + assert!(prev + .expect("expected previous context") + .bind(py) + .is(a.bind(py))); + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(b.bind(py))); + + // Reset to the sentinel so we don't leak into another test that + // reuses this OS thread from the test harness's pool. + swap_current_context(None); + }); + } + + #[test] + fn task_local_takes_precedence_over_thread_local() { + Python::initialize(); + Python::attach(|py| { + let task_ctx = test_context(py, "TASKCTX"); + + // Outside any task, `current_context` reads the thread-local, which + // here is `None` / the sentinel. + assert!(current_context(py).is_none()); + assert!(LogContextHandle::task_current().is_none()); + + let log_context = LogContextHandle { + context: Arc::new(Some(task_ctx.clone_ref(py))), + }; + + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(log_context.scope(async { + // Inside the scope, both the Rust handle and the + // `current_context` (what the log filter calls) return the + // task-local context, even though the thread-local is still + // unset. + assert!(LogContextHandle::task_current().is_some()); + Python::attach(|py| { + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(task_ctx.bind(py))); + }); + })); + + // Once the scope ends, we fall back to the thread-local again. + assert!(LogContextHandle::task_current().is_none()); + assert!(current_context(py).is_none()); + }); + } + + #[test] + fn swap_during_scoped_poll_leaves_thread_local_untouched() { + Python::initialize(); + Python::attach(|py| { + let thread_ctx = test_context(py, "THREAD"); + let task_ctx = test_context(py, "TASK"); + let stray = test_context(py, "STRAY"); + + // Give this thread a current context. + swap_current_context(Some(thread_ctx.clone_ref(py))); + + let log_context = LogContextHandle { + context: Arc::new(Some(task_ctx.clone_ref(py))), + }; + + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(log_context.scope(async { + Python::attach(|py| { + // Swapping while a task-local is in scope is refused: it + // returns `None` and leaves the thread-local unchanged. + assert!(swap_current_context(Some(stray.clone_ref(py))).is_none()); + + // The task-local still governs reads inside the scope. + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(task_ctx.bind(py))); + }); + })); + + // The scope has ended, the thread-local still holds the original + // context, not the stray value from the invalid swap. + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(thread_ctx.bind(py))); + + // Reset to the sentinel so we don't leak into another test that + // reuses this OS thread from the test harness's pool. + swap_current_context(None); + }); + } +} diff --git a/rust/src/logging/mod.rs b/rust/src/logging/mod.rs new file mode 100644 index 0000000000..1d6d898cb4 --- /dev/null +++ b/rust/src/logging/mod.rs @@ -0,0 +1,23 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Rust counterparts to the `synapse.logging` Python package. +//! +//! Submodules live at the Rust module path that mirrors their Python logging +//! namespace (e.g. `synapse::logging::context` -> `synapse.logging.context`), so +//! that log records emitted from here via the `log` crate land under the matching +//! Python logger without an explicit `target:`. + +pub mod context; diff --git a/rust/src/msc4388_rendezvous/mod.rs b/rust/src/msc4388_rendezvous/mod.rs index 2f1b004ec1..ffd4f240d9 100644 --- a/rust/src/msc4388_rendezvous/mod.rs +++ b/rust/src/msc4388_rendezvous/mod.rs @@ -21,7 +21,7 @@ use http::StatusCode; use pyo3::{ pyclass, pymethods, types::{PyAnyMethods, PyModule, PyModuleMethods}, - Bound, IntoPyObject, Py, PyAny, PyResult, Python, + Bound, Py, PyAny, PyResult, Python, }; use serde::Deserialize; use ulid::Ulid; @@ -30,9 +30,9 @@ use self::session::Session; use crate::{ duration::SynapseDuration, errors::{NotFoundError, SynapseError}, + homeserver::HomeServer, http::http_request_from_twisted, msc4388_rendezvous::session::{GetResponse, PostResponse, PutResponse}, - UnwrapInfallible, }; mod session; @@ -92,25 +92,21 @@ impl MSC4388RendezvousHandler { #[pyo3(signature = (homeserver, /, soft_limit=100, hard_limit=200,max_content_length=4*1024, eviction_interval=60*1000, ttl=2*60*1000))] fn new( py: Python<'_>, - homeserver: &Bound<'_, PyAny>, + homeserver: HomeServer, soft_limit: usize, hard_limit: usize, max_content_length: u64, eviction_interval: u64, ttl: u64, ) -> PyResult> { - let clock = homeserver - .call_method0("get_clock")? - .into_pyobject(py) - .unwrap_infallible() - .unbind(); + let clock = homeserver.get_clock(py)?; // Construct a Python object so that we can get a reference to the // evict method and schedule it to run. let self_ = Py::new( py, Self { - clock, + clock: clock.clone_ref(py), sessions: BTreeMap::new(), soft_limit, hard_limit, @@ -122,11 +118,9 @@ impl MSC4388RendezvousHandler { let eviction_duration = SynapseDuration::from_milliseconds(eviction_interval); let evict = self_.getattr(py, "_evict")?; - homeserver.call_method0("get_clock")?.call_method( - "looping_call", - (evict, &eviction_duration), - None, - )?; + clock + .bind(py) + .call_method("looping_call", (evict, &eviction_duration), None)?; Ok(self_) } diff --git a/rust/src/push/base_rules.rs b/rust/src/push/base_rules.rs index 47d5289006..af7e6c83fb 100644 --- a/rust/src/push/base_rules.rs +++ b/rust/src/push/base_rules.rs @@ -145,6 +145,45 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[ default: true, default_enabled: true, }, + PushRule { + rule_id: Cow::Borrowed("global/override/.org.matrix.msc4075.rule.rtc.invite_for_me"), + priority_class: 5, + conditions: Cow::Borrowed(&[ + Condition::Known(KnownCondition::EventMatch(EventMatchCondition { + key: Cow::Borrowed("type"), + pattern: Cow::Borrowed("org.matrix.msc4075.rtc.notification"), + })), + Condition::Known(KnownCondition::ExactEventPropertyContainsType( + EventPropertyIsTypeCondition { + key: Cow::Borrowed(r"content.m\.mentions.user_ids"), + value_type: Cow::Borrowed(&EventMatchPatternType::UserId), + }, + )), + ]), + actions: Cow::Borrowed(&[Action::Notify, RING_ACTION]), + default: true, + default_enabled: true, + }, + PushRule { + rule_id: Cow::Borrowed("global/override/.org.matrix.msc4075.rule.rtc.invite_for_room"), + priority_class: 5, + conditions: Cow::Borrowed(&[ + Condition::Known(KnownCondition::EventMatch(EventMatchCondition { + key: Cow::Borrowed("type"), + pattern: Cow::Borrowed("org.matrix.msc4075.rtc.notification"), + })), + Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition { + key: Cow::Borrowed(r"content.m\.mentions.room"), + value: Cow::Owned(SimpleJsonValue::Bool(true)), + })), + Condition::Known(KnownCondition::SenderNotificationPermission { + key: Cow::Borrowed("room"), + }), + ]), + actions: Cow::Borrowed(&[Action::Notify, RING_ACTION]), + default: true, + default_enabled: true, + }, PushRule { rule_id: Cow::Borrowed("global/override/.m.rule.is_user_mention"), priority_class: 5, @@ -734,3 +773,135 @@ lazy_static! { .map(|rule| { (&*rule.rule_id, rule) }) .collect(); } + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + use std::collections::BTreeMap; + + use super::RING_ACTION; + use crate::push::evaluator::PushRuleEvaluator; + use crate::push::{Action, FilteredPushRules, JsonValue, PushRules, SimpleJsonValue}; + + const RTC_NOTIFICATION_TYPE: &str = "org.matrix.msc4075.rtc.notification"; + const ALICE: &str = "@alice:example.org"; + const BOB: &str = "@bob:example.org"; + + /// The default push rules with no user-defined rules. + /// + /// The generic mention rules are disabled so that they cannot mask the + /// MSC4075 rules, which have identical actions and come immediately + /// before them in the override list. + fn push_rules(msc4075_enabled: bool) -> FilteredPushRules { + let mut enabled_map = BTreeMap::new(); + enabled_map.insert("global/override/.m.rule.is_user_mention".to_string(), false); + enabled_map.insert("global/override/.m.rule.is_room_mention".to_string(), false); + + FilteredPushRules::py_new( + PushRules::new(vec![]), + enabled_map, + false, // msc1767_enabled + false, // msc3381_polls_enabled + false, // msc3664_enabled + false, // msc4028_push_encrypted_events + msc4075_enabled, + false, // msc4210_enabled + false, // msc4306_enabled + ) + } + + /// Builds an evaluator for an event of the given type whose `m.mentions` + /// lists the given user IDs and optionally mentions the room, sent by a + /// user with the given power level. + fn build_evaluator( + event_type: &'static str, + mentioned_user_ids: &[&'static str], + mentions_room: bool, + sender_power_level: i64, + ) -> PushRuleEvaluator { + let mut flattened_keys = BTreeMap::new(); + flattened_keys.insert( + "type".to_string(), + JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed(event_type))), + ); + flattened_keys.insert( + r"content.m\.mentions.user_ids".to_string(), + JsonValue::Array( + mentioned_user_ids + .iter() + .map(|&user_id| SimpleJsonValue::Str(Cow::Borrowed(user_id))) + .collect(), + ), + ); + if mentions_room { + flattened_keys.insert( + r"content.m\.mentions.room".to_string(), + JsonValue::Value(SimpleJsonValue::Bool(true)), + ); + } + + PushRuleEvaluator::py_new( + flattened_keys, + true, + 10, + Some(sender_power_level), + BTreeMap::new(), + BTreeMap::new(), + false, + vec![], + false, + false, + false, + ) + .unwrap() + } + + #[test] + fn test_rtc_invite_for_me_notifies_mentioned_user() { + let evaluator = build_evaluator(RTC_NOTIFICATION_TYPE, &[ALICE], false, 0); + + let actions = evaluator.run(&push_rules(true), Some(ALICE), None, None); + assert_eq!(actions, vec![Action::Notify, RING_ACTION]); + } + + #[test] + fn test_rtc_invite_for_me_ignores_other_users() { + let evaluator = build_evaluator(RTC_NOTIFICATION_TYPE, &[BOB], false, 0); + + let actions = evaluator.run(&push_rules(true), Some(ALICE), None, None); + assert_eq!(actions, vec![]); + } + + #[test] + fn test_rtc_invite_for_room_notifies_with_permission() { + // The default `notifications.room` power level is 50. + let evaluator = build_evaluator(RTC_NOTIFICATION_TYPE, &[], true, 50); + + let actions = evaluator.run(&push_rules(true), Some(ALICE), None, None); + assert_eq!(actions, vec![Action::Notify, RING_ACTION]); + } + + #[test] + fn test_rtc_invite_for_room_requires_permission() { + let evaluator = build_evaluator(RTC_NOTIFICATION_TYPE, &[], true, 0); + + let actions = evaluator.run(&push_rules(true), Some(ALICE), None, None); + assert_eq!(actions, vec![]); + } + + #[test] + fn test_rtc_rules_ignore_other_event_types() { + let evaluator = build_evaluator("org.example.other", &[ALICE], true, 50); + + let actions = evaluator.run(&push_rules(true), Some(ALICE), None, None); + assert_eq!(actions, vec![]); + } + + #[test] + fn test_rtc_rules_are_omitted_when_msc4075_is_disabled() { + let evaluator = build_evaluator(RTC_NOTIFICATION_TYPE, &[ALICE], true, 50); + + let actions = evaluator.run(&push_rules(false), Some(ALICE), None, None); + assert_eq!(actions, vec![]); + } +} diff --git a/rust/src/push/evaluator.rs b/rust/src/push/evaluator.rs index 1cbca4c635..250d2890c0 100644 --- a/rust/src/push/evaluator.rs +++ b/rust/src/push/evaluator.rs @@ -637,6 +637,7 @@ fn test_requires_room_version_supports_condition() { false, false, false, + false, ), None, None, diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs index 780d7a8cbd..868a57d00f 100644 --- a/rust/src/push/mod.rs +++ b/rust/src/push/mod.rs @@ -557,6 +557,7 @@ pub struct FilteredPushRules { msc3381_polls_enabled: bool, msc3664_enabled: bool, msc4028_push_encrypted_events: bool, + msc4075_enabled: bool, msc4210_enabled: bool, msc4306_enabled: bool, } @@ -572,6 +573,7 @@ impl FilteredPushRules { msc3381_polls_enabled: bool, msc3664_enabled: bool, msc4028_push_encrypted_events: bool, + msc4075_enabled: bool, msc4210_enabled: bool, msc4306_enabled: bool, ) -> Self { @@ -582,6 +584,7 @@ impl FilteredPushRules { msc3381_polls_enabled, msc3664_enabled, msc4028_push_encrypted_events, + msc4075_enabled, msc4210_enabled, msc4306_enabled, } @@ -626,6 +629,15 @@ impl FilteredPushRules { return false; } + if !self.msc4075_enabled + && (rule.rule_id + == "global/override/.org.matrix.msc4075.rule.rtc.invite_for_me" + || rule.rule_id + == "global/override/.org.matrix.msc4075.rule.rtc.invite_for_room") + { + return false; + } + if self.msc4210_enabled && (rule.rule_id == "global/override/.m.rule.contains_display_name" || rule.rule_id == "global/content/.m.rule.contains_user_name" diff --git a/rust/src/reactor.rs b/rust/src/reactor.rs new file mode 100644 index 0000000000..c17601c8b3 --- /dev/null +++ b/rust/src/reactor.rs @@ -0,0 +1,62 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! A typed wrapper around the Twisted reactor. + +use pyo3::{intern, prelude::*}; + +/// The Twisted reactor, as seen from Rust. +/// +/// This is a wrapper around a foreign Python object. No validation is +/// performed, on the assumption the Python side is correctly typed. +pub struct Reactor(Py); + +impl<'a, 'py> FromPyObject<'a, 'py> for Reactor { + type Error = PyErr; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> PyResult { + Ok(Reactor(obj.to_owned().unbind())) + } +} + +impl Reactor { + /// `reactor.addReader(reader)`: have the reactor poll `reader.fileno()` + /// and call `reader.doRead()` when it is readable. + /// + /// Must be called on the reactor thread. + pub fn add_reader(&self, py: Python<'_>, reader: &Bound<'_, PyAny>) -> PyResult<()> { + self.0 + .bind(py) + .call_method1(intern!(py, "addReader"), (reader,))?; + + Ok(()) + } + + /// `reactor.removeReader(reader)`: stop polling a reader added with + /// [`Reactor::add_reader`]. A no-op if it was never added. + /// + /// Must be called on the reactor thread. + pub fn remove_reader(&self, py: Python<'_>, reader: &Bound<'_, PyAny>) -> PyResult<()> { + self.0 + .bind(py) + .call_method1(intern!(py, "removeReader"), (reader,))?; + + Ok(()) + } + + pub fn clone_ref(&self, py: Python<'_>) -> Reactor { + Reactor(self.0.clone_ref(py)) + } +} diff --git a/rust/src/rendezvous/mod.rs b/rust/src/rendezvous/mod.rs index 9a6da9fcc3..64a5123f31 100644 --- a/rust/src/rendezvous/mod.rs +++ b/rust/src/rendezvous/mod.rs @@ -29,7 +29,7 @@ use pyo3::{ exceptions::PyValueError, pyclass, pymethods, types::{PyAnyMethods, PyModule, PyModuleMethods}, - Bound, IntoPyObject, Py, PyAny, PyResult, Python, + Bound, Py, PyAny, PyResult, Python, }; use ulid::Ulid; @@ -37,8 +37,8 @@ use self::session::Session; use crate::{ duration::SynapseDuration, errors::{NotFoundError, SynapseError}, + homeserver::HomeServer, http::{http_request_from_twisted, http_response_to_twisted, HeaderMapPyExt}, - UnwrapInfallible, }; mod session; @@ -113,25 +113,17 @@ impl RendezvousHandler { #[pyo3(signature = (homeserver, /, capacity=100, max_content_length=4*1024, eviction_interval=60*1000, ttl=60*1000))] fn new( py: Python<'_>, - homeserver: &Bound<'_, PyAny>, + homeserver: HomeServer, capacity: usize, max_content_length: u64, eviction_interval: u64, ttl: u64, ) -> PyResult> { - let base: String = homeserver - .getattr("config")? - .getattr("server")? - .getattr("public_baseurl")? - .extract()?; + let base = homeserver.config(py)?.server.public_baseurl; let base = Uri::try_from(format!("{base}_synapse/client/rendezvous")) .map_err(|_| PyValueError::new_err("Invalid base URI"))?; - let clock = homeserver - .call_method0("get_clock")? - .into_pyobject(py) - .unwrap_infallible() - .unbind(); + let clock = homeserver.get_clock(py)?; let eviction_duration = SynapseDuration::from_milliseconds(eviction_interval); @@ -141,7 +133,7 @@ impl RendezvousHandler { py, Self { base, - clock, + clock: clock.clone_ref(py), sessions: BTreeMap::new(), capacity, max_content_length, @@ -150,11 +142,9 @@ impl RendezvousHandler { )?; let evict = self_.getattr(py, "_evict")?; - homeserver.call_method0("get_clock")?.call_method( - "looping_call", - (evict, &eviction_duration), - None, - )?; + clock + .bind(py) + .call_method("looping_call", (evict, &eviction_duration), None)?; Ok(self_) } diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index 2860bfdbd7..1f8570a03e 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -161,6 +161,26 @@ pub struct RoomVersion { /// This is similar to how doubly-linked lists can potentially not refer to previous items correctly /// without verifying the list's integrity, but doing it on every insert is too expensive. pub msc4242_state_dags: bool, + /// Determines whether a room version *SHOULD* rather than *MAY* reject invites/knocks + /// with invalid stripped state events. + /// + /// According to Matrix v1.18 (introduced in MSC4311) for the `PUT + /// /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + /// > The `invite_room_state` has additional validation, which servers MAY apply to + /// > room versions 1 through 11 and SHOULD apply to all other room versions. As with + /// > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + /// > - The `m.room.create` event is missing from `invite_room_state`. + /// > - One or more entries in `invite_room_state` are not formatted according to the room's version. + /// > - One or more events fails a signature check. + /// > - One or more events does not reside in the same room as the invite. + /// + /// Regardless of room version (we should always do these things): + /// 1. The `m.room.create` event *MUST* be included in + /// `invite_room_state`/`knock_room_state` when sending invites/knocks over the + /// federation API's. + /// 2. Use full PDU's in the `invite_room_state`/`knock_room_state` in the federation + /// API. The client API still uses stripped state. + msc4311_stripped_state: bool, } impl RoomVersion { @@ -187,6 +207,7 @@ impl RoomVersion { msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, msc4242_state_dags: false, + msc4311_stripped_state: false, }; pub const V2: RoomVersion = RoomVersion { @@ -304,6 +325,7 @@ impl RoomVersion { state_res: StateResolutionVersions::V2_1, msc4289_creator_power_enabled: true, msc4291_room_ids_as_hashes: true, + msc4311_stripped_state: true, ..Self::V11 }; diff --git a/rust/src/runtime.rs b/rust/src/runtime.rs new file mode 100644 index 0000000000..e4b84bf22b --- /dev/null +++ b/rust/src/runtime.rs @@ -0,0 +1,256 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! The per-homeserver state for the Rust side of Synapse. +//! +//! A [`RustRuntime`] is created once per homeserver (`hs.get_rust_runtime()`) +//! and holds everything the Rust side keeps for the lifetime of that +//! homeserver. Rust consumers (e.g. the HTTP client) clone the inner +//! [`Arc`] at construction time and don't need the GIL (or +//! the Python-facing object) to reach it afterwards. +//! +//! The tokio runtime is shut down with the homeserver, via a handler +//! registered with `HomeServer.register_sync_shutdown_handler`. + +use std::ops::Deref; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Duration; + +use anyhow::Context; +use pyo3::{exceptions::PyRuntimeError, prelude::*}; +use tokio::runtime::{Handle, Runtime}; + +use crate::homeserver::HomeServer; +use crate::reactor::Reactor; +use crate::twisted_dispatch::{self, TwistedDispatchReader, TwistedDispatcher}; + +/// How long to wait for in-flight tokio tasks to be cancelled when shutting +/// down with the reactor. +/// +/// Note that any [`Runtime::spawn_blocking`] work that is still running when +/// the timeout expires is leaked, along with the worker thread running it. +/// +/// See [`tokio::runtime`] for details. +const SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100); + +/// State of the lazily-started tokio runtime. +enum TokioState { + /// Not started yet; the runtime is built on first use. + NotStarted, + Running(Runtime), + /// Shut down with the homeserver. Cannot be restarted. + Shutdown, +} + +/// The state shared between the Python-facing [`RustRuntime`] handle and any +/// Rust-side consumers holding an `Arc` of this. +pub struct RustRuntimeInner { + reactor: Reactor, + tokio: Mutex, + worker_threads: usize, + + /// Runs closures on the Twisted reactor thread without taking the GIL on + /// the calling thread. See [`crate::twisted_dispatch`]. + dispatcher: Arc, + /// The reactor-facing half of `dispatcher`, registered with the Twisted + /// reactor via `addReader` until `shutdown`. + dispatch_reader: Py, +} + +impl RustRuntimeInner { + /// Queue `f` to run on the Twisted reactor thread with the GIL held, and + /// wake the reactor. Never takes the GIL itself, so a tokio task can call + /// it to hand a result back to Twisted. See [`crate::twisted_dispatch`]. + /// + /// This is the equivalent of calling `reactor.callFromThread` in Python and + /// should be used by Rust code instead of `callFromThread`. + /// + /// Returns an error once the homeserver has shut down. + pub fn dispatch_to_twisted(&self, f: F) -> anyhow::Result<()> + where + F: FnOnce(Python<'_>) + Send + 'static, + { + self.dispatcher.dispatch(f) + } + + /// Get a handle to the tokio runtime, starting the runtime if it hasn't + /// been started yet. + pub fn tokio_handle(&self) -> PyResult { + let mut state = self + .tokio + .lock() + .map_err(|_| PyRuntimeError::new_err("tokio runtime lock poisoned"))?; + + match &*state { + TokioState::Running(runtime) => Ok(runtime.handle().clone()), + TokioState::NotStarted => { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(self.worker_threads) + .enable_all() + .build() + .context("building tokio runtime")?; + let handle = runtime.handle().clone(); + *state = TokioState::Running(runtime); + Ok(handle) + } + TokioState::Shutdown => Err(PyRuntimeError::new_err( + "the tokio runtime has been shut down", + )), + } + } + + /// Shut the tokio runtime down, cancelling all in-flight tasks and waiting + /// for up to [`SHUTDOWN_TIMEOUT`] for them to finish. Called via + /// [`ShutdownHook`] when the reactor shuts down. + /// + /// Note that any [`Runtime::spawn_blocking`] work is leaked until it is + /// finished, along with the worker thread running it. + fn shutdown(&self, py: Python<'_>) -> PyResult<()> { + let mut state = self + .tokio + .lock() + .map_err(|_| PyRuntimeError::new_err("tokio runtime lock poisoned"))?; + let previous_state = std::mem::replace(&mut *state, TokioState::Shutdown); + // Don't hold the lock while blocking on the shutdown below. + drop(state); + + if let TokioState::Running(runtime) = previous_state { + // Shutdown the runtime, waiting for a small grace period for + // in-flight tasks to be cancelled. + // + // See [`tokio::runtime`] for details. + py.detach(|| runtime.shutdown_timeout(SHUTDOWN_TIMEOUT)); + } + + // Unregister the wakeup socket from the Twisted reactor, then close + // the dispatcher and run the closures still queued so that their + // deferreds fire. + // + // All tokio tasks should be stopped by now, so only long running + // blocking threads may still be active. If they try to dispatch new + // work they get an error and should stop. + self.reactor + .remove_reader(py, self.dispatch_reader.bind(py).as_any())?; + self.dispatch_reader.get().close_and_drain(py); + + Ok(()) + } +} + +impl Drop for RustRuntimeInner { + fn drop(&mut self) { + // Backstop for homeservers whose shutdown trigger never fires (e.g. in + // tests). We use `shutdown_background` rather than a blocking shutdown, + // because the last `Arc` may be dropped from a task running on this + // very runtime, where blocking would panic. + if let Ok(state) = self.tokio.get_mut() { + if let TokioState::Running(runtime) = std::mem::replace(state, TokioState::Shutdown) { + runtime.shutdown_background(); + } + } + } +} + +/// A cheaply-clonable handle to the per-homeserver Rust state, and the +/// Python-facing class for it. +/// +/// One instance is constructed per homeserver by +/// `HomeServer.get_rust_runtime()`. Rust classes that need it take it as a +/// constructor argument and store their own clone, which is just an `Arc` +/// refcount bump. Derefs to [`RustRuntimeInner`]. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct RustRuntime { + inner: Arc, +} + +impl Deref for RustRuntime { + type Target = RustRuntimeInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[pymethods] +impl RustRuntime { + #[new] + #[pyo3(signature = (hs, worker_threads = 4))] + fn py_new(py: Python<'_>, hs: HomeServer, worker_threads: usize) -> PyResult { + let reactor = hs.get_reactor(py)?; + + // Register the read end of the dispatcher's wakeup socket with the + // Twisted reactor, so that closures dispatched by tokio tasks run on + // the reactor thread. `shutdown` removes it again. + let (dispatcher, dispatch_reader) = twisted_dispatch::new_pair()?; + let dispatch_reader = Py::new(py, dispatch_reader)?; + reactor.add_reader(py, dispatch_reader.bind(py).as_any())?; + + let inner = Arc::new(RustRuntimeInner { + reactor, + tokio: Mutex::new(TokioState::NotStarted), + worker_threads, + dispatcher, + dispatch_reader, + }); + + // Shut the tokio runtime down when the homeserver is shut down. The + // trigger holds only a `Weak` reference, as otherwise we risk a + // reference cycle passing through a Rust field that Python's GC cannot + // see into. + let hook = Py::new( + py, + ShutdownHook { + inner: Arc::downgrade(&inner), + }, + )?; + hs.register_sync_shutdown_handler(py, hook.bind(py).as_any())?; + + Ok(RustRuntime { inner }) + } +} + +/// The callable registered with `HomeServer.register_sync_shutdown_handler`, +/// which runs it on `HomeServer.shutdown()` or when the reactor stops. +#[pyclass(frozen)] +struct ShutdownHook { + inner: Weak, +} + +#[pymethods] +impl ShutdownHook { + fn __call__(&self, py: Python<'_>) -> PyResult<()> { + if let Some(inner) = self.inner.upgrade() { + inner.shutdown(py)?; + } + + Ok(()) + } +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "runtime")?; + + child_module.add_class::()?; + + m.add_submodule(&child_module)?; + + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.runtime", child_module)?; + + Ok(()) +} diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 89003d574b..030d3cbf99 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -40,6 +40,7 @@ use pyo3::{ }; use crate::deferred::run_python_awaitable; +use crate::runtime::RustRuntime; use crate::storage::db::{ DatabasePool, DbRow, DbValue, ErasedInteraction, ErasedResult, Transaction, }; @@ -102,26 +103,22 @@ pub struct PythonDatabasePoolWrapper { /// We use a *weak* reference to ensure the homeserver can cleanly shut down as /// otherwise we hold onto `DatabasePool` which references the homeserver and keeps /// the homeserver from being garbage collected. - database_pool_py_ref: Py, - - /// The Twisted reactor. We need this to marshal database work back onto the - /// reactor thread (via `callFromThread`) when starting transactions, since - /// Twisted's thread pool machinery must be driven from there. /// - /// A strong reference is fine here: the reactor is a process-global singleton that - /// never gets garbage collected and never points back at the homeserver, so it is - /// not part of any reference cycle. Ideally, we could worry about it but - /// practically probably doesn't matter. - reactor: Py, + /// In an `Arc` so it can be handed to the reactor thread without the GIL. + database_pool_py_ref: Arc>, + + /// A strong reference is fine here, the runtime does not reference the + /// homeserver and so does not cause a reference cycle. + runtime: RustRuntime, } impl PythonDatabasePoolWrapper { /// Build a wrapper around the Python `DatabasePool` (e.g. - /// `hs.get_datastores().main.db_pool`) and the Twisted `reactor`. - pub fn new(database_pool: &Bound<'_, PyAny>, reactor: Py) -> PyResult { + /// `hs.get_datastores().main.db_pool`) and the homeserver's `RustRuntime`. + pub fn new(database_pool: &Bound<'_, PyAny>, runtime: RustRuntime) -> PyResult { Ok(Self { - database_pool_py_ref: PyWeakrefReference::new(database_pool)?.unbind(), - reactor, + database_pool_py_ref: Arc::new(PyWeakrefReference::new(database_pool)?.unbind()), + runtime, }) } } @@ -145,93 +142,85 @@ impl DatabasePool for PythonDatabasePoolWrapper { // failed. let result_slot: Arc>> = Arc::new(Mutex::new(None)); - // Build the callback that Python's `runInteraction` invokes on a DB - // thread with a `LoggingTransaction`, plus owned handles we can move onto - // the reactor thread. We drive `func` to completion in the callback; the - // Python query path is synchronous under the hood, so it's safe to block - // this dedicated DB thread until the future resolves. + // Call Python's `runInteraction` with the callback and wait for the + // result. + // + // First we need to build a Python callable that can be passed to + // `runInteraction`, which wraps the given `func`. This requires the GIL + // and so we build the callable within `run_python_awaitable` (which is + // run on the Twisted reactor thread). + // + // Once we have built the Python callable, we can pass it to + // `runInteraction` and wait for the result. let callback_slot = Arc::clone(&result_slot); - let (callback, database_pool_py, reactor) = Python::attach(|py| -> PyResult<_> { - let callback = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let py = args.py(); - let txn_py = args.get_item(0)?; - let mut txn = txn_py.extract::()?; + let func = Arc::new(func); + let database_pool_py_ref = Arc::clone(&self.database_pool_py_ref); + let run_interaction_outcome = run_python_awaitable(&self.runtime, move |py| { + // Build a Python callable that wraps the Rust `func` and stores its + // result in the callback slot. + let func = Arc::clone(&func); + let callback_slot = Arc::clone(&callback_slot); + let callback = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + let txn_py = args.get_item(0)?; + let mut txn = txn_py.extract::()?; - // Since we expect people to only call `.await` on - // [`Transaction`] related methods (mentioned in the - // [`Transaction`] docstring) AND because there is no actual - // async work to suspend on in the Python [`Transaction`] - // (resolves synchonously), we can get away with polling once as - // it should immediately resolve to [`Poll::Ready`]. Getting - // [`Poll::Pending`] would be considered a programming error. - // - // Alternatively, we could just use `futures::executor::block_on` - // which is probably cleaner but a single-shot poll is more - // enforcing of the concept we want to represent. - match func(&mut txn).now_or_never() { - Some(Ok(value)) => { - let mut callback_slot = callback_slot - .lock() - .map_err(|err| anyhow::anyhow!("Failed to acquire lock on `callback_slot`: {:#}", err))?; - *callback_slot = Some(Ok(value)); - Ok(py.None()) - } - Some(Err(err)) => { - // Re-raise into Python so `runInteraction` rolls the - // transaction back (and can apply its retry logic for - // serialization/deadlock errors). - let py_err = anyhow_to_pyerr(&err); - let mut callback_slot = callback_slot - .lock() - .map_err(|err| anyhow::anyhow!("Failed to acquire lock on `callback_slot`: {:#}", err))?; - *callback_slot = Some(Err(err)); - Err(py_err) - } - None => { - Err(PyAssertionError::new_err( - "The `run_interaction` transaction callback future returned `Poll::Pending`, \ - but we expect Synapse Python database work to resolve synchronously. \ - This is a Synapse programming error: genuine async work is \ - not supported here.", - )) - } + // Since we expect people to only call `.await` on + // [`Transaction`] related methods (mentioned in the + // [`Transaction`] docstring) AND because there is no actual + // async work to suspend on in the Python [`Transaction`] + // (resolves synchonously), we can get away with polling once as + // it should immediately resolve to [`Poll::Ready`]. Getting + // [`Poll::Pending`] would be considered a programming error. + // + // Alternatively, we could just use `futures::executor::block_on` + // which is probably cleaner but a single-shot poll is more + // enforcing of the concept we want to represent. + match func(&mut txn).now_or_never() { + Some(Ok(value)) => { + let mut callback_slot = callback_slot.lock().map_err(|err| { + anyhow::anyhow!("Failed to lock `callback_slot`: {:#}", err) + })?; + *callback_slot = Some(Ok(value)); + Ok(py.None()) } - }, - )? - .unbind(); + Some(Err(err)) => { + // Re-raise into Python so `runInteraction` rolls the + // transaction back (and can apply its retry logic for + // serialization/deadlock errors). + let py_err = anyhow_to_pyerr(py, &err); + let mut callback_slot = callback_slot.lock().map_err(|err| { + anyhow::anyhow!("Failed to lock `callback_slot`: {:#}", err) + })?; + *callback_slot = Some(Err(err)); + Err(py_err) + } + None => Err(PyAssertionError::new_err( + "The `run_interaction` transaction callback future returned \ + `Poll::Pending`, but we expect Synapse Python database work to \ + resolve synchronously. This is a Synapse programming error: \ + genuine async work is not supported here.", + )), + } + }, + )?; - // Upgrade our weak reference to the Python `DatabasePool` into a strong - // one for the duration of this interaction. - let database_pool_py = self - .database_pool_py_ref - .bind(py) - .upgrade() - .ok_or_else(|| { - PyRuntimeError::new_err( - "The Python `DatabasePool` has already been dropped \ - (the homeserver is likely shutdown), so we cannot \ - run the database interaction.", - ) - })? - .unbind(); + // Upgrade our weak reference to the Python `DatabasePool` into a strong + // one for the duration of this interaction. + let database_pool_py = database_pool_py_ref.bind(py).upgrade().ok_or_else(|| { + PyRuntimeError::new_err( + "The Python `DatabasePool` has already been dropped \ + (the homeserver is likely shutdown), so we cannot \ + run the database interaction.", + ) + })?; - Ok(( - callback, - database_pool_py, - self.reactor.clone_ref(py), - )) - }) - .map_err(anyhow::Error::from)?; - - // Use `runInteraction` directly - let run_interaction_outcome = run_python_awaitable(reactor, move |py| { - database_pool_py - .bind(py) - .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) + // Use `runInteraction` directly + database_pool_py.call_method1(intern!(py, "runInteraction"), (name, callback)) }) .await; @@ -250,11 +239,12 @@ impl DatabasePool for PythonDatabasePoolWrapper { // completed successfully and run the provided `callback` which runs the // `func` and we capture a result or it fails. None => Err(anyhow::anyhow!( - "Expected to capture result after running `runInteraction` and seeing it succeed (but saw nothing). \ - This is a Synapse programming error." + "Expected to capture result after running `runInteraction` and seeing it \ + succeed (but saw nothing). This is a Synapse programming error." )), }, - Err(py_err) => Err(anyhow::Error::from(py_err)).with_context(|| format!("run_interaction(name={}) failed", name)), + Err(py_err) => Err(anyhow::Error::from(py_err)) + .with_context(|| format!("run_interaction(name={}) failed", name)), } } } @@ -265,9 +255,9 @@ impl DatabasePool for PythonDatabasePoolWrapper { /// surfaced through [`Transaction::query`]), we re-raise *that* exception so /// Synapse's transaction machinery can apply its retry logic /// (serialization/deadlock detection) on the real error. -fn anyhow_to_pyerr(err: &anyhow::Error) -> PyErr { +fn anyhow_to_pyerr(py: Python<'_>, err: &anyhow::Error) -> PyErr { if let Some(py_err) = err.downcast_ref::() { - return Python::attach(|py| py_err.clone_ref(py)); + return py_err.clone_ref(py); } PyRuntimeError::new_err(format!("{err:#}")) } diff --git a/rust/src/tokio_runtime.rs b/rust/src/tokio_runtime.rs deleted file mode 100644 index f239040f30..0000000000 --- a/rust/src/tokio_runtime.rs +++ /dev/null @@ -1,107 +0,0 @@ -/* - * This file is licensed under the Affero General Public License (AGPL) version 3. - * - * Copyright (C) 2026 Element Creations Ltd - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * See the GNU Affero General Public License for more details: - * . - * - */ - -use anyhow::Context; -use pyo3::prelude::*; -use tokio::runtime::Runtime; - -/// This is the name of the attribute where we store the runtime on the reactor -static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; - -/// A Python wrapper around a Tokio runtime. -/// -/// This allows us to 'store' the runtime on the reactor instance, starting it -/// when the reactor starts, and stopping it when the reactor shuts down. -#[pyclass] -pub struct PyTokioRuntime { - runtime: Option, -} - -#[pymethods] -impl PyTokioRuntime { - fn start(&mut self) -> PyResult<()> { - // TODO: allow customization of the runtime like the number of threads - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .enable_all() - .build()?; - - self.runtime = Some(runtime); - - Ok(()) - } - - fn shutdown(&mut self) -> PyResult<()> { - let runtime = self - .runtime - .take() - .context("Runtime was already shutdown")?; - - // Dropping the runtime will shut it down - drop(runtime); - - Ok(()) - } -} - -impl PyTokioRuntime { - /// Get the handle to the Tokio runtime, if it is running. - pub fn handle(&self) -> PyResult<&tokio::runtime::Handle> { - let handle = self - .runtime - .as_ref() - .context("Tokio runtime is not running")? - .handle(); - - Ok(handle) - } -} - -/// Get a handle to the Tokio runtime stored on the reactor instance, or create -/// a new one. -pub fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { - install_runtime(reactor)?; - } - - get_runtime(reactor) -} - -/// Install a new Tokio runtime on the reactor instance. -fn install_runtime(reactor: &Bound) -> PyResult<()> { - let py = reactor.py(); - let runtime = PyTokioRuntime { runtime: None }; - let runtime = runtime.into_pyobject(py)?; - - // Attach the runtime to the reactor, starting it when the reactor is - // running, stopping it when the reactor is shutting down - reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; - reactor.call_method1( - "addSystemEventTrigger", - ("after", "shutdown", runtime.getattr("shutdown")?), - )?; - reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; - - Ok(()) -} - -/// Get a reference to a Tokio runtime handle stored on the reactor instance. -fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is - // not a `Runtime`. Careful that this could happen if the user sets it - // manually, or if multiple versions of `pyo3-twisted` are used! - let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; - Ok(runtime.borrow()) -} diff --git a/rust/src/twisted_dispatch.rs b/rust/src/twisted_dispatch.rs new file mode 100644 index 0000000000..158dd0b9c6 --- /dev/null +++ b/rust/src/twisted_dispatch.rs @@ -0,0 +1,335 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Run closures on the Twisted reactor thread from Tokio worker threads, +//! without taking the GIL on the worker. +//! +//! Taking the GIL on a worker would block the worker until the Python thread +//! releases it, which under load means waiting for the interpreter's switch +//! interval. This could severely impact the performance of the Tokio runtime. +//! +//! # Implementation +//! +//! The implementation combines a queue of closures with the self-pipe trick, +//! which is the same implementation as Twisted uses for `callFromThread`. +//! +//! ```text +//! Tokio worker Twisted reactor +//! │ │ +//! TwistedDispatcher | +//! ├─ Add a closure to a Rust queue │ +//! └─ Write one byte to socket A ───────► socket B becomes readable +//! │ +//! TwistedDispatchReader +//! └─ Reader.doRead() +//! ├─ Drain socket B +//! ├─ Take queued closures +//! └─ Execute them +//! ``` +//! +//! The [`TwistedDispatcher`] worker holds a queue of closures and the write end +//! of a unix socket pair. A worker pushes a closure and writes one byte. The +//! [`TwistedDispatchReader`] holds the read end and is registered with the +//! Twisted reactor via `addReader`. The Twisted reactor is woken up and calls +//! the `doRead` method on the Twisted reactor thread, which then reads from the +//! queue and runs any pending closures. +//! +//! One pair exists per homeserver, owned by [`crate::runtime::RustRuntime`], +//! which registers the reader on construction and removes it at shutdown. +//! +//! # Homeserver shutdown +//! +//! When the homeserver shuts down, the dispatcher is closed and `dispatch` will +//! return an error. The caller should stop what it is doing as its associated +//! homeserver has been shut down. + +use std::{ + any::Any, + io::{ErrorKind, Read, Write}, + os::{ + fd::{AsRawFd, RawFd}, + unix::net::UnixStream, + }, + panic::{catch_unwind, AssertUnwindSafe}, + sync::{Arc, Mutex}, +}; + +use anyhow::Context; +use log::error; +use pyo3::prelude::*; + +use crate::logging::context::with_logcontext; + +/// A queued closure. Runs on the Twisted reactor thread. +type DispatchedCall = Box) + Send + 'static>; + +/// The Tokio-facing half: a queue of closures plus the write end of the wakeup +/// socket. +pub struct TwistedDispatcher { + /// The queue of closures to run on the Twisted reactor thread. Or None if + /// the dispatcher has been closed, indicating the homeserver has been + /// shutdown. + queue: Mutex>>, + write_end: UnixStream, +} + +impl TwistedDispatcher { + /// Queue `f` to run on the Twisted reactor thread, and wake the reactor. + /// + /// Never takes the GIL, so this is safe to call from a Tokio worker. + /// + /// Returns an error if the dispatcher has been closed, which indicates the + /// homeserver has been shutdown. + pub fn dispatch(&self, f: F) -> anyhow::Result<()> + where + F: FnOnce(Python<'_>) + Send + 'static, + { + // First add the closure to the queue. + { + let mut queue = self.queue.lock().expect("dispatcher poisoned"); + let Some(queue) = &mut *queue else { + return Err(anyhow::anyhow!("dispatcher is closed")); + }; + + queue.push(Box::new(f)); + } + + // Second, write a byte to the socket to wakeup the Twisted reactor. + // + // Errors are ignored. `WouldBlock` means the socket buffer is full, so + // a wakeup is already pending. Nothing else can fail while the reader + // holds the other end. + let _ = (&self.write_end).write(b"x"); + + Ok(()) + } +} + +/// The Twisted-reactor-facing half of a [`TwistedDispatcher`]. +#[pyclass(frozen)] +pub struct TwistedDispatchReader { + dispatcher: Arc, + read_end: UnixStream, +} + +#[pymethods] +impl TwistedDispatchReader { + fn fileno(&self) -> RawFd { + self.read_end.as_raw_fd() + } + + #[pyo3(name = "logPrefix")] + fn log_prefix(&self) -> &'static str { + "synapse-rust-twisted-dispatch" + } + + /// Called by the Twisted reactor when the wakeup socket has something in it. + /// + /// Runs on the reactor thread. + #[pyo3(name = "doRead")] + fn do_read(&self, py: Python<'_>) { + // Drain the socket before taking the queue. This way around avoids + // races between the socket being drained and closures being added to + // the queue. + let mut buf = [0u8; 64]; + loop { + match (&self.read_end).read(&mut buf) { + Ok(0) => break, + Ok(_) => continue, + // Interrupted: try again. + Err(err) if err.kind() == ErrorKind::Interrupted => continue, + // WouldBlock: the socket buffer is empty, so we can stop + // draining. + Err(err) if err.kind() == ErrorKind::WouldBlock => break, + Err(err) => { + // This should not happen, as the socket is never closed + // except when this struct is dropped. + // + // We can't do much here except log the error and break out + // of the loop. + error!("Error reading from wakeup socket: {:?}", err); + break; + } + } + } + + // Take from the queue. Ensuring the lock is released before the + // closures run. This may be `None` if the dispatcher has been closed. + let jobs = self + .dispatcher + .queue + .lock() + .expect("dispatcher poisoned") + .as_mut() + .map(std::mem::take); + if let Some(jobs) = jobs { + self.run_dispatched(py, jobs); + } + + // Returning `None` keeps the reader registered. + } + + /// Called by the Twisted reactor at shutdown. The socket closes when the + /// owning `RustRuntime` drops the reader, so there is nothing to do here. + #[pyo3(name = "connectionLost")] + fn connection_lost(&self, _reason: &Bound<'_, PyAny>) {} +} + +impl TwistedDispatchReader { + /// Close the dispatcher so that no further closures can be queued, then + /// run the ones already queued. + pub fn close_and_drain(&self, py: Python<'_>) { + let Some(jobs) = self + .dispatcher + .queue + .lock() + .expect("dispatcher poisoned") + .take() + else { + return; + }; + + self.run_dispatched(py, jobs); + } + + /// Run closures taken from the queue. Must be called with the queue lock + /// released. + fn run_dispatched(&self, py: Python<'_>, jobs: Vec) { + for job in jobs { + // A panic here would leave `doRead` as an exception, and Twisted + // responds to that by dropping the reader. No later closure would + // ever run. Log it instead. + // + // `catch_unwind(AssertUnwindSafe(..))` is exactly the same as what + // pyo3 does when catching panics in Rust code called from Python. + // + // `UnwindSafe` guards against code after `catch_unwind` reading + // state that the panicking code was part way through mutating. + // Since we never read `job` after this, it is safe to catch the + // panic. `Python` is already `UnwindSafe`. + let ran = catch_unwind(AssertUnwindSafe(move || { + // Ensure the dispatched job gets run within the sentinel + // logcontext, and that the sentinel context is restored + // afterwards. + with_logcontext(py, None, move || { + job(py); + Ok(()) + }) + })); + + match ran { + Ok(Ok(())) => {} + Ok(Err(err)) => { + // Log an error if setting the logcontexts in the dispatched + // Rust completion fails. + log::error!( + "error setting logcontexts in dispatched Rust completion: {}", + err + ); + } + Err(err) => { + // Log an error if the dispatched Rust completion panicked. + log::error!( + "panic while running a dispatched Rust completion: {}", + panic_payload_as_str(&*err) + ); + } + } + } + } +} + +/// Convert a panic payload to a string for logging purposes. +/// +/// This is the object passed to [`panic!`] and returned from [`catch_unwind`]. +/// The vast majority of panics are either a `&'static str` or a `String`, but +/// technically they could be anything. +fn panic_payload_as_str(payload: &dyn Any) -> &str { + if let Some(&s) = payload.downcast_ref::<&'static str>() { + s + } else if let Some(s) = payload.downcast_ref::() { + s.as_str() + } else { + "" + } +} + +/// Build a connected [`TwistedDispatcher`]/[`TwistedDispatchReader`] pair. +pub fn new_pair() -> PyResult<(Arc, TwistedDispatchReader)> { + let (read_end, write_end) = + UnixStream::pair().context("creating the Tokio wakeup socket pair")?; + + // Neither end may block. The writer is a Tokio worker and the reader is + // the Twisted reactor thread. + read_end + .set_nonblocking(true) + .context("making the Tokio wakeup socket non-blocking")?; + write_end + .set_nonblocking(true) + .context("making the Tokio wakeup socket non-blocking")?; + + let dispatcher = Arc::new(TwistedDispatcher { + queue: Mutex::new(Some(Vec::new())), + write_end, + }); + let reader = TwistedDispatchReader { + dispatcher: Arc::clone(&dispatcher), + read_end, + }; + + Ok((dispatcher, reader)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::logging::context::{current_context, set_current_context, testing::test_context}; + + /// `close_and_drain` runs from the homeserver's shutdown handler, in the + /// caller's logcontext. Ensure that the logcontext is correctly preserved + /// across `close_and_drain` no matter what the dispatch job leaves it as. + #[test] + fn close_and_drain_runs_at_sentinel_and_restores_caller_logcontext() { + Python::initialize(); + Python::attach(|py| { + let (dispatcher, reader) = new_pair().expect("creating the dispatcher pair"); + + // Create and set the "shutdown" log context. + let caller = test_context(py, "shutdown"); + set_current_context(py, Some(caller.clone_ref(py))).expect("setting caller context"); + + // Dispatch a closure that changes the logcontext. + dispatcher + .dispatch(move |py| { + // Mimic the dispatch job not restoring the sentinel context. + let awaiter = test_context(py, "requester"); + set_current_context(py, Some(awaiter)).expect("switching to the awaiter"); + }) + .expect("dispatching while open"); + + // Call `close_and_drain`, which should restore the caller's + // context. + reader.close_and_drain(py); + + // Check the caller's context is current again. + let after = current_context(py).expect("caller's context was lost"); + assert!(after.is(&caller), "caller's context was not restored"); + + // Reset to the sentinel so we don't leak into another test that + // reuses this OS thread from the test harness's pool. + set_current_context(py, None).expect("resetting to the sentinel"); + }); + } +} diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index b34bc374f3..1a929912a3 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.161/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.162/synapse-config.schema.json type: object properties: modules: @@ -252,6 +252,9 @@ properties: display names) of other users through the client API. Note that profile data is also available via the federation API, unless `allow_profile_lookup_over_federation` is set to false. + + This option must be enabled in order to use + `limit_profile_requests_to_users_who_share_rooms`. default: false examples: - true @@ -262,6 +265,10 @@ properties: order to retrieve their profile information. Only checked on Client-Server requests. Profile requests from other servers should be checked by the requesting server. + + The check can only be applied to authenticated requests, so this option + requires `require_auth_for_profile_requests` to be enabled as well. + Synapse will refuse to start if this option is enabled without it. default: false examples: - true @@ -330,7 +337,11 @@ properties: _Changed in Synapse 1.157:_ the default version room version was increased from [10](https://spec.matrix.org/v1.12/rooms/v10/) to [11](https://spec.matrix.org/v1.12/rooms/v11/). - default: "11" + + _Changed in Synapse 1.162:_ the default room version was increased from + [11](https://spec.matrix.org/v1.16/rooms/v11/) to + [12](https://spec.matrix.org/v1.16/rooms/v12/) + default: "12" examples: - "8" gc_thresholds: @@ -2387,6 +2398,20 @@ properties: default: per_second: 0.016 burst_count: 200.0 + rc_profile: + $ref: "#/$defs/rc" + description: >- + This option allows admins to ratelimit profile lookups by clients. + + + Requests are limited per user when the request is authenticated, + otherwise per client IP address. + + + _Added in Synapse 1.162.0._ + default: + per_second: 1.0 + burst_count: 500.0 federation_rr_transactions_per_room_per_second: type: integer description: >- @@ -5767,6 +5792,20 @@ properties: default: null examples: - worker1 + task_scheduler: + type: object + description: >- + Configuration for the task scheduler. + properties: + max_concurrent_tasks: + type: integer + minimum: 1 + description: >- + The maximum number of tasks that can run concurrently in the task + scheduler. Setting this too high may swamp the database connection pool. + default: 5 + examples: + - max_concurrent_tasks: 5 update_user_directory_from_worker: type: ["string", "null"] description: >- diff --git a/stubs/txredisapi.pyi b/stubs/txredisapi.pyi index 7454ad5d14..8ec2f31420 100644 --- a/stubs/txredisapi.pyi +++ b/stubs/txredisapi.pyi @@ -86,7 +86,7 @@ class RedisFactory(protocol.ReconnectingClientFactory): convertNumbers: int | None = True, username: str | None = None, ): ... - def buildProtocol(self, addr: IAddress) -> RedisProtocol: ... + def buildProtocol(self, addr: IAddress | None) -> RedisProtocol: ... class SubscriberFactory(RedisFactory): def __init__(self) -> None: ... diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py index ff6e4e297b..64782147fb 100644 --- a/synapse/_scripts/register_new_matrix_user.py +++ b/synapse/_scripts/register_new_matrix_user.py @@ -199,7 +199,7 @@ def main() -> None: parser.add_argument( "--exists-ok", action="store_true", - help="Do not fail if user already exists.", + help="Do not fail if the user already exists. The user account will not be updated.", ) password_group = parser.add_mutually_exclusive_group() password_group.add_argument( diff --git a/synapse/api/auth/__init__.py b/synapse/api/auth/__init__.py index d8d3b31b9d..dd0f05b1d3 100644 --- a/synapse/api/auth/__init__.py +++ b/synapse/api/auth/__init__.py @@ -98,6 +98,20 @@ class Auth(Protocol): AuthError if access is denied for the user in the access token """ + async def get_optional_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, + ) -> Requester | None: + """Like `get_user_by_req`, except returns None when the request carries + no access token at all. A token that is present but invalid still + raises, as with `get_user_by_req`. + + For endpoints where authentication is optional. + """ + async def get_user_by_req_experimental_feature( self, request: SynapseRequest, diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index 7b8214fabb..93f579c69a 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -19,6 +19,7 @@ # # import logging +from abc import ABC, abstractmethod from http import HTTPStatus from typing import TYPE_CHECKING @@ -49,7 +50,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -class BaseAuth: +class BaseAuth(ABC): """Common base class for all auth implementations.""" def __init__(self, hs: "HomeServer"): @@ -187,6 +188,7 @@ class BaseAuth: 403, "Application service has not registered this user (%s)" % user_id ) + @abstractmethod async def is_server_admin(self, requester: Requester) -> bool: """Check if the given user is a local server admin. @@ -240,6 +242,40 @@ class BaseAuth: return user_level >= send_level + @abstractmethod + async def get_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, + ) -> Requester: + """Get a registered user's ID. See `Auth.get_user_by_req`.""" + raise NotImplementedError() + + async def get_optional_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, + ) -> Requester | None: + """Like `get_user_by_req`, except returns None when the request carries + no access token at all. A token that is present but invalid still + raises, as with `get_user_by_req`. + + For endpoints where authentication is optional. + """ + if not self.has_access_token(request): + return None + + return await self.get_user_by_req( + request, + allow_guest=allow_guest, + allow_expired=allow_expired, + allow_locked=allow_locked, + ) + @staticmethod def has_access_token(request: Request) -> bool: """Checks if the request has an access_token. diff --git a/synapse/api/auth/mas.py b/synapse/api/auth/mas.py index ed0427d6f3..d094410ff9 100644 --- a/synapse/api/auth/mas.py +++ b/synapse/api/auth/mas.py @@ -109,7 +109,7 @@ class MasDelegatedAuth(BaseAuth): self._http_client = hs.get_proxied_http_client() self._rust_http_client = HttpClient( - reactor=hs.get_reactor(), + runtime=hs.get_rust_runtime(), user_agent=self._http_client.user_agent.decode("utf8"), http2_only=self._config.force_http2, ) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 17e6a4cfdb..bfdb3ef926 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -502,6 +502,8 @@ class StickyEvent: Maximum number of sticky events to include in /sync. This is the default specified in the MSC. Chosen arbitrarily. + + Does not apply when joining a new room, see https://github.com/matrix-org/matrix-spec-proposals/pull/4354#discussion_r3021907998 """ diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 4814370684..7efa6d8fe2 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -372,10 +372,12 @@ def listen_manhole( ) +# These listener helpers do not inspect the factory's protocol type. Erase that +# invariant type parameter, quoting the annotation for older Twisted versions. def listen_tcp( bind_addresses: StrCollection, port: int, - factory: ServerFactory, + factory: "ServerFactory[Any]", reactor: IReactorTCP = reactor, backlog: int = 50, ) -> list[Port]: @@ -400,7 +402,7 @@ def listen_tcp( def listen_unix( path: str, mode: int, - factory: ServerFactory, + factory: "ServerFactory[Any]", reactor: IReactorUNIX = reactor, backlog: int = 50, ) -> list[Port]: @@ -541,7 +543,7 @@ def listen_http( def listen_ssl( bind_addresses: StrCollection, port: int, - factory: ServerFactory, + factory: "ServerFactory[Any]", context_factory: IOpenSSLContextFactory, reactor: IReactorSSL = reactor, backlog: int = 50, diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 159cd44237..6014d328ba 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -93,6 +93,7 @@ from synapse.storage.databases.main.push_rule import PushRulesWorkerStore from synapse.storage.databases.main.pusher import PusherWorkerStore from synapse.storage.databases.main.receipts import ReceiptsWorkerStore from synapse.storage.databases.main.registration import RegistrationWorkerStore +from synapse.storage.databases.main.rejections import RejectionsStore from synapse.storage.databases.main.relations import RelationsWorkerStore from synapse.storage.databases.main.room import RoomWorkerStore from synapse.storage.databases.main.roommember import RoomMemberWorkerStore @@ -153,6 +154,7 @@ class GenericWorkerStore( EventFederationWorkerStore, EventPushActionsWorkerStore, PurgeEventsStore, + RejectionsStore, StateGroupWorkerStore, SignatureWorkerStore, UserErasureWorkerStore, diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index f226afc2b4..3e2869c74b 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -52,6 +52,7 @@ from synapse.config import ( # noqa: F401 spam_checker, sso, stats, + task_scheduler, third_party_event_rules, tls, tracer, @@ -122,6 +123,7 @@ class RootConfig: background_updates: background_updates.BackgroundUpdateConfig auto_accept_invites: auto_accept_invites.AutoAcceptInvitesConfig user_types: user_types.UserTypesConfig + task_scheduler: task_scheduler.TaskSchedulerConfig mas: mas.MasConfig matrix_rtc: matrixrtc.MatrixRtcConfig diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 1c2f021322..c80b2740c3 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -191,6 +191,9 @@ class ExperimentalConfig(Config): "msc4069_profile_inhibit_propagation", False ) + # MSC4075: MatrixRTC invites and notifications + self.msc4075_enabled: bool = experimental.get("msc4075_enabled", False) + # MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version: # See: https://github.com/element-hq/synapse/issues/19434 self.msc4108_enabled = experimental.get("msc4108_enabled", False) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index e2ae182fc3..e3b5ab84c3 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -60,6 +60,7 @@ from .server_notices import ServerNoticesConfig from .spam_checker import SpamCheckerConfig from .sso import SSOConfig from .stats import StatsConfig +from .task_scheduler import TaskSchedulerConfig from .third_party_event_rules import ThirdPartyRulesConfig from .tls import TlsConfig from .tracer import TracerConfig @@ -120,6 +121,7 @@ class HomeServerConfig(RootConfig): BackgroundUpdateConfig, AutoAcceptInvitesConfig, UserTypesConfig, + TaskSchedulerConfig, # This must be last, as it checks for conflicts with other config options. MasConfig, ] diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 13c9c4dba0..e5f29b6dc1 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -258,3 +258,9 @@ class RatelimitConfig(Config): "rc_user_directory", defaults={"per_second": 0.016, "burst_count": 200}, ) + + self.rc_profile = RatelimitSettings.parse( + config, + "rc_profile", + defaults={"per_second": 1, "burst_count": 500}, + ) diff --git a/synapse/config/server.py b/synapse/config/server.py index 9c5a42fc52..56ac229c00 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -176,7 +176,7 @@ DEFAULT_IP_RANGE_BLOCKLIST = [ "fec0::/10", ] -DEFAULT_ROOM_VERSION = "11" +DEFAULT_ROOM_VERSION = "12" # Defaults for the presence state machine timers, in milliseconds. Overridden # by the corresponding options in the `presence` config section. @@ -569,6 +569,20 @@ class ServerConfig(Config): False, ) + # The shared-room check needs to know who is asking, and profile requests + # are only authenticated when `require_auth_for_profile_requests` is set. + # Without it the limit would silently never apply, so refuse to start. + if ( + self.limit_profile_requests_to_users_who_share_rooms + and not self.require_auth_for_profile_requests + ): + raise ConfigError( + "'limit_profile_requests_to_users_who_share_rooms' can only be" + " enforced on authenticated requests, so" + " 'require_auth_for_profile_requests' must also be enabled.", + ("limit_profile_requests_to_users_who_share_rooms",), + ) + # Whether to retrieve and display profile data for a user when they # are invited to a room self.include_profile_data_on_invite = config.get( diff --git a/synapse/config/task_scheduler.py b/synapse/config/task_scheduler.py new file mode 100644 index 0000000000..7ac96b6e69 --- /dev/null +++ b/synapse/config/task_scheduler.py @@ -0,0 +1,47 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2026 Nevil Anson Dsouza +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +from typing import Any + +from synapse.types import JsonDict + +from ._base import Config, ConfigError + +DEFAULT_MAX_CONCURRENT_TASKS = 5 + + +class TaskSchedulerConfig(Config): + section = "task_scheduler" + + def read_config(self, config: JsonDict, **kwargs: Any) -> None: + task_scheduler_config = config.get("task_scheduler") or {} + if not isinstance(task_scheduler_config, dict): + raise ConfigError("'task_scheduler' must be a mapping", ("task_scheduler",)) + + max_concurrent_tasks = task_scheduler_config.get( + "max_concurrent_tasks", DEFAULT_MAX_CONCURRENT_TASKS + ) + # In Python, bool is a subclass of int (isinstance(True, int) is True). + # We explicitly check for bool to reject YAML booleans like `true`/`false`. + if ( + not isinstance(max_concurrent_tasks, int) + or isinstance(max_concurrent_tasks, bool) + or max_concurrent_tasks <= 0 + ): + raise ConfigError( + "'max_concurrent_tasks' must be a positive integer", + ("task_scheduler", "max_concurrent_tasks"), + ) + + self.max_concurrent_tasks = max_concurrent_tasks diff --git a/synapse/config/tls.py b/synapse/config/tls.py index de4d676e08..9be944a9ac 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -118,7 +118,7 @@ class TlsConfig(Config): # Parse the CA certificates try: - cert_base = Certificate.loadPEM(content) + cert_base = Certificate.loadPEM(content.encode("ascii")) certs.append(cert_base) except Exception as e: raise ConfigError( diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 81a454b544..c2a18889a3 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -164,6 +164,7 @@ async def check_state_independent_auth_rules( store: _EventSourceStore, event: "EventBase", batched_auth_events: Mapping[str, "EventBase"] | None = None, + batched_rejections: Mapping[str, str] | None = None, ) -> None: """Check that an event complies with auth rules that are independent of room state @@ -175,6 +176,11 @@ async def check_state_independent_auth_rules( event: the event being checked. batched_auth_events: if the event being authed is part of a batch, any events from the same batch that may be necessary to auth the current event + batched_rejections: event ID to rejection reason for any events in `batched_auth_events` + which have been rejected. Events in the batch have not been persisted + yet, so their rejection is recorded neither on the event nor in the database, + and the caller has to tell us. Needed for MSC4242 State DAG rooms, where + rejection cascades through `prev_state_events`. Raises: AuthError if the checks fail @@ -244,11 +250,14 @@ async def check_state_independent_auth_rules( ) # 2.3 If there are entries which were themselves rejected under the checks performed on # receipt of a PDU, reject. - if prev_state_event.rejected_reason is not None: + rejected_reason = prev_state_event.rejected_reason + if rejected_reason is None and batched_rejections is not None: + rejected_reason = batched_rejections.get(prev_state_event.event_id) + if rejected_reason is not None: raise AuthError( 403, f"During auth for event {event.event_id} in room {event.room_id}, event has a " - + f"prev_state_event which is rejected ({prev_state_event.rejected_reason}): " + + f"prev_state_event which is rejected ({rejected_reason}): " + f"{prev_state_event.event_id}", ) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index b01e786550..cab2fe1a46 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -199,6 +199,46 @@ class StrippedStateEvent: sender: str content: dict[str, Any] + def as_json_dict(self) -> JsonDict: + """ + Serialize to the JSON representation of 'Stripped State' according to the Matrix + spec + """ + + return { + "type": self.type, + "state_key": self.state_key, + "sender": self.sender, + "content": self.content, + } + + @staticmethod + def from_json_dict(raw_stripped_event: JsonDict) -> "StrippedStateEvent | None": + """ + Given a raw value from an event's `unsigned` field, attempt to parse it into a + `StrippedStateEvent`. + """ + if isinstance(raw_stripped_event, dict): + # All of these fields are required + type = raw_stripped_event.get("type") + state_key = raw_stripped_event.get("state_key") + sender = raw_stripped_event.get("sender") + content = raw_stripped_event.get("content") + if ( + isinstance(type, str) + and isinstance(state_key, str) + and isinstance(sender, str) + and isinstance(content, dict) + ): + return StrippedStateEvent( + type=type, + state_key=state_key, + sender=sender, + content=content, + ) + + return None + @attr.s(slots=True, frozen=True, auto_attribs=True) class EventMetadata: diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 55891f9b15..c3bc6fa9da 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -37,7 +37,6 @@ from synapse.api.constants import ( CANONICALJSON_MAX_INT, CANONICALJSON_MIN_INT, MAX_PDU_SIZE, - EventTypes, ) from synapse.api.errors import Codes, SynapseError from synapse.logging.opentracing import SynapseTags, set_tag, trace @@ -55,7 +54,7 @@ from synapse.synapse_rust.events import ( from synapse.synapse_rust.types import Requester from synapse.types import JsonDict -from . import EventBase, StrippedStateEvent +from . import EventBase # These are imported only to re-export them (callers import them from this # module); listing them in __all__ stops the unused-import lint flagging them @@ -549,45 +548,9 @@ def strip_event(event: EventBase) -> JsonDict: Stripped state events can only have the `sender`, `type`, `state_key` and `content` properties present. """ - # MSC4311: Ensure the create event is available on invites and knocks. - # TODO: Implement the rest of MSC4311 - if ( - event.room_version.msc4291_room_ids_as_hashes - and event.type == EventTypes.Create - and event.get_state_key() == "" - ): - return event.get_pdu_json() - return { "type": event.type, "state_key": event.state_key, "content": dict(event.content), "sender": event.sender, } - - -def parse_stripped_state_event(raw_stripped_event: Any) -> StrippedStateEvent | None: - """ - Given a raw value from an event's `unsigned` field, attempt to parse it into a - `StrippedStateEvent`. - """ - if isinstance(raw_stripped_event, dict): - # All of these fields are required - type = raw_stripped_event.get("type") - state_key = raw_stripped_event.get("state_key") - sender = raw_stripped_event.get("sender") - content = raw_stripped_event.get("content") - if ( - isinstance(type, str) - and isinstance(state_key, str) - and isinstance(sender, str) - and isinstance(content, dict) - ): - return StrippedStateEvent( - type=type, - state_key=state_key, - sender=sender, - content=content, - ) - - return None diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index e46392944b..214f88e0e1 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -23,6 +23,7 @@ import itertools import logging +from http import HTTPStatus from typing import ( TYPE_CHECKING, AbstractSet, @@ -59,6 +60,7 @@ from synapse.api.room_versions import ( RoomVersions, ) from synapse.events import EventBase, builder, make_event_from_dict +from synapse.events.snapshot import EventContext from synapse.federation.federation_base import ( FederationBase, InvalidEventSignatureError, @@ -70,7 +72,12 @@ from synapse.http.client import is_unknown_endpoint from synapse.http.types import QueryParams from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, tag_args, trace from synapse.metrics import SERVER_NAME_LABEL -from synapse.types import JsonDict, StrCollection, UserID, get_domain_from_id +from synapse.types import ( + JsonDict, + StrCollection, + UserID, + get_domain_from_id, +) from synapse.util.async_helpers import concurrently_execute from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.duration import Duration @@ -1352,12 +1359,12 @@ class FederationClient(FederationBase): self, destination: str, room_id: str, - event_id: str, pdu: EventBase, + context: EventContext, ) -> EventBase: room_version = await self.store.get_room_version(room_id) - content = await self._do_send_invite(destination, pdu, room_version) + content = await self._do_send_invite(destination, pdu, context, room_version) pdu_dict = content["event"] @@ -1378,11 +1385,21 @@ class FederationClient(FederationBase): return pdu async def _do_send_invite( - self, destination: str, pdu: EventBase, room_version: RoomVersion + self, + destination: str, + pdu: EventBase, + context: EventContext, + room_version: RoomVersion, ) -> JsonDict: """Actually sends the invite, first trying v2 API and falling back to v1 API if necessary. + Args: + destination: + pdu: Invite event + context: + room_version: + Returns: The event as a dict as returned by the remote server @@ -1393,6 +1410,19 @@ class FederationClient(FederationBase): """ time_now = self._clock.time_msec() + # MSC4311: For the federation API, format events in `invite_room_state` as full + # PDU's + # + # Find the full events based on the state at the time of the invite + state_ids = await self.store.get_stripped_room_state_ids_from_event_context( + pdu, + context, + ) + state_events = await self.store.get_events(state_ids) + assert set(state_ids) == set(state_events.keys()), ( + "We should have all events available that were set as stripped state." + ) + try: return await self.transport_layer.send_invite_v2( destination=destination, @@ -1401,7 +1431,11 @@ class FederationClient(FederationBase): content={ "event": pdu.get_pdu_json(time_now), "room_version": room_version.identifier, - "invite_room_state": pdu.unsigned.get("invite_room_state", []), + "invite_room_state": [ + # Use full PDU's according to MSC4311 + state_event.get_pdu_json(time_now) + for state_event in state_events.values() + ], }, ) except HttpResponseException as e: @@ -1416,18 +1450,86 @@ class FederationClient(FederationBase): "User's homeserver does not support this room version", Codes.UNSUPPORTED_ROOM_VERSION, ) + # Matrix v1.18 (introduced in MSC4311) says that for a `400` response, "If + # `M_MISSING_PARAM` or `M_INVALID_PARAM` is returned and the request is + # associated with a Client-Server API request, the Client-Server API request + # SHOULD fail with a 5xx error rather than being passed through." (see + # https://spec.matrix.org/v1.18/server-server-api/#put_matrixfederationv2inviteroomideventid) + # + # But this was actually further clarified in MSC4528 that it should be + # translated to a `400` with `M_INCOMPATIBLE_SERVER`. + elif err.code == HTTPStatus.BAD_REQUEST and err.errcode in ( + Codes.MISSING_PARAM, + Codes.INVALID_PARAM, + ): + raise SynapseError( + 400, + f"Invite was rejected by the recipient's server.\n\n" + f"The remote homeserver ({destination}) returned {err.code} {err.errcode} " + "which indicates a compatibility problem between your homeserver and the " + "homeserver you're trying to send the invite to (either one could be at fault).", + # FIXME(MSC4528): Use `M_INCOMPATIBLE_SERVER` stable error code once + # the MSC is merged + Codes.UNKNOWN, + additional_fields={ + "cause": err.msg, + "destination_server": destination, + }, + ) else: raise err # Didn't work, try v1 API. # Note the v1 API returns a tuple of `(200, content)` - _, content = await self.transport_layer.send_invite_v1( - destination=destination, - room_id=pdu.room_id, - event_id=pdu.event_id, - content=pdu.get_pdu_json(time_now), - ) + try: + # Use full PDU's for `invite_room_state` according to MSC4311 + # + # With the v1 invite API, `invite_room_state` is carried inside the event + # instead of a separate field like in v2 so we must munge it in ourselves + event_json = pdu.get_pdu_json(time_now) + event_json.setdefault("unsigned", {})["invite_room_state"] = [ + # Use full PDU's according to MSC4311 + state_event.get_pdu_json(time_now) + for state_event in state_events.values() + ] + + _, content = await self.transport_layer.send_invite_v1( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=event_json, + ) + except HttpResponseException as e: + # Matrix v1.18 (introduced in MSC4311) says that for a `400` response, "If + # `M_MISSING_PARAM` or `M_INVALID_PARAM` is returned and the request is + # associated with a Client-Server API request, the Client-Server API request + # SHOULD fail with a 5xx error rather than being passed through." (see + # https://spec.matrix.org/v1.18/server-server-api/#put_matrixfederationv2inviteroomideventid) + # + # But this was actually further clarified in MSC4528 that it should be + # translated to a `400` with `M_INCOMPATIBLE_SERVER`. + err = e.to_synapse_error() + if err.code == HTTPStatus.BAD_REQUEST and err.errcode in ( + Codes.MISSING_PARAM, + Codes.INVALID_PARAM, + ): + raise SynapseError( + 400, + f"Invite was rejected by the recipient's server.\n\n" + f"The remote homeserver ({destination}) returned {err.code} {err.errcode} " + "which indicates a compatibility problem between your homeserver and the " + "homeserver you're trying to send the invite to (either one could be at fault).", + # FIXME(MSC4528): Use `M_INCOMPATIBLE_SERVER` stable error code once + # the MSC is merged + Codes.UNKNOWN, + additional_fields={ + "cause": err.msg, + "destination_server": destination, + }, + ) + else: + raise err return content async def send_leave(self, destinations: Iterable[str], pdu: EventBase) -> None: diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index b90bb67ba2..7a582ca255 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -198,8 +198,6 @@ class FederationServer(FederationBase): hs.config.federation.federation_metrics_domains ) - self._room_prejoin_state_types = hs.config.api.room_prejoin_state - # Whether we have started handling old events in the staging area. self._started_handling_of_staged_events = False @@ -235,12 +233,23 @@ class FederationServer(FederationBase): # We pause a bit so that we don't start handling all rooms at once. await self._clock.sleep(Duration(seconds=random.uniform(0, 0.1))) + async def _assert_not_state_dag_room(self, room_id: str, endpoint: str) -> None: + room_version = await self.store.get_room_version(room_id) + if room_version.msc4242_state_dags: + # Not 5xx else the server will back off from us + raise SynapseError( + 400, + f"{endpoint} is not implemented for MSC4242 state DAG rooms", + errcode=Codes.UNRECOGNIZED, + ) + async def on_backfill_request( self, origin: str, room_id: str, versions: list[str], limit: int ) -> tuple[int, dict[str, Any]]: async with self._server_linearizer.queue((origin, room_id)): origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) + await self._assert_not_state_dag_room(room_id, "/backfill") pdus = await self.handler.on_backfill_request( origin, room_id, versions, limit @@ -656,6 +665,7 @@ class FederationServer(FederationBase): await self._event_auth_handler.assert_host_in_room(room_id, origin) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) + await self._assert_not_state_dag_room(room_id, "/state") # we grab the linearizer to protect ourselves from servers which hammer # us. In theory we might already have the response to this query @@ -696,6 +706,7 @@ class FederationServer(FederationBase): await self._event_auth_handler.assert_host_in_room(room_id, origin) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) + await self._assert_not_state_dag_room(room_id, "/state_ids") resp = await self._state_ids_resp_cache.wrap( (room_id, event_id), @@ -817,8 +828,27 @@ class FederationServer(FederationBase): return {"event": pdu.get_templated_pdu_json(), "room_version": room_version} async def on_invite_request( - self, origin: str, content: JsonDict, room_version_id: str + self, + *, + origin: str, + expected_room_id: str, + expected_event_id: str, + event_json: JsonDict, + room_version_id: str, ) -> dict[str, Any]: + """ + Args: + origin: + expected_room_id: The room ID specified in the + `/_matrix/federation/v1/invite/{roomId}/{eventId}` request that we expect to + match in the actual event itself. + expected_event_id: The event ID specified in the + `/_matrix/federation/v1/invite/{roomId}/{eventId}` request that we expect to + match in the actual event itself. + event_json: + room_version_id: + """ + room_version = KNOWN_ROOM_VERSIONS.get(room_version_id) if not room_version: raise SynapseError( @@ -827,9 +857,21 @@ class FederationServer(FederationBase): Codes.UNSUPPORTED_ROOM_VERSION, ) - pdu = event_from_pdu_json(content, room_version) + pdu = event_from_pdu_json(event_json, room_version) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, pdu.room_id) + if pdu.event_id != expected_event_id: + raise SynapseError( + 400, + "Invite event ID must match event ID specified in the federation `/invite` request", + Codes.INVALID_PARAM, + ) + if pdu.room_id != expected_room_id: + raise SynapseError( + 400, + "The room_id specified in the invite event must match room ID specified in the federation `/invite` request", + Codes.INVALID_PARAM, + ) if await self._spam_checker_module_callbacks.should_drop_federated_event(pdu): logger.info( "Federated event contains spam, dropping %s", @@ -842,7 +884,11 @@ class FederationServer(FederationBase): errmsg = f"event id {pdu.event_id}: {e}" logger.warning("%s", errmsg) raise SynapseError(403, errmsg, Codes.FORBIDDEN) - ret_pdu = await self.handler.on_invite_request(origin, pdu, room_version) + ret_pdu = await self.handler.on_invite_request( + origin=origin, + event=pdu, + room_version=room_version, + ) time_now = self._clock.time_msec() return {"event": ret_pdu.get_pdu_json(time_now)} @@ -1061,19 +1107,31 @@ class FederationServer(FederationBase): Returns: The stripped room state. """ - _, context = await self._on_send_membership_event( + time_now = self._clock.time_msec() + + event, context = await self._on_send_membership_event( origin, content, Membership.KNOCK, room_id ) - # Retrieve stripped state events from the room and send them back to the remote - # server. This will allow the remote server's clients to display information - # related to the room while the knock request is pending. - stripped_room_state = ( - await self.store.get_stripped_room_state_from_event_context( - context, self._room_prejoin_state_types - ) + # MSC4311: For the federation API, format events in `knock_room_state` as full + # PDU's + # + # Find the full events based on the state at the time of the knock + state_ids = await self.store.get_stripped_room_state_ids_from_event_context( + event, context ) - return {"knock_room_state": stripped_room_state} + state_events = await self.store.get_events(state_ids) + assert set(state_ids) == set(state_events.keys()), ( + "We should have all events available that were set as stripped state." + ) + + return { + "knock_room_state": [ + # Use full PDU's according to MSC4311 + state_event.get_pdu_json(time_now) + for state_event in state_events.values() + ] + } async def _on_send_membership_event( self, origin: str, content: JsonDict, membership_type: str, room_id: str @@ -1109,6 +1167,14 @@ class FederationServer(FederationBase): # Note that get_room_version throws if the room does not exist here. room_version = await self.store.get_room_version(room_id) + if room_version.msc4242_state_dags and membership_type != Membership.JOIN: + # 4xx rather than 5xx, so the caller does not back off from us + raise SynapseError( + 400, + f"/send_{membership_type} is not implemented for MSC4242 state DAG rooms", + errcode=Codes.UNRECOGNIZED, + ) + if await self.store.is_partial_state_room(room_id): # If our server is still only partially joined, we can't give a complete # response to /send_join, /send_knock or /send_leave. @@ -1202,6 +1268,7 @@ class FederationServer(FederationBase): await self._event_auth_handler.assert_host_in_room(room_id, origin) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) + await self._assert_not_state_dag_room(room_id, "/event_auth") time_now = self._clock.time_msec() auth_pdus = await self.handler.on_event_auth(event_id, room_id) @@ -1287,6 +1354,7 @@ class FederationServer(FederationBase): origin, room_id, earliest_events, latest_events, limit ) else: + await self._assert_not_state_dag_room(room_id, "/get_missing_events") missing_events = await self.handler.on_get_missing_events( origin, room_id, earliest_events, latest_events, limit ) diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index af6c733029..6d2f3f0e43 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -491,7 +491,11 @@ class FederationV1InviteServlet(BaseFederationServerServlet): # state resolution algorithm, and we don't use that for processing # invites result = await self.handler.on_invite_request( - origin, content, room_version_id=RoomVersions.V1.identifier + origin=origin, + expected_room_id=room_id, + expected_event_id=event_id, + event_json=content, + room_version_id=RoomVersions.V1.identifier, ) # V1 federation API is defined to return a content of `[200, {...}]` @@ -513,9 +517,6 @@ class FederationV2InviteServlet(BaseFederationServerServlet): room_id: str, event_id: str, ) -> tuple[int, JsonDict]: - # TODO(paul): assert that room_id/event_id parsed from path actually - # match those given in content - room_version = content["room_version"] event = content["event"] invite_room_state = content.get("invite_room_state", []) @@ -524,12 +525,15 @@ class FederationV2InviteServlet(BaseFederationServerServlet): invite_room_state = [] # Synapse expects invite_room_state to be in unsigned, as it is in v1 - # API - + # API. We will sanitize this inside `on_invite_request(...)` event.setdefault("unsigned", {})["invite_room_state"] = invite_room_state result = await self.handler.on_invite_request( - origin, event, room_version_id=room_version + origin=origin, + expected_room_id=room_id, + expected_event_id=event_id, + event_json=event, + room_version_id=room_version, ) # We only store invite_room_state for internal use, so remove it before diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 68b8aa71f1..8d82a98559 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -374,13 +374,21 @@ class ApplicationServicesHandler: # follow the base stream position. new_token = MultiWriterStreamToken(stream=new_token.stream) - events = await self._handle_receipts(service, new_token) - self.scheduler.enqueue_for_appservice(service, ephemeral=events) + while True: + events, reached_token = await self._handle_receipts( + service, new_token + ) + self.scheduler.enqueue_for_appservice( + service, ephemeral=events + ) - # Persist the latest handled stream token for this appservice - await self.store.set_appservice_stream_type_pos( - service, "read_receipt", new_token.stream - ) + # Persist the latest handled stream token for this appservice + await self.store.set_appservice_stream_type_pos( + service, "read_receipt", reached_token.stream + ) + + if reached_token.stream >= new_token.stream: + break elif stream_key == StreamKeyType.PRESENCE: assert isinstance(new_token, int) @@ -460,14 +468,16 @@ class ApplicationServicesHandler: async def _handle_receipts( self, service: ApplicationService, new_token: MultiWriterStreamToken - ) -> list[JsonMapping]: + ) -> tuple[list[JsonMapping], MultiWriterStreamToken]: """ - Return the latest read receipts that the given application service should receive. + Return the next batch of read receipts that the given application service + should receive. - First fetch all read receipts between the last receipt stream token that this - application service should have previously received (non-inclusive) and the - latest read receipt stream token (inclusive). Then from that set, return only - those read receipts that the given application service may be interested in. + First fetch the oldest read receipts between the last receipt stream token that + this application service should have previously received (non-inclusive) and + the latest read receipt stream token (inclusive), up to a cap. Then from that + set, return only those read receipts that the given application service may be + interested in. Args: service: The application service to check for which events it should receive. @@ -476,23 +486,40 @@ class ApplicationServicesHandler: token. Prevents accidentally duplicating work. Returns: - A list of JSON dictionaries containing data derived from the read receipts that - should be sent to the given application service. + A two-tuple containing the following: + * A list of JSON dictionaries containing data derived from the read + receipts that should be sent to the given application service. + * The receipt stream token up to which receipts were actually handled. + This is earlier than `new_token` if the fetch was truncated; callers + must call this method again to fetch the remaining receipts. """ from_key = await self.store.get_type_stream_id_for_appservice( service, "read_receipt" ) - if new_token is not None and new_token.stream <= from_key: - logger.debug("Rejecting token lower than or equal to stored: %s", new_token) - return [] + if new_token.stream <= from_key: + if new_token.stream == from_key: + logger.debug( + "Receipt token %s already handled for appservice %s", + new_token, + service.id, + ) + else: + logger.warning( + "Receipt token %s is behind stored position %s for appservice %s", + new_token, + from_key, + service.id, + ) + + return [], MultiWriterStreamToken(stream=from_key) from_token = MultiWriterStreamToken(stream=from_key) receipts_source = self.event_sources.sources.receipt - receipts, _ = await receipts_source.get_new_events_as( + receipts, reached_token = await receipts_source.get_new_events_as( service=service, from_key=from_token, to_key=new_token ) - return receipts + return receipts, reached_token async def _handle_presence( self, diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index fc4373e0a5..47f83d4aad 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -486,7 +486,11 @@ class AuthHandler: """ sid: str | None = None - authdict = clientdict.pop("auth", {}) + # `auth` is optional, so an explicit `null` means the same as `{}`, ie. no + # auth stage attempted yet. + authdict = clientdict.pop("auth", None) + if authdict is None: + authdict = {} if "session" in authdict: sid = authdict["session"] @@ -647,7 +651,7 @@ class AuthHandler: not send a session ID, returns None. """ sid = None - if clientdict and "auth" in clientdict: + if clientdict and clientdict.get("auth") is not None: authdict = clientdict["auth"] if "session" in authdict: sid = authdict["session"] diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 34596ade16..a1aa2a51dc 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -27,7 +27,13 @@ from synapse.api.errors import SynapseError from synapse.replication.http.deactivate_account import ( ReplicationNotifyAccountDeactivatedServlet, ) -from synapse.types import Codes, Requester, UserID, create_requester +from synapse.types import ( + Codes, + Requester, + UserID, + create_requester, + get_localpart_from_id, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -47,6 +53,7 @@ class DeactivateAccountHandler: self._room_member_handler = hs.get_room_member_handler() self._identity_handler = hs.get_identity_handler() self._profile_handler = hs.get_profile_handler() + self._delayed_events_handler = hs.get_delayed_events_handler() self._pusher_pool = hs.get_pusherpool() self.user_directory_handler = hs.get_user_directory_handler() self._server_name = hs.hostname @@ -109,6 +116,12 @@ class DeactivateAccountHandler: id_server, ) + # Cancel the user's delayed events (MSC4140). Do this first, so that none + # of them get sent while the rest of the deactivation is in progress. + await self._delayed_events_handler.cancel_all_for_user( + get_localpart_from_id(user_id) + ) + # FIXME: Theoretically there is a race here wherein user resets # password using threepid. diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index ac2da15112..c86194f77c 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -28,6 +28,7 @@ from synapse.logging.opentracing import set_tag from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions from synapse.replication.http.delayed_events import ( ReplicationAddedDelayedEventRestServlet, + ReplicationCancelDelayedEventsForUserRestServlet, ) from synapse.storage.databases.main.delayed_events import ( DelayedEventDetails, @@ -129,6 +130,9 @@ class DelayedEventsHandler: ) else: self._repl_client = ReplicationAddedDelayedEventRestServlet.make_client(hs) + self._cancel_all_for_user_client = ( + ReplicationCancelDelayedEventsForUserRestServlet.make_client(hs) + ) @property def _is_master(self) -> bool: @@ -440,6 +444,30 @@ class DelayedEventsHandler: if self._next_send_ts_changed(next_send_ts): self._schedule_next_at_or_none(next_send_ts) + async def cancel_all_for_user(self, user_localpart: str) -> None: + """ + Cancels the scheduled delivery of all delayed events owned by the local user + with the given localpart, e.g. because their account is being deactivated. + + Delayed events that are already being sent are left alone. + + Goes through replication if this is not the main process, as only the + main process handles sending delayed events. + """ + if not self._is_master: + await self._cancel_all_for_user_client( + instance_name=MAIN_PROCESS_INSTANCE_NAME, + user_localpart=user_localpart, + ) + return + + await make_deferred_yieldable(self._initialized_from_db) + + next_send_ts = await self._store.cancel_delayed_events_for_user(user_localpart) + + if self._next_send_ts_changed(next_send_ts): + self._schedule_next_at_or_none(next_send_ts) + async def restart(self, request: SynapseRequest, delay_id: str) -> None: """ Restarts the scheduled delivery of the matching delayed event. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3558e0bd6b..2e5cc7c688 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -30,15 +30,19 @@ from http import HTTPStatus from typing import ( TYPE_CHECKING, AbstractSet, + Any, Iterable, ) import attr -from prometheus_client import Histogram +from prometheus_client import Counter, Histogram from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json +from typing_extensions import assert_never from unpaddedbase64 import decode_base64 +from twisted.internet.defer import CancelledError + from synapse import event_auth from synapse.api.constants import ( MAX_DEPTH, @@ -64,10 +68,14 @@ from synapse.api.errors import ( from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.crypto.event_signing import compute_event_signature from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase, event_exists_in_state_dag +from synapse.events import EventBase, StrippedStateEvent, event_exists_in_state_dag from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import EventContext, UnpersistedEventContextBase from synapse.events.validator import EventValidator +from synapse.federation.federation_base import ( + InvalidEventSignatureError, + event_from_pdu_json, +) from synapse.federation.federation_client import InvalidResponseError from synapse.handlers.pagination import PURGE_PAGINATION_LOCK_NAME from synapse.http.servlet import assert_params_in_dict @@ -79,7 +87,7 @@ from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.storage.invite_rule import InviteRule from synapse.types import JsonDict, StrCollection, get_domain_from_id from synapse.types.state import StateFilter -from synapse.util.async_helpers import Linearizer +from synapse.util.async_helpers import Linearizer, concurrently_execute from synapse.util.duration import Duration from synapse.util.retryutils import NotRetryingDestination from synapse.visibility import filter_events_for_server @@ -112,6 +120,14 @@ backfill_processing_before_timer = Histogram( ), ) +# So we have some sort of measure of Matrix ecosystem adoption so we know better how +# safe it is to make the migration to strict validation after 2027-06-01. +invalid_stripped_state_counter = Counter( + "synapse_federation_saw_invalid_stripped_state", + "The number of invites/knocks we saw with invalid stripped state.", + labelnames=[SERVER_NAME_LABEL], +) + NUMBER_OF_EVENTS_TO_BACKFILL = 100 """ @@ -135,6 +151,22 @@ class _BackfillPoint: type: _BackfillPointType +class InvalidStrippedStateBehaviour(Enum): + """ + What to do when parsing stripped state events + """ + + remove_invalid = enum.auto() + """ + Remove the individual event from the set of stripped state. + """ + + reject_all = enum.auto() + """ + Reject the entire set of stripped state events if there is a single invalid event + """ + + class FederationHandler: """Handles general incoming federation requests @@ -222,6 +254,9 @@ class FederationHandler: Returns: True if we actually tried to backfill something, otherwise False. """ + if (await self.store.get_room_version(room_id)).msc4242_state_dags: + return False + # Starting the processing time here so we can include the room backfill # linearizer lock queue in the timing processing_start_time = self.clock.time_msec() if record_time else 0 @@ -573,7 +608,9 @@ class FederationHandler: return False - async def send_invite(self, target_host: str, event: EventBase) -> EventBase: + async def send_invite( + self, target_host: str, event: EventBase, context: EventContext + ) -> EventBase: """Sends the invite to the remote server for signing. Invites must be signed by the invitee's server before distribution. @@ -582,8 +619,8 @@ class FederationHandler: pdu = await self.federation_client.send_invite( destination=target_host, room_id=event.room_id, - event_id=event.event_id, pdu=event, + context=context, ) except RequestSendFailed: raise SynapseError(502, f"Can't connect to server {target_host}") @@ -950,15 +987,53 @@ class FederationHandler: # This is a bit of a hack and is cribbing off of invites. Basically we # store the room state here and retrieve it again when this event appears # in the invitee's sync stream. It is stripped out for all other local users. - stripped_room_state = knock_response.get("knock_room_state") - - if stripped_room_state is None: - raise KeyError("Missing 'knock_room_state' field in send_knock response") - - if not isinstance(stripped_room_state, list): - raise TypeError("'knock_room_state' has wrong type") - - event.unsigned["knock_room_state"] = stripped_room_state + # + # Parse/validate `knock_room_state` + try: + stripped_room_state = await self._parse_stripped_room_state( + stripped_room_state=knock_response.get("knock_room_state"), + room_id=event.room_id, + room_version=event_format_version, + # According to Matrix v1.18 (introduced in MSC4311) for the `PUT + # /_matrix/federation/v1/send_knock/{roomId}/{eventId}` endpoint: + # > Entries which are improperly signed or formatted SHOULD be removed by the + # > server prior to supplying them over the Client-Server API. + # + # FIXME(MSC4311): This should be + # `InvalidStrippedStateBehaviour.remove_invalid` but that means this + # never raises and we'd always assign empty `knock_room_state` for some + # server that's still using the legacy stripped-state format. After + # 2027-06-01 (to allow some time for the ecosystem to adapt and support + # MSC4311), update this to be + # `InvalidStrippedStateBehaviour.remove_invalid` alongside removing the + # whole `_minimal_parse_stripped_room_state` fallback. + invalid_stripped_state_behaviour=InvalidStrippedStateBehaviour.reject_all, + ) + # Replace with our sanitized `knock_room_state` + event.unsigned["knock_room_state"] = [ + stripped_state_event.as_json_dict() + for stripped_state_event in stripped_room_state + ] + except CancelledError: + raise + except Exception: + # FIXME(MSC4311): Remove this whole block after we always enforce the + # validation above. The only reason this is here is because the validation + # can fail for non-compliant servers but we should still use stripped state. + stripped_room_state_for_client = self._minimal_parse_stripped_room_state( + stripped_room_state=knock_response.get("knock_room_state"), + # According to Matrix v1.18 (introduced in MSC4311) for the `PUT + # /_matrix/federation/v1/send_knock/{roomId}/{eventId}` endpoint: + # > Entries which are improperly signed or formatted SHOULD be removed by the + # > server prior to supplying them over the Client-Server API. + invalid_stripped_state_behaviour=InvalidStrippedStateBehaviour.remove_invalid, + ) + if stripped_room_state_for_client is not None: + # Replace with our sanitized `knock_room_state` + event.unsigned["knock_room_state"] = [ + stripped_state_event.as_json_dict() + for stripped_state_event in stripped_room_state_for_client + ] context = EventContext.for_outlier(self._storage_controllers) stream_id = await self._federation_event_handler.persist_events_and_notify( @@ -1125,8 +1200,260 @@ class FederationHandler: await self._event_auth_handler.check_auth_rules_from_context(event) return event + def _minimal_parse_stripped_room_state( + self, + *, + stripped_room_state: Any, + invalid_stripped_state_behaviour: InvalidStrippedStateBehaviour, + ) -> list[StrippedStateEvent] | None: + """ + The goal of this function is to sanitize whatever we got from federation and + make it presentable to the client. The minimum amount of parsing necessary to + ensure `invite_room_state`/`knock_room_state` is at-least a list of stripped + state events (compared to `_parse_stripped_room_state`). + + Args: + stripped_room_state: The raw `invite_room_state`/`knock_room_state` JSON + invalid_stripped_state_behaviour: How to handle the scenario where we see a + single invalid stripped state event + """ + parsed_stripped_room_state: list[StrippedStateEvent] = [] + + # Scrutinize JSON values + # + # In previous versions of the Matrix spec, + # `invite_room_state`/`knock_room_state` was an optional list of stripped state + # events which means we can't strictly enforce that this is always present. + if stripped_room_state is None: + return None + # We're going to strictly enforce that they at-least gave us a list. + elif not isinstance(stripped_room_state, list): + if ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.reject_all + ): + raise TypeError("Stripped state must be a list of PDU's") + elif ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.remove_invalid + ): + return parsed_stripped_room_state + else: + assert_never(invalid_stripped_state_behaviour) + + for raw_stripped_event in stripped_room_state: + # Parse each stripped event + parsed_stripped_event = StrippedStateEvent.from_json_dict( + raw_stripped_event + ) + if parsed_stripped_event is None: + # Drop any invalid stripped state events as this is spec'ed and we + # might as well save the client from dealing with anything crazy. + continue + parsed_stripped_room_state.append(parsed_stripped_event) + + return parsed_stripped_room_state + + async def _parse_stripped_room_state( + self, + *, + stripped_room_state: Any, + room_id: str, + room_version: RoomVersion, + invalid_stripped_state_behaviour: InvalidStrippedStateBehaviour, + ) -> list[StrippedStateEvent]: + """ + Parse and validate `invite_room_state`/`knock_room_state` according to the + Matrix spec (c.f. MSC4311). + + According to Matrix v1.18 (introduced in MSC4311) for the `PUT + /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + > The `invite_room_state` has additional validation, which servers MAY apply to + > room versions 1 through 11 and SHOULD apply to all other room versions. As with + > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + > - [...] + > - One or more entries in `invite_room_state` are not formatted according to the room's version. + > - One or more events fails a signature check. + > - One or more events does not reside in the same room as the invite. + + According to Matrix v1.18 (introduced in MSC4311) for the `PUT + /_matrix/federation/v1/send_knock/{roomId}/{eventId}` endpoint: + > Entries which are improperly signed or formatted SHOULD be removed by the + > server prior to supplying them over the Client-Server API. + + We refer to `invite_room_state`/`knock_room_state` as `stripped_room_state` but + the events contained within can be full PDU's or stripped state events (older + version of the Matrix spec). + + Args: + stripped_room_state: The raw `invite_room_state`/`knock_room_state` JSON + room_id: The room ID the invite/knock is happening in + room_version: The version of the room the invite/knock is happening in + invalid_stripped_state_behaviour: How to handle the scenario where we see a + single invalid stripped state event + + Returns: + A list of parsed `StrippedStateEvent` + + Raises: + When `InvalidStrippedStateBehaviour.reject_all`, raises + `TypeError`/`ValueError` when one of the stripped room state events is + invalid + """ + parsed_stripped_room_state: list[StrippedStateEvent] = [] + + # Scrutinize JSON values + if not isinstance(stripped_room_state, list): + invalid_stripped_state_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(1) + + if ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.reject_all + ): + raise TypeError( + "Stripped state must be a list of PDU's that includes the `m.room.create` event" + ) + elif ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.remove_invalid + ): + return parsed_stripped_room_state + else: + assert_never(invalid_stripped_state_behaviour) + + # First assemble an ordered list of PDUs (remember the order of the stripped + # state) as we want to concurrently process below and then re-assemble in the + # original order. + pdus: list[EventBase] = [] + already_counted_fail_metric = False + for raw_stripped_event in stripped_room_state: + try: + # Scrutinize JSON values + if not isinstance(raw_stripped_event, dict): + raise ValueError( + "PDU from stripped state must be a JSON dictionary" + ) + + # Validate PDU + try: + pdu = event_from_pdu_json(raw_stripped_event, room_version) + pdus.append(pdu) + except Exception as exc: + raise ValueError( + "Unable to parse one of the stripped state events as a PDU" + ) from exc + except ValueError as exc: + # Count how many times we saw invalid stripped state. We're counting the + # number of times we saw invalid state overall, not each individual + # invalid stripped state event. + if not already_counted_fail_metric: + invalid_stripped_state_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(1) + already_counted_fail_metric = True + + # React to invalid event + if ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.reject_all + ): + raise exc + elif ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.remove_invalid + ): + continue + else: + assert_never(invalid_stripped_state_behaviour) + + # Because we're processing things in parallel, we need to worry about assembling + # things in the correct order after we're done. + # + # Map from `event_id` to `StrippedStateEvent` + parsed_stripped_room_state_map: dict[str, StrippedStateEvent] = {} + + async def _validate_stripped_state_pdu_event(pdu: EventBase) -> None: + nonlocal already_counted_fail_metric + + try: + # Validate that it's from the same room + if pdu.room_id != room_id: + raise ValueError( + "PDU from stripped state must be from the room ID specified in the request" + ) + # Validate signature/hashes + try: + # Note: `_check_sigs_and_hash` will redact the event in the case of + # a hash mismatch (proper thing to use according to the spec)) so + # let's be sure to use the new updated PDU returned from this call. + pdu = await self.federation_client._check_sigs_and_hash( + room_version, pdu + ) + except InvalidEventSignatureError as exc: + raise ValueError( + "PDU from stripped state must pass signature/hash checks" + ) from exc + + # Parse the stripped events to ensure it has all of the fields necessary + parsed_stripped_event = StrippedStateEvent.from_json_dict( + pdu.get_dict() + ) + if parsed_stripped_event is None: + raise ValueError("Unable to parse as stripped event") + parsed_stripped_room_state_map[pdu.event_id] = parsed_stripped_event + except ValueError as exc: + # Count how many times we saw invalid stripped state. We're counting the + # number of times we saw invalid state overall, not each individual + # invalid stripped state event. + if not already_counted_fail_metric: + invalid_stripped_state_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(1) + already_counted_fail_metric = True + + # React to invalid event + if ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.reject_all + ): + raise exc + elif ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.remove_invalid + ): + pass + else: + assert_never(invalid_stripped_state_behaviour) + + # Check in parallel as `_check_sigs_and_hash` can take some time if we don't + # already have the server signatures. + await concurrently_execute( + _validate_stripped_state_pdu_event, + pdus, + # Arbitrary concurrency + 1000, + ) + + # Create an ordered list that matches the original `stripped_room_state` + parsed_stripped_room_state = [ + parsed_stripped_room_state_map[pdu.event_id] + # Iterate in the original order + for pdu in pdus + # But check to make sure we it ended up being parsed correctly as a stripped + # event + if pdu.event_id in parsed_stripped_room_state_map + ] + + return parsed_stripped_room_state + async def on_invite_request( - self, origin: str, event: EventBase, room_version: RoomVersion + self, + *, + origin: str, + event: EventBase, + room_version: RoomVersion, ) -> EventBase: """We've got an invite event. Process and persist it. Sign it. @@ -1135,6 +1462,14 @@ class FederationHandler: if event.state_key is None: raise SynapseError(400, "The invite event did not have a state key") + if room_version.msc4242_state_dags: + # 4xx rather than 5xx, so the caller does not back off from us + raise SynapseError( + 400, + "/invite is not implemented for MSC4242 state DAG rooms", + errcode=Codes.UNRECOGNIZED, + ) + is_blocked = await self.store.is_room_blocked(event.room_id) if is_blocked: raise SynapseError(403, "This room has been blocked on this server") @@ -1199,6 +1534,89 @@ class FederationHandler: room_id=event.room_id, room_version=room_version ) + # Parse/validate `invite_room_state` + try: + stripped_room_state = await self._parse_stripped_room_state( + stripped_room_state=event.unsigned.get("invite_room_state"), + room_id=event.room_id, + room_version=room_version, + # According to Matrix v1.18 (introduced in MSC4311) for the `PUT + # /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + # > The `invite_room_state` has additional validation, which servers MAY apply to + # > room versions 1 through 11 and SHOULD apply to all other room versions. As with + # > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + # > - The `m.room.create` event is missing from `invite_room_state`. + # > - One or more entries in `invite_room_state` are not formatted according to the room's version. + # > - One or more events fails a signature check. + # > - One or more events does not reside in the same room as the invite. + invalid_stripped_state_behaviour=InvalidStrippedStateBehaviour.reject_all, + ) + # Validate `m.room.create` event is included (see spec blurb above) + includes_create_event = any( + (stripped_state_event.type, stripped_state_event.state_key) + == (EventTypes.Create, "") + for stripped_state_event in stripped_room_state + ) + if not includes_create_event: + invalid_stripped_state_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(1) + raise ValueError( + "Stripped state must include `m.room.create` event (MSC4311)" + ) + # Replace with our sanitized `invite_room_state` + event.unsigned["invite_room_state"] = [ + stripped_state_event.as_json_dict() + for stripped_state_event in stripped_room_state + ] + except CancelledError: + raise + except Exception as exc: + # FIXME(MSC4311): Apply this validation for all room versions after + # 2027-06-01 (to allow some time for the ecosystem to adapt and support + # MSC4311), see https://github.com/element-hq/synapse/issues/19943 + # + # The Matrix spec says that for version 12+ rooms, servers SHOULD rather than + # MAY respond to such requests with 400 M_INVALID_PARAM. Given we have the + # lee-way to enforce this in all room versions, we might as well. + # + # For now, we'll only log in room versions 12+ where this SHOULD be the case + # already. + if room_version.msc4311_stripped_state: + # FIXME(MSC4311): Instead of logging, reject with 400 `M_INVALID_PARAM` + # after 2027-06-01. Given Synapse claimed to support room version 12 but + # didn't adhere to this behavior until 2026-07, we will only warn for + # now. + logger.warning( + "Continuing anyway but failed to validate `invite_room_state` on invite %s (room_version=%s): %s", + event, + room_version, + exc, + ) + + # FIXME(MSC4311): Remove this whole block after we always enforce the + # validation above. The only reason this is here is because the validation + # can fail for non-compliant servers but we should still use stripped state. + stripped_room_state_for_client = self._minimal_parse_stripped_room_state( + stripped_room_state=event.unsigned.get("invite_room_state"), + # According to Matrix v1.18 (introduced in MSC4311) for the `PUT + # /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + # > The `invite_room_state` has additional validation, which servers MAY apply to + # > room versions 1 through 11 and SHOULD apply to all other room versions. As with + # > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + # > - The `m.room.create` event is missing from `invite_room_state`. + # > - One or more entries in `invite_room_state` are not formatted according to the room's version. + # > - One or more events fails a signature check. + # > - One or more events does not reside in the same room as the invite. + invalid_stripped_state_behaviour=InvalidStrippedStateBehaviour.reject_all, + ) + if stripped_room_state_for_client is not None: + # Replace with our sanitized `invite_room_state` + event.unsigned["invite_room_state"] = [ + stripped_state_event.as_json_dict() + for stripped_state_event in stripped_room_state_for_client + ] + event.internal_metadata.outlier = True event.internal_metadata.out_of_band_membership = True diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 55a9f5a647..6665be9086 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -558,7 +558,11 @@ class FederationEventHandler: raise SynapseError(400, "Room version mismatch") if room_version.msc4242_state_dags and not is_state_dag_connected( - [ev for ev in state if supports_msc4242_state_dag(ev)] + [ + ev + for ev in itertools.chain(state, [event]) + if supports_msc4242_state_dag(ev) + ] ): # We should have been given a connected state DAG. If there is a gap in it we # cannot calculate the state, so the response is invalid and we refuse the join. @@ -584,10 +588,19 @@ class FederationEventHandler: from_send_join=True, ) if room_version.msc4242_state_dags and has_rejected_events: - # The state DAG must not include rejected events - raise SynapseError( - 502, - "Unable to join because the remote server passed back a state DAG that includes rejected events (invalid)", + # A conformant server excludes rejected events from the state DAG it sends to + # new joiners, so their presence means the remote server is faulty. We log + # that rather than refusing the join: rejected events are excluded from the + # state DAG forward extremities, so they cannot contribute to the room state + # we calculate, and any event referencing one is rejected in turn. Refusing + # would cost availability without buying any safety. + logger.warning( + "Remote server %s sent a state DAG containing rejected events when we " + "joined %s. This is a bug in that server: rejected events must be " + "excluded from /send_join responses. Joining anyway; the rejected events " + "form no part of the room state.", + origin, + room_id, ) # and now persist the join event itself. @@ -898,6 +911,14 @@ class FederationEventHandler: backfilled: True if this is part of a historical batch of events (inhibits notification to clients, and validation of device keys.) """ + if not events: + return + + # Our callers only ever pass events for a single room. + first_event = next(iter(events)) + room_id = first_event.room_id + is_state_dag_room = first_event.room_version.msc4242_state_dags + set_tag( SynapseTags.FUNC_ARG_PREFIX + "event_ids", str([event.event_id for event in events]), @@ -932,6 +953,7 @@ class FederationEventHandler: new_events: list[EventBase] = [] for event in events: + assert event.room_id == room_id event_id = event.event_id # If we've already seen this event ID... @@ -967,10 +989,50 @@ class FederationEventHandler: # ascending) because one backfill event is likely to be the `prev_event` of # the next event we're going to process. sorted_events = sorted(new_events, key=lambda x: x.depth) + + # Backfilled events are persisted without recalculating the current state, + # which is right for the room DAG but not always for the state DAG. Backfill + # can teach us about state events we have never seen, which then belong on + # the state DAG frontier and so do change the current state. That happens + # when an event's `prev_events` reach state events which its + # `prev_state_events` do not, because the state DAG walk in + # /get_missing_events only ever follows `prev_state_events`. + # This is defence-in-depth: prev_events on honest servers is a superset of the state + # events in that event's prev_state_events, and only when they disagree (e.g. refer to + # an event on a fork) will this occur, but we have to handle the adversarial case too. + # + # Remember the frontier so that we can recalculate the current state if it moves. + state_dag_extremities_before: frozenset[str] | None = None + if ( + backfilled # Non-backfilled pulled events will automatically update the current state, this check is only for /backfill + and sorted_events + and is_state_dag_room + ): + state_dag_extremities_before = ( + await self._store.get_state_dag_extremities(room_id) + ) + for ev in sorted_events: with nested_logging_context(ev.event_id): await self._process_pulled_event(origin, ev, backfilled=backfilled) + if state_dag_extremities_before is not None: + state_dag_extremities_after = ( + await self._store.get_state_dag_extremities(room_id) + ) + if state_dag_extremities_after != state_dag_extremities_before: + # This is unusual because it implies the graph induced by prev_events vs prev_state_events + # has diverged, so warn about it. + logger.warning( + "Backfill from %s moved the state DAG forward extremities of %s from %s " + "to %s, recalculating the current state", + origin, + room_id, + shortstr(state_dag_extremities_before), + shortstr(state_dag_extremities_after), + ) + await self._state_handler.update_current_state(room_id) + # Check if we've already tried to process these events at some point in the # past. We aren't concerned with the expontntial backoff here, just whether it # has failed to be processed before. @@ -1205,6 +1267,15 @@ class FederationEventHandler: if not missing_prevs: return await self._state_handler.compute_event_context(event) + if event.room_version.msc4242_state_dags: + raise FederationError( + "ERROR", + 500, + "Fetching the state at missing prev_events is not implemented for " + "MSC4242 state DAG rooms", + affected=event_id, + ) + logger.info( "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Event in room %s is missing prev_events %s: " "calculating state for a backwards extremity", @@ -2063,14 +2134,47 @@ class FederationEventHandler: # # This is just an optimisation, so it doesn't need to be watertight - the event # persister does another round of deduplication. + has_rejected_events = False seen_remotes = await self._store.have_seen_events(room_id, event_map.keys()) if is_state_dag_room: # Pretend we haven't seen events which lack state groups. Events which lack state groups # in state DAG rooms are exclusively out-of-band memberships. This ensures we calculate # state for out-of-band events we have previously persisted. seen_remotes = await self._store.get_events_with_state_groups(seen_remotes) - for s in seen_remotes: - event_map.pop(s, None) + unseen_event_map = dict(event_map) + if seen_remotes: + for s in seen_remotes: + unseen_event_map.pop(s, None) + if is_state_dag_room: + rejected_event_ids = await self._store.has_rejected_event_ids( + seen_remotes + ) + if len(rejected_event_ids) > 0: + has_rejected_events = True + # The copies in `event_map` came off the wire, so they claim to be + # unrejected even for events we rejected when we first saw them, e.g + # during an earlier join of this room. Swap in the persisted copies, + # which carry `rejected_reason`, so that rejection cascades into the + # events in this batch which reference them via `prev_state_events`. + # + # Events rejected during *this* batch are not covered by this: they + # have not been persisted yet, so they are tracked separately and + # passed to the auth rules as `batched_rejections`. + persisted_rejected_events = await self._store.get_events( + rejected_event_ids, + allow_rejected=True, + redact_behaviour=EventRedactBehaviour.as_is, + ) + missing = rejected_event_ids - set(persisted_rejected_events) + if missing: + # We have seen these events and they have rejection rows, so we + # should be able to load them. If we cannot, rejection would + # silently stop cascading, so complain loudly instead. + raise AssertionError( + f"Could not load rejected events {missing} " + + f"out of {rejected_event_ids}" + ) + event_map.update(persisted_rejected_events) # XXX: it might be possible to kick this process off in parallel with fetching # the events. @@ -2084,32 +2188,39 @@ class FederationEventHandler: if supports_msc4242_state_dag(ev) else ev.auth_event_ids() ) - if e_id in event_map + if e_id in unseen_event_map ] - for ev in event_map.values() + for ev in unseen_event_map.values() } - sorted_event_ids = sorted_topologically(event_map.keys(), auth_graph) - sorted_events = [event_map[e_id] for e_id in sorted_event_ids] + unseen_sorted_event_ids = sorted_topologically( + unseen_event_map.keys(), auth_graph + ) + unseen_sorted_events = [ + unseen_event_map[e_id] for e_id in unseen_sorted_event_ids + ] logger.info( "Persisting %i remaining outliers: %s", - len(sorted_events), - shortstr(e.event_id for e in sorted_events), + len(unseen_sorted_events), + shortstr(e.event_id for e in unseen_sorted_events), ) if is_state_dag_room: - return await self._auth_and_persist_state_dag_outliers( - room_id, event_map, sorted_events, from_send_join + has_new_rejected_events = await self._auth_and_persist_state_dag_outliers( + room_id, event_map, unseen_sorted_events, from_send_join ) + return has_rejected_events or has_new_rejected_events events_and_contexts_to_persist: list[EventPersistencePair] = [] # get all the auth events for all the events in this batch. By now, they should # have been persisted. auth_event_ids = { - aid for event in sorted_events for aid in event.auth_event_ids() + aid for event in unseen_sorted_events for aid in event.auth_event_ids() } auth_map = { - ev.event_id: ev for ev in sorted_events if ev.event_id in auth_event_ids + ev.event_id: ev + for ev in unseen_sorted_events + if ev.event_id in auth_event_ids } missing_events = auth_event_ids.difference(auth_map) @@ -2166,7 +2277,7 @@ class FederationEventHandler: events_and_contexts_to_persist.append((event, context)) - for i, event in enumerate(sorted_events): + for i, event in enumerate(unseen_sorted_events): await prep(event) # The above function is typically not async, and so won't yield to @@ -2175,10 +2286,12 @@ class FederationEventHandler: if (i + 1) % 1000 == 0: await self._clock.sleep(Duration(seconds=0)) - has_rejected_events = any( - context.rejected is not None - for _, context in events_and_contexts_to_persist - ) + if not has_rejected_events: + # Check if any of the persisted events were rejected + has_rejected_events = any( + context.rejected is not None + for _, context in events_and_contexts_to_persist + ) # Also persist the new event in batches for similar reasons as above. for batch in batch_iter(events_and_contexts_to_persist, 1000): @@ -2215,7 +2328,12 @@ class FederationEventHandler: Params: room_id: the room that the events are meant to be in (though this has not yet been checked) - event_map: all of the events being persisted, by event ID + event_map: every event we were given, by event ID. This includes events we + have already persisted and so are not persisting again, because we may + still need them to work out the auth events of an event we have not seen + before. Any of those which were rejected are the persisted copies, + so they carry `rejected_reason`. On /send_join this is the complete + state DAG. sorted_events: the events being persisted, sorted so that an event's `prev_state_events` come before it from_send_join: True if `event_map` is the complete state DAG from a @@ -2223,32 +2341,87 @@ class FederationEventHandler: Returns: True if any of the events were rejected. """ + for event in sorted_events: + self._sanity_check_event(event) + has_rejected_events = False events_and_contexts_to_persist: list[EventPersistencePair] = [] event_id_to_state_group: dict[str, int] = {} state_group_to_state_map: dict[int, StateMap[str]] = {} # Tracks which events have been processed in causal order (prev_state_events first) processed_event_map: dict[str, EventBase] = {} + # The events in this batch which we have rejected, and why. Rejection cascades + # through `prev_state_events`, but nothing in the batch is persisted until we have + # processed all of it, so an event we rejected a moment ago still looks accepted + # both in the database and on the in-memory event. We therefore have to remember + # our own decisions and tell the auth rules about them. + rejected_event_id_to_reason_map: dict[str, str] = {} - async def process(event: EventBase) -> EventPersistencePair: + async def prev_state_maps(event: MSC4242Event) -> dict[int, StateMap[str]]: + """The state at each of an event's `prev_state_events`, by state group. + + A `prev_state_event` processed earlier in this batch has not been persisted + yet, so the state at it is only in memory. A `prev_state_event` persisted in the past + e.g. from a previous room join are stored in the database. A single event can reference + both, so fill in from the database on a miss and remember it for the rest of + the batch. + """ + missing = [ + prev_state_event_id + for prev_state_event_id in event.prev_state_events + if prev_state_event_id not in event_id_to_state_group + ] + if missing: + # Anything we have not remembered was persisted before this batch, so + # the database knows the state at it. + groups = ( + await self._state_storage_controller.get_state_group_for_events( + missing, + await_full_state=False, + ) + ) + unknown = [ + prev_state_event_id + for prev_state_event_id in missing + if groups.get(prev_state_event_id) is None + ] + if unknown: + # `get_state_group_for_events` claims to raise when it can't + # find a state group, but that only covers lookups which reached the + # database :/ cached entries come back as None. + # We can't calculate the state at `event` without knowing the state groups + # for ALL prev_state_events so bail out if we're missing them. + raise RuntimeError( + f"No state group for prev_state_events {unknown} of " + f"{event.event_id}, so the state before it cannot be calculated" + ) + + group_to_state = ( + await self._state_storage_controller.get_state_for_groups( + set(groups.values()) + ) + ) + # Now populate the in-memory map + for prev_state_event_id, state_group in groups.items(): + event_id_to_state_group[prev_state_event_id] = state_group + state_group_to_state_map[state_group] = group_to_state[state_group] + + return { + event_id_to_state_group[prev_state_event_id]: state_group_to_state_map[ + event_id_to_state_group[prev_state_event_id] + ] + for prev_state_event_id in event.prev_state_events + } + + async def process(event: MSC4242Event) -> EventPersistencePair: assert supports_msc4242_state_dag(event) with nested_logging_context(suffix=event.event_id): - # We can only use the state we have remembered if we have remembered it - # for every one of the event's `prev_state_events`. If we haven't, the - # missing ones were already persisted, so use the database instead. + # On a /send_join the events in this batch are not persisted until we + # have processed all of them, so the state at an event's + # `prev_state_events` can be spread across memory and the database. known_prev_state_maps = None - if from_send_join and all( - prev_state_event_id in event_id_to_state_group - for prev_state_event_id in event.prev_state_events - ): - known_prev_state_maps = { - event_id_to_state_group[prev_state_event_id]: ( - state_group_to_state_map[ - event_id_to_state_group[prev_state_event_id] - ] - ) - for prev_state_event_id in event.prev_state_events - } + if from_send_join: + known_prev_state_maps = await prev_state_maps(event) ( context, @@ -2267,32 +2440,24 @@ class FederationEventHandler: if from_send_join: # event_map is the complete state DAG - # The events in this batch aren't persisted yet, so pull the auth - # events out of the batch, falling back to the database for events we - # had already seen and hence filtered out of the batch. + # The events being persisted aren't in the database yet, so pull the + # auth events out of `event_map`, which holds the whole state DAG, + # including the events we had already persisted before this join. + missing_auth_event_ids = set(calculated_auth_event_ids).difference( + event_map + ) + if missing_auth_event_ids: + raise SynapseError( + 502, + f"state DAG is missing the calculated auth events {missing_auth_event_ids} " + f"for event {event.event_id}", + ) calculated_auth_events = { event_id: event_map[event_id] for event_id in calculated_auth_event_ids } - # In theory we should not be missing any auth events because event_map contains - # the entire state DAG, but if we do we can ask the database if it knows about them. - missing_auth_event_ids = set(calculated_auth_event_ids).difference( - calculated_auth_events - ) - if missing_auth_event_ids: - calculated_auth_events.update( - await self._store.get_events( - missing_auth_event_ids, - # Allow rejected events so events which depend on them fail with coherent - # error messages. If we filtered them out here, we'd instead fail with - # a missing events error which is misleading. - allow_rejected=True, - redact_behaviour=EventRedactBehaviour.as_is, - ) - ) # In order to run auth rule checks we need the events referenced in the event # (prev_state_events) and the events we calculated (auth_events) so load them now. - # If we are missing any, auth rule checks will produce a coherent error message. batched_auth_events = { event_id: event_map[event_id] for event_id in itertools.chain( @@ -2321,7 +2486,10 @@ class FederationEventHandler: try: validate_event_for_room_version(event) await check_state_independent_auth_rules( - self._store, event, batched_auth_events + self._store, + event, + batched_auth_events, + batched_rejections=rejected_event_id_to_reason_map, ) check_state_dependent_auth_rules( event, calculated_auth_events.values() @@ -2341,11 +2509,16 @@ class FederationEventHandler: logger.warning("While validating received event %r: %s", event, e) context.rejected = RejectedReason.OVERSIZED_EVENT + if context.rejected is not None: + rejected_event_id_to_reason_map[event.event_id] = context.rejected processed_event_map[event.event_id] = event return event, context if from_send_join: for i, event in enumerate(sorted_events): + assert supports_msc4242_state_dag( + event + ) # to convert from EventBase to MSC4242Event event_and_context = await process(event) if event_and_context[1].rejected is not None: has_rejected_events = True @@ -2368,6 +2541,9 @@ class FederationEventHandler: ) else: for event in sorted_events: + assert supports_msc4242_state_dag( + event + ) # to convert from EventBase to MSC4242Event event_and_context = await process(event) if event_and_context[1].rejected is not None: has_rejected_events = True @@ -2420,6 +2596,7 @@ class FederationEventHandler: if known_prev_state_maps: if len(known_prev_state_maps) == 1: # There's only 1 state map so we don't need to do state resolution at all. + # resolve_state_groups should not be called for a single state group as per its docstring state_ids = list(known_prev_state_maps.values())[0] else: res = await self._state_resolution_handler.resolve_state_groups( @@ -2439,10 +2616,19 @@ class FederationEventHandler: # There is no filter to apply here and we want all the events anyway since we need # to persist the entire calculate state. state_filter=None, - # we need to have finished processing the events causally prior to `event` in the - # state DAG before we can compute the state for this `event`, but this function is - # called unilaterally WHEN we are processing events in the state DAG. - # To wait here would be circular, we'd deadlock. + # By this point, we expect to have already finished processing the + # events *causally prior* to `event` in the state DAG because this + # function is called unilaterally WHEN we are processing events in the + # state DAG all the way down the chain and the caller ensures we process + # events in topological order. + # + # If this assumption is ever violated, `compute_state_after_events` will + # yell loudly about missing state groups. + # + # In the context of partially stated rooms (faster remote room joins) + # this would be part of background step verifying the entire state DAG, + # so we know that we don't have the full state processed yet. To wait + # here would be circular, we'd deadlock. await_full_state=False, ) @@ -2844,6 +3030,12 @@ class FederationEventHandler: if supports_msc4242_state_dag(event): return await self._load_calculated_auth_events(event, context) return event_auth_events.values() + if supports_msc4242_state_dag(event): + raise AuthError( + code=HTTPStatus.FORBIDDEN, + msg=f"Unknown prev_state_events {missing_auth_event_ids}; fetching them " + "is not implemented for MSC4242 state DAG rooms", + ) if destination is None: # this shouldn't happen: destination must be set unless we know we have already # persisted the auth events. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5230772e49..7a8866c8ed 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -509,8 +509,6 @@ class EventCreationHandler: self._worker_lock_handler = hs.get_worker_locks_handler() self._policy_handler = hs.get_room_policy_handler() - self.room_prejoin_state_types = self.hs.config.api.room_prejoin_state - self.send_events = ReplicationSendEventsRestServlet.make_client(hs) self.request_ratelimiter = hs.get_request_ratelimiter() @@ -2081,9 +2079,8 @@ class EventCreationHandler: event.unsigned, "invite_room_state", await self.store.get_stripped_room_state_from_event_context( + event, context, - self.room_prejoin_state_types, - membership_user_id=event.sender, ), ) @@ -2094,7 +2091,7 @@ class EventCreationHandler: # to get them to sign the event. returned_invite = await federation_handler.send_invite( - invitee.domain, event + invitee.domain, event, context ) # TODO: Make sure the signatures actually are correct. @@ -2106,8 +2103,8 @@ class EventCreationHandler: event.unsigned, "knock_room_state", await self.store.get_stripped_room_state_from_event_context( + event, context, - self.room_prejoin_state_types, ), ) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 112ac4c8c5..7521975cc5 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -985,9 +985,9 @@ class ProfileHandler: self, target_user: UserID, requester: UserID | None = None ) -> None: """Checks whether a profile query is allowed. If the - 'require_auth_for_profile_requests' config flag is set to True and a - 'requester' is provided, the query is only allowed if the two users - share a room. + 'limit_profile_requests_to_users_who_share_rooms' config flag is set to + True and a 'requester' is provided, the query is only allowed if the two + users share a room. Args: target_user: The owner of the queried profile. diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index c208b4e91a..8d41582c58 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -21,7 +21,7 @@ import logging from typing import TYPE_CHECKING, Callable, Iterable, Sequence -from synapse.api.constants import EduTypes, ReceiptTypes +from synapse.api.constants import Direction, EduTypes, ReceiptTypes from synapse.appservice import ApplicationService from synapse.streams import EventSource from synapse.types import ( @@ -355,15 +355,35 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): A two-tuple containing the following: * A list of json dictionaries derived from read receipts that the appservice may be interested in. - * The current read receipt stream token. + * The read receipt stream token up to which receipts were actually + fetched. This is earlier than `to_key` if the fetch was + truncated; callers must call this method again from the returned + token to fetch the remaining receipts. """ if from_key == to_key: return [], to_key - # Fetch all read receipts for all rooms, up to a limit of 100. This is ordered - # by most recent. - rooms_to_events = await self.store.get_linearized_receipts_for_all_rooms( - from_key=from_key, to_key=to_key + # A stored stream position of 1 means no read receipts have ever been + # delivered to this application service: + # `get_type_stream_id_for_appservice` returns 1 when there is no stored + # position, and actual receipts occupy stream IDs 2 and upwards. + is_first_receipt_delivery = from_key.stream <= 1 + + # Fetch read receipts for all rooms, in ascending stream order. The number + # of receipts fetched is capped, in which case `reached_token` tells us how + # far we actually got. + # + # On the first delivery, don't backfill the entire receipt history: just + # send the most recent receipts and fast-forward past everything older. + ( + rooms_to_events, + reached_token, + ) = await self.store.get_linearized_receipts_for_all_rooms( + from_key=from_key, + to_key=to_key, + order=Direction.BACKWARDS + if is_first_receipt_delivery + else Direction.FORWARDS, ) # Then filter down to rooms that the AS can read @@ -379,7 +399,7 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): # https://spec.matrix.org/v1.19/application-service-api/#pushing-ephemeral-data events = self._filter_private_receipts(events, service.is_interested_in_user) - return events, to_key + return events, reached_token def get_current_key(self) -> MultiWriterStreamToken: return self.store.get_max_receipt_stream_id() diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 92d82aeda1..99a3396c1d 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -2542,10 +2542,19 @@ class RoomShutdownHandler: result["kicked_users"].append(user_id) if update_result_fct: await update_result_fct(result) - except Exception: - logger.exception( - "Failed to leave old room and join new room for %r", user_id - ) + except Exception as exc: + if ( + isinstance(exc, SynapseError) + and exc.errcode == Codes.USER_ACCOUNT_SUSPENDED + ): + logger.warning( + "Failed to leave old room and join new room for suspended user (probably expected) %r", + user_id, + ) + else: + logger.exception( + "Failed to leave old room and join new room for %r", user_id + ) result["failed_to_kick_users"].append(user_id) if update_result_fct: await update_result_fct(result) diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index c79a94e930..64330de558 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -369,7 +369,7 @@ class RoomSummaryHandler: # inaccessible to the requesting user. if room_entry: # Add the room (including the stripped m.space.child events). - rooms_result.append(room_entry.as_json(for_client=True)) + rooms_result.append(room_entry.as_json()) # If this room is not at the max-depth, check if there are any # children to process. @@ -889,7 +889,8 @@ class RoomSummaryHandler: remote_room_hosts: list[str] | None = None, ) -> JsonDict: """ - Implementation of the room summary C-S API from MSC3266 + Implementation of the room summary C-S API, see + https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1room_summaryroomidoralias Args: requester: user id of the user making this request, will be None @@ -997,25 +998,14 @@ class _RoomEntry: # This may not include all children. children_state_events: Sequence[JsonDict] = () - def as_json(self, for_client: bool = False) -> JsonDict: + def as_json(self) -> JsonDict: """ Returns a JSON dictionary suitable for the room hierarchy endpoint. It returns the room summary including the stripped m.space.child events as a sub-key. - - Args: - for_client: If true, any server-server only fields are stripped from - the result. - """ result = dict(self.room) - - # Before returning to the client, remove the allowed_room_ids key, if it - # exists. - if for_client: - result.pop("allowed_room_ids", False) - result["children_state"] = self.children_state_events return result diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py index 8cdf9c6a87..94d65e3dfe 100644 --- a/synapse/handlers/send_email.py +++ b/synapse/handlers/send_email.py @@ -97,6 +97,9 @@ async def _sendmail( ) if force_tls: + # Appease type-checking. + # EmailConfig prevents `force_tls` from being True when TLS is disabled anyhow. + assert tlsname is not None factory = TLSMemoryBIOFactory(optionsForClientTLS(tlsname), True, factory) endpoint = HostnameEndpoint( diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 836cee6c20..f4d1f484e7 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -37,7 +37,6 @@ from synapse.api.constants import ( from synapse.api.errors import SlidingSyncUnknownPosition from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import StrippedStateEvent -from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import start_active_span, trace from synapse.storage.databases.main.sliding_sync import UPDATE_INTERVAL_LAST_USED_TS from synapse.storage.databases.main.state import ( @@ -1618,7 +1617,7 @@ class SlidingSyncRoomLists: stripped_state_map = {} if isinstance(raw_stripped_state_events, list): for raw_stripped_event in raw_stripped_state_events: - stripped_state_event = parse_stripped_state_event( + stripped_state_event = StrippedStateEvent.from_json_dict( raw_stripped_event ) if stripped_state_event is not None: diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c66034cd0e..2eb5eb79c6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -23,10 +23,12 @@ import itertools import json import logging import os +from collections import OrderedDict from typing import ( TYPE_CHECKING, AbstractSet, Any, + Collection, Mapping, Sequence, ) @@ -73,6 +75,7 @@ from synapse.types import ( MutableStateMap, Requester, RoomStreamToken, + StateKey, StateMap, StrCollection, StreamKeyType, @@ -129,6 +132,96 @@ LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_SIZE = 16 LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_KEY = os.urandom(32) +@attr.s(slots=True, auto_attribs=True) +class LazyLoadedMembersCache: + """The membership events a lazy-loading client has been sent, so that later + syncs can leave them out. One per (user, device). + + A member may only be left out if the client processed the response that + carried it. Each entry therefore records the `next_batch` position of the + response that carried it. A client that has processed a response syncs + from its `next_batch` next, so a request from `since` means the client + processed every response up to `since` and none after it. `start_request` + forgets the entries recorded after `since`. They were in responses the + client lost, by timing out a slow request and retrying it (#19978) or by + reloading from an older persisted token (#20278). Entries recorded at or + before `since` stay, so a client resuming from a token a few minutes old is + only sent again what it lost in those minutes. An initial sync forgets + everything. + + Forgetting everything whenever `since` is before the newest entry would be + simpler, and `_newest` alone would suffice. But Element Web only persists + its sync token every few minutes and resumes from it on every reload, so + that would re-send every cached member on each reload. + + The notifier may build several results for one long-poll and returns only + the first non-empty one. An empty result carries no members, so it records + nothing. + + Leaving out a member the client never received loses it for good. A + `state_after` client does not apply state from the timeline, and a `state` + client only sees the membership changes that fall inside the timeline. + """ + + # user_id -> (event_id of the membership we last sent, room position of the + # `next_batch` of the response that carried it). Least recently used first. + # + # Not an `LruCache`, as that cannot be iterated over and `start_request` + # needs to check every entry's position against `since`. + _sent: OrderedDict[str, tuple[str, RoomStreamToken]] = attr.Factory(OrderedDict) + + # An upper bound on the positions of the entries, so that `start_request` + # need not look at them when nothing was recorded after `since`. That is the + # steady state, where `since` is the `next_batch` of the last response. + _newest: RoomStreamToken | None = None + + def start_request(self, since_token: StreamToken | None) -> None: + """Forget the members the client did not receive. Called at the start of + each sync request for this device.""" + if since_token is None: + self._sent.clear() + self._newest = None + return + + # Check for the common case where nothing has been recorded after + # `since`. + since = since_token.room_key + if self._newest is None or self._newest.is_before_or_eq(since): + return + + # Remove all entries that were sent after `since`. This only has + # `LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE` entries. + for user_id, (_, sent_at) in list(self._sent.items()): + if not sent_at.is_before_or_eq(since): + del self._sent[user_id] + + # Everything left is at or before `since`. + self._newest = since + + def was_sent(self, user_id: str, event_id: str) -> bool: + """Whether `event_id` is the membership of `user_id` we last sent.""" + entry = self._sent.get(user_id) + if entry is None or entry[0] != event_id: + return False + self._sent.move_to_end(user_id) + return True + + def mark_sent(self, user_id: str, event_id: str, next_batch: StreamToken) -> None: + """Record that `event_id` is being sent in the response ending at + `next_batch`.""" + sent_at = next_batch.room_key + self._sent[user_id] = (event_id, sent_at) + self._sent.move_to_end(user_id) + while len(self._sent) > LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE: + self._sent.popitem(last=False) + + # Only move the bound forwards. Positions taken by this process only + # ever advance, but a response that started earlier may record its + # entries after a later one has. + if self._newest is None or self._newest.is_before_or_eq(sent_at): + self._newest = sent_at + + SyncRequestKey = tuple[Any, ...] @@ -344,9 +437,9 @@ class SyncHandler: timeout=hs.config.caches.sync_response_cache_duration, ) - # ExpiringCache((User, Device)) -> LruCache(user_id => event_id) + # ExpiringCache((User, Device)) -> LazyLoadedMembersCache self.lazy_loaded_members_cache: ExpiringCache[ - tuple[str, str | None], LruCache[str, str] + tuple[str, str | None], LazyLoadedMembersCache ] = ExpiringCache( cache_name="lazy_loaded_members_cache", server_name=self.server_name, @@ -701,6 +794,37 @@ class SyncHandler: return now_token, sticky_by_room + async def sticky_events_for_newly_joined_rooms( + self, + now_token: StreamToken, + newly_joined_rooms: Collection[str], + ) -> dict[str, list[str]]: + """Get all the sticky events for each newly-joined room the user is in + Args: + now_token: Where the server is currently up to. + newly_joined_rooms: Room IDs of rooms that are newly-joined + Returns: + Dict from room ID to list of sticky event IDs + """ + now = self.clock.time_msec() + with Measure( + self.clock, + name="sticky_events_for_newly_joined_rooms", + server_name=self.server_name, + ): + _, sticky_by_room = await self.store.get_sticky_events_in_rooms( + newly_joined_rooms, + # Since the start of time + from_id=0, + to_id=now_token.sticky_events_key, + now=now, + # Unfortunately, we're meant to return all sticky events in one go + # (we don't have a good alternative). + # See: https://github.com/matrix-org/matrix-spec-proposals/pull/4354#discussion_r3021907998 + limit=None, + ) + return sticky_by_room + async def _load_filtered_recents( self, room_id: str, @@ -1056,7 +1180,7 @@ class SyncHandler: member_ids[hero_id] for hero_id in summary["m.heroes"] if ( - cache.get(hero_id) != member_ids[hero_id] + not cache.was_sent(hero_id, member_ids[hero_id]) and hero_id not in existing_members ) ] @@ -1064,27 +1188,21 @@ class SyncHandler: missing_hero_state = await self.store.get_events(missing_hero_event_ids) for s in missing_hero_state.values(): - cache.set(s.state_key, s.event_id) + cache.mark_sent(s.state_key, s.event_id, now_token) state[(EventTypes.Member, s.state_key)] = s return summary def get_lazy_loaded_members_cache( self, cache_key: tuple[str, str | None] - ) -> LruCache[str, str]: - # FIXME: This cache may be subject to losing members in the case that - # a sync is interrupted and retried, see https://github.com/element-hq/synapse/issues/19978 - cache: LruCache[str, str] | None = self.lazy_loaded_members_cache.get(cache_key) + ) -> LazyLoadedMembersCache: + cache = self.lazy_loaded_members_cache.get(cache_key) if cache is None: - logger.debug("creating LruCache for %r", cache_key) - cache = LruCache( - max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE, - clock=self.clock, - server_name=self.server_name, - ) + logger.debug("creating LazyLoadedMembersCache for %r", cache_key) + cache = LazyLoadedMembersCache() self.lazy_loaded_members_cache[cache_key] = cache else: - logger.debug("found LruCache for %r", cache_key) + logger.debug("found LazyLoadedMembersCache for %r", cache_key) return cache def get_lazy_loaded_profile_fields_cache( @@ -1123,6 +1241,7 @@ class SyncHandler: sync_config: SyncConfig, since_token: StreamToken | None, end_token: StreamToken, + now_token: StreamToken, full_state: bool, joined: bool, ) -> MutableStateMap[EventBase]: @@ -1137,6 +1256,7 @@ class SyncHandler: end_token: Token of the end of the current batch. Normally this will be the same as the global "now_token", but if the user has left the room, the point just after their leave event. + now_token: The `next_batch` token of the response being built. full_state: Whether to force returning the full state. `lazy_load_members` still applies when `full_state` is `True`. joined: whether the user is currently joined to the room @@ -1222,6 +1342,11 @@ class SyncHandler: # sync's timeline and the start of the current sync's timeline. # See the docstring above for details. state_ids: StateMap[str] + # The keys in `state_ids` whose state changed since `since_token`. + # These are sent whether or not the lazy-loaded members cache says + # the client has them. The rest are memberships fetched for + # timeline senders, which the client may already have. + changed_keys: set[StateKey] = set() # We need to know whether the state we fetch may be partial, so check # whether the room is partial stated *before* fetching it. is_partial_state_room = await self.store.is_partial_state_room(room_id) @@ -1241,7 +1366,10 @@ class SyncHandler: # is indeed the case. assert since_token is not None - state_ids = await self._compute_state_delta_for_incremental_sync( + ( + state_ids, + changed_keys, + ) = await self._compute_state_delta_for_incremental_sync( room_id, sync_config, batch, @@ -1277,30 +1405,28 @@ class SyncHandler: cache_key = (sync_config.user.to_string(), sync_config.device_id) cache = self.get_lazy_loaded_members_cache(cache_key) - # if it's a new sync sequence, then assume the client has had - # amnesia and doesn't want any recent lazy-loaded members - # de-duplicated. - if since_token is None: - logger.debug("clearing LruCache for %r", cache_key) - cache.clear() - else: - # only send members which aren't in our LruCache (either - # because they're new to this client or have been pushed out - # of the cache) - logger.debug("filtering state from %r...", state_ids) - state_ids = { - t: event_id - for t, event_id in state_ids.items() - if cache.get(t[1]) != event_id - } - logger.debug("...to %r", state_ids) + # Only send members which aren't in our cache (either because + # they're new to this client or have been pushed out of the + # cache). For an initial sync the cache has already been + # cleared. + # + # A state change since `since_token` is always sent, regardless + # of whether it was previously sent to the client (e.g due to a + # client retrying the request). + logger.debug("filtering state from %r...", state_ids) + state_ids = { + t: event_id + for t, event_id in state_ids.items() + if t in changed_keys or not cache.was_sent(t[1], event_id) + } + logger.debug("...to %r", state_ids) - # add any member IDs we are about to send into our LruCache + # add any member IDs we are about to send into our cache for t, event_id in itertools.chain( state_ids.items(), timeline_state.items() ): if t[0] == EventTypes.Member: - cache.set(t[1], event_id) + cache.mark_sent(t[1], event_id, now_token) state: dict[str, EventBase] = {} if state_ids: @@ -1473,14 +1599,16 @@ class SyncHandler: members_to_fetch: set[str] | None, timeline_state: StateMap[str], joined: bool, - ) -> StateMap[str]: + ) -> tuple[StateMap[str], set[StateKey]]: """Calculate the state events to be included in an incremental sync response. If lazy-loading of membership events is enabled (as indicated by `members_to_fetch` being not-`None`), the result will include the membership events for each member in `members_to_fetch`. The caller (`compute_state_delta`) is responsible for keeping track of which membership - events we have already sent to the client, and hence ripping them out. + events we have already sent to the client, and hence ripping them out. It + may only rip out entries that are not state changes, so this also returns + the keys whose state changed since `since_token`. Note that whether this returns the state at the start or the end of the batch depends on `sync_config.use_state_after` (c.f. MSC4222). @@ -1501,7 +1629,10 @@ class SyncHandler: Returns: A map from (type, state_key) to event_id, for each event that we believe - should be included in the `state` or `state_after` part of the sync response. + should be included in the `state` or `state_after` part of the sync + response, and the subset of its keys whose state changed since + `since_token`. The other keys are memberships fetched for timeline + senders, which the client may have been sent before. """ if members_to_fetch is not None: # Lazy-loading is enabled. Only return the state that is needed. @@ -1517,6 +1648,7 @@ class SyncHandler: # timeline. If at the end we can just use the current state delta stream. if sync_config.use_state_after: delta_state_ids: MutableStateMap[str] = {} + changed_keys: set[StateKey] = set() if members_to_fetch: # We're lazy-loading, so the client might need some more member @@ -1571,9 +1703,11 @@ class SyncHandler: # Note that deltas are in stream ordering, so if there are # multiple deltas for a given type/state_key we'll always pick # the latest one. - delta_state_ids[(delta.event_type, delta.state_key)] = delta.event_id + key = (delta.event_type, delta.state_key) + delta_state_ids[key] = delta.event_id + changed_keys.add(key) - return delta_state_ids + return delta_state_ids, changed_keys # For a non-gappy sync if the events in the timeline are simply a linear # chain (i.e. no merging/branching of the graph), then we know the state @@ -1621,7 +1755,9 @@ class SyncHandler: await_full_state=False, ) ) - return state_ids + # A linear, non-gappy timeline means the only changes to the `state` + # are lazy-loaded members. + return state_ids, set() if batch: state_at_timeline_start = ( @@ -1680,7 +1816,16 @@ class SyncHandler: lazy_load_members=lazy_load_members, ) - return state_ids + # Everything that differs from the state at the previous sync is a + # change. What is left is the memberships of timeline senders that + # `_calculate_state` adds back for lazy loading. + changed_keys = { + key + for key, event_id in state_ids.items() + if state_at_previous_sync.get(key) != event_id + } + + return state_ids, changed_keys async def _find_missing_partial_state_memberships( self, @@ -2618,7 +2763,13 @@ class SyncHandler: """ since_token = sync_result_builder.since_token - user_id = sync_result_builder.sync_config.user.to_string() + sync_config = sync_result_builder.sync_config + user_id = sync_config.user.to_string() + + if sync_config.filter_collection.lazy_load_members(): + self.get_lazy_loaded_members_cache( + (user_id, sync_config.device_id) + ).start_request(since_token) blocks_all_rooms = ( sync_result_builder.sync_config.filter_collection.blocks_all_rooms() @@ -2729,6 +2880,16 @@ class SyncHandler: newly_joined_rooms = room_changes.newly_joined_rooms newly_left_rooms = room_changes.newly_left_rooms + if self.hs_config.experimental.msc4354_enabled and newly_joined_rooms: + # If we have any newly-joined rooms, load all sticky events from them. + # We then send down all the sticky events, including historical ones, + # rather than just the ones that are new since the `since` token. + sticky_by_room.update( + await self.sticky_events_for_newly_joined_rooms( + sync_result_builder.now_token, newly_joined_rooms + ) + ) + # 4. We need to apply further processing to `room_entries` (rooms considered # joined or archived). async def handle_room_entries(room_entry: "RoomSyncResultBuilder") -> None: @@ -3276,6 +3437,7 @@ class SyncHandler: sync_config, since_token, room_builder.end_token, + now_token, full_state=full_state, joined=room_builder.rtype == "joined", ) diff --git a/synapse/http/client.py b/synapse/http/client.py index 78f03ae58a..075363e534 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -47,7 +47,7 @@ from twisted.internet.interfaces import ( IAddress, IDelayedCall, IHostResolution, - IOpenSSLContextFactory, + IOpenSSLClientConnectionCreator, IReactorCore, IReactorPluggableNameResolver, IResolutionReceiver, @@ -55,6 +55,7 @@ from twisted.internet.interfaces import ( ) from twisted.internet.protocol import connectionDone from twisted.internet.task import Cooperator +from twisted.protocols.tls import TLSMemoryBIOProtocol from twisted.python.failure import Failure from twisted.web._newclient import ResponseDone from twisted.web.client import ( @@ -1314,7 +1315,7 @@ def encode_query_args(args: QueryParams | None) -> bytes: return query_str.encode("utf8") -@implementer(IPolicyForHTTPS) +@implementer(IPolicyForHTTPS, IOpenSSLClientConnectionCreator) class InsecureInterceptableContextFactory(ssl.ContextFactory): """ Factory for PyOpenSSL SSL contexts which accepts any certificate for any domain. @@ -1329,9 +1330,16 @@ class InsecureInterceptableContextFactory(ssl.ContextFactory): def getContext(self) -> SSL.Context: return self._context - def creatorForNetloc(self, hostname: bytes, port: int) -> IOpenSSLContextFactory: + def creatorForNetloc( + self, hostname: bytes, port: int + ) -> IOpenSSLClientConnectionCreator: return self + def clientConnectionForTLS( + self, tls_protocol: TLSMemoryBIOProtocol + ) -> SSL.Connection: + return SSL.Connection(self.getContext(), None) + def is_unknown_endpoint( e: HttpResponseException, synapse_error: SynapseError | None = None diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py index 285e6d3546..514aeea752 100644 --- a/synapse/http/connectproxyclient.py +++ b/synapse/http/connectproxyclient.py @@ -22,7 +22,7 @@ import abc import base64 import logging -from typing import Union +from typing import Callable, Optional, Union, cast import attr from zope.interface import implementer @@ -162,8 +162,14 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): if isinstance(self.wrapped_factory, ClientFactory): return self.wrapped_factory.startedConnecting(connector) - def buildProtocol(self, addr: IAddress) -> "HTTPConnectProtocol": - wrapped_protocol = self.wrapped_factory.buildProtocol(addr) + def buildProtocol(self, addr: Optional[IAddress]) -> "HTTPConnectProtocol": + # IProtocolFactory's annotation does not yet allow the absent address + # accepted by Factory.buildProtocol. + build_protocol = cast( + Callable[[Optional[IAddress]], Optional[IProtocol]], + self.wrapped_factory.buildProtocol, + ) + wrapped_protocol = build_protocol(addr) if wrapped_protocol is None: raise TypeError("buildProtocol produced None instead of a Protocol") diff --git a/synapse/http/server.py b/synapse/http/server.py index a0ae20be16..37dd77b008 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -758,10 +758,15 @@ class _ByteProducer: self._request: Request | None = request self._iterator = iterator self._paused = False - self.tracing_scope = start_active_span( - "write_bytes_to_request", - ) - self.tracing_scope.__enter__() + + # Start a span for writing bytes to the request. We manually manage its + # lifecycle, so set `finish_on_close=False`. + # + # Note that we cannot use `active_span()` in the functions below, as + # they are called from the reactor and therefore outside the request's + # context. + with start_active_span("write_bytes_to_request", finish_on_close=False): + self._span = active_span() try: self._request.registerProducer(self, True) @@ -772,8 +777,7 @@ class _ByteProducer: logger.info("Connection disconnected before response was written: %r", e) # We drop our references to data we'll not use. - self._iterator = iter(()) - self.tracing_scope.__exit__(type(e), None, e.__traceback__) + self.stopProducing() else: # Start producing if `registerProducer` was successful self.resumeProducing() @@ -787,9 +791,8 @@ class _ByteProducer: self._request.write(b"".join(data)) def pauseProducing(self) -> None: - opentracing_span = active_span() - if opentracing_span is not None: - opentracing_span.log_kv({"event": "producer_paused"}) + if self._span is not None: + self._span.log_kv({"event": "producer_paused"}) self._paused = True def resumeProducing(self) -> None: @@ -800,9 +803,8 @@ class _ByteProducer: self._paused = False - opentracing_span = active_span() - if opentracing_span is not None: - opentracing_span.log_kv({"event": "producer_resumed"}) + if self._span is not None: + self._span.log_kv({"event": "producer_resumed"}) # Write until there's backpressure telling us to stop. while not self._paused: @@ -836,9 +838,13 @@ class _ByteProducer: self._send_data(buffer) def stopProducing(self) -> None: - # Clear a circular reference. + # Clear a circular reference and drop references to the data. + self._iterator = iter(()) self._request = None - self.tracing_scope.__exit__(None, None, None) + + if self._span is not None: + self._span.finish() + self._span = None def _encode_json_bytes(json_object: object) -> bytes: diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index c182497f2d..d896a2a93e 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -27,8 +27,10 @@ import urllib.parse as urlparse from http import HTTPStatus from typing import ( TYPE_CHECKING, + Iterable, Literal, Mapping, + Pattern, Sequence, TypeVar, overload, @@ -944,9 +946,9 @@ class RestServlet: An implementing class can either provide its own custom 'register' method, or use the automatic pattern handling provided by the base class. - To use this latter, the implementing class instead provides a `PATTERN` - class attribute containing a pre-compiled regular expression. The automatic - register method will then use this method to register any of the following + To use this latter, the implementing class instead provides a `PATTERNS` + class attribute containing pre-compiled regular expressions. The automatic + register method will then use them to register any of the following instance methods associated with the corresponding HTTP method: on_GET @@ -958,9 +960,11 @@ class RestServlet: into the appropriate HTTP response. """ + PATTERNS: Iterable[Pattern[str]] | None = None + def register(self, http_server: HttpServer) -> None: """Register this servlet with the given HTTP server.""" - patterns = getattr(self, "PATTERNS", None) + patterns = self.PATTERNS if patterns: for method in ("GET", "PUT", "POST", "DELETE"): if hasattr(self, "on_%s" % (method,)): diff --git a/synapse/http/site.py b/synapse/http/site.py index 4b841a21e2..d4a405f61b 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -23,7 +23,7 @@ import json import logging import time from http import HTTPStatus -from typing import TYPE_CHECKING, Any, Generator +from typing import TYPE_CHECKING, Any, Generator, Optional import attr from zope.interface import implementer @@ -907,7 +907,10 @@ class SynapseSite(ProxySite): self.server_version_string = server_version_string.encode("ascii") self.connections: list[Protocol] = [] - def buildProtocol(self, addr: IAddress) -> SynapseProtocol: + # Twisted 26.4.0 types HTTPFactory.buildProtocol as returning its concrete + # `_GenericHTTPChannelProtocol` wrapper. We intentionally return our own HTTPChannel + # subclass instead. Thus we add a type-ignore. + def buildProtocol(self, addr: Optional[IAddress]) -> SynapseProtocol: # type: ignore[override] protocol = SynapseProtocol( self, self.server_name, diff --git a/synapse/logging/_remote.py b/synapse/logging/_remote.py index 72faa3c746..b10c3b9b01 100644 --- a/synapse/logging/_remote.py +++ b/synapse/logging/_remote.py @@ -128,7 +128,7 @@ class RemoteHandler(logging.Handler): if _reactor is None: from twisted.internet import reactor - _reactor = reactor # type: ignore[assignment] + _reactor = reactor try: ip = ip_address(self.host) diff --git a/synapse/logging/context.py b/synapse/logging/context.py index b6535be388..c7121cf260 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -31,7 +31,6 @@ See doc/log_contexts.rst for details on how this works. """ import logging -import threading import typing from types import TracebackType from typing import ( @@ -42,6 +41,7 @@ from typing import ( Literal, TypeVar, Union, + cast, overload, ) @@ -52,17 +52,24 @@ from twisted.internet import defer, threads from twisted.python.threadpool import ThreadPool from synapse.logging.loggers import ExplicitlyConfiguredLogger +from synapse.synapse_rust.logcontext import ( + DEBUG_LOGGER_NAME, + # Not used in this module, but re-exported: callers import it from here. + ContextResourceUsage, # noqa: F401 + LoggingContext, + current_context as _rust_current_context, + set_current_context as _rust_set_current_context, +) from synapse.util.stringutils import random_string_insecure_fast if TYPE_CHECKING: - from synapse.logging.scopecontextmanager import _LogContextScope from synapse.types import ISynapseReactor logger = logging.getLogger(__name__) original_logger_class = logging.getLoggerClass() logging.setLoggerClass(ExplicitlyConfiguredLogger) -logcontext_debug_logger = logging.getLogger("synapse.logging.context.debug") +logcontext_debug_logger = logging.getLogger(DEBUG_LOGGER_NAME) """ A logger for debugging when the logcontext switches. @@ -74,146 +81,12 @@ configuration and does not inherit the log level from the parent logger. # Restore the original logger class logging.setLoggerClass(original_logger_class) -try: - import resource - - # Python doesn't ship with a definition of RUSAGE_THREAD but it's defined - # to be 1 on linux so we hard code it. - RUSAGE_THREAD = 1 - - # If the system doesn't support RUSAGE_THREAD then this should throw an - # exception. - resource.getrusage(RUSAGE_THREAD) - - is_thread_resource_usage_supported = True - - def get_thread_resource_usage() -> "resource.struct_rusage | None": - return resource.getrusage(RUSAGE_THREAD) - -except Exception: - # If the system doesn't support resource.getrusage(RUSAGE_THREAD) then we - # won't track resource usage. - is_thread_resource_usage_supported = False - - def get_thread_resource_usage() -> "resource.struct_rusage | None": - return None - # a hook which can be set during testing to assert that we aren't abusing logcontexts. def logcontext_error(msg: str) -> None: logger.warning(msg) -# get an id for the current thread. -# -# threading.get_ident doesn't actually return an OS-level tid, and annoyingly, -# on Linux it actually returns the same value either side of a fork() call. However -# we only fork in one place, so it's not worth the hoop-jumping to get a real tid. -# -get_thread_id = threading.get_ident - - -class ContextResourceUsage: - """Object for tracking the resources used by a log context - - Attributes: - ru_utime (float): user CPU time (in seconds) - ru_stime (float): system CPU time (in seconds) - db_txn_count (int): number of database transactions done - db_sched_duration_sec (float): amount of time spent waiting for a - database connection - db_txn_duration_sec (float): amount of time spent doing database - transactions (excluding scheduling time) - evt_db_fetch_count (int): number of events requested from the database - """ - - __slots__ = [ - "ru_stime", - "ru_utime", - "db_txn_count", - "db_txn_duration_sec", - "db_sched_duration_sec", - "evt_db_fetch_count", - ] - - def __init__(self, copy_from: "ContextResourceUsage | None" = None) -> None: - """Create a new ContextResourceUsage - - Args: - copy_from: if not None, an object to copy stats from - """ - if copy_from is None: - self.reset() - else: - # FIXME: mypy can't infer the types set via reset() above, so specify explicitly for now - self.ru_utime: float = copy_from.ru_utime - self.ru_stime: float = copy_from.ru_stime - self.db_txn_count: int = copy_from.db_txn_count - - self.db_txn_duration_sec: float = copy_from.db_txn_duration_sec - self.db_sched_duration_sec: float = copy_from.db_sched_duration_sec - self.evt_db_fetch_count: int = copy_from.evt_db_fetch_count - - def copy(self) -> "ContextResourceUsage": - return ContextResourceUsage(copy_from=self) - - def reset(self) -> None: - self.ru_stime = 0.0 - self.ru_utime = 0.0 - self.db_txn_count = 0 - - self.db_txn_duration_sec = 0.0 - self.db_sched_duration_sec = 0.0 - self.evt_db_fetch_count = 0 - - def __repr__(self) -> str: - return ( - "" - ) % ( - self.ru_stime, - self.ru_utime, - self.db_txn_count, - self.db_txn_duration_sec, - self.db_sched_duration_sec, - self.evt_db_fetch_count, - ) - - def __iadd__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - """Add another ContextResourceUsage's stats to this one's. - - Args: - other: the other resource usage object - """ - self.ru_utime += other.ru_utime - self.ru_stime += other.ru_stime - self.db_txn_count += other.db_txn_count - self.db_txn_duration_sec += other.db_txn_duration_sec - self.db_sched_duration_sec += other.db_sched_duration_sec - self.evt_db_fetch_count += other.evt_db_fetch_count - return self - - def __isub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - self.ru_utime -= other.ru_utime - self.ru_stime -= other.ru_stime - self.db_txn_count -= other.db_txn_count - self.db_txn_duration_sec -= other.db_txn_duration_sec - self.db_sched_duration_sec -= other.db_sched_duration_sec - self.evt_db_fetch_count -= other.evt_db_fetch_count - return self - - def __add__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - res = ContextResourceUsage(copy_from=self) - res += other - return res - - def __sub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - res = ContextResourceUsage(copy_from=self) - res -= other - return res - - @attr.s(slots=True, auto_attribs=True) class ContextRequest: """ @@ -237,9 +110,6 @@ class ContextRequest: user_agent: str -LoggingContextOrSentinel = Union["LoggingContext", "_Sentinel"] - - class _Sentinel: """ Sentinel to represent the root context @@ -273,10 +143,10 @@ class _Sentinel: def __str__(self) -> str: return "sentinel" - def start(self, rusage: "resource.struct_rusage | None") -> None: + def start(self, rusage: "tuple[float, float] | None") -> None: pass - def stop(self, rusage: "resource.struct_rusage | None") -> None: + def stop(self, rusage: "tuple[float, float] | None") -> None: pass def add_database_transaction(self, duration_sec: float) -> None: @@ -294,285 +164,41 @@ class _Sentinel: SENTINEL_CONTEXT = _Sentinel() +LoggingContextOrSentinel = Union[LoggingContext, _Sentinel] -class LoggingContext: - """Additional context for log formatting. Contexts are scoped within a - "with" block. - If a parent is given when creating a new context, then: - - logging fields are copied from the parent to the new context on entry - - when the new context exits, the cpu usage stats are copied from the - child to the parent - - Args: - name: Name for the context for logging. - server_name: The name of the server this context is associated with - (`config.server.server_name` or `hs.hostname`) - parent_context (LoggingContext|None): The parent of the new context - request: Synapse Request Context object. Useful to associate all the logs - happening to a given request. +def current_context() -> LoggingContextOrSentinel: + """Get the current logging context. + The storage lives in the Rust extension, which represents "no context" as + `None`. This wrapper maps that to `SENTINEL_CONTEXT`, so callers never see + `None`. """ + context = _rust_current_context() + return SENTINEL_CONTEXT if context is None else context - __slots__ = [ - "previous_context", - "name", - "server_name", - "parent_context", - "_resource_usage", - "usage_start", - "main_thread", - "finished", - "request", - "tag", - "scope", - ] - def __init__( - self, - *, - name: str, - server_name: str, - parent_context: "LoggingContext | None" = None, - request: ContextRequest | None = None, - ) -> None: - self.previous_context = current_context() +def set_current_context(context: LoggingContextOrSentinel) -> LoggingContextOrSentinel: + """Set the current logging context, returning the context that was + previously current. - # track the resources used by this context so far - self._resource_usage = ContextResourceUsage() + The implementation lives in the Rust extension: it reads the thread CPU + usage once (`getrusage(RUSAGE_THREAD)`) and does the `stop`/`start` + accounting in Rust. Rust represents the sentinel as `None`. This wrapper + converts in both directions, so callers only ever see + `LoggingContextOrSentinel`. + """ + # everything blows up if we allow current_context to be set to None, so + # sanity-check that now. + if context is None: + raise TypeError("'context' argument may not be None") - # The thread resource usage when the logcontext became active. None - # if the context is not currently active. - self.usage_start: resource.struct_rusage | None = None - - self.name = name - self.server_name = server_name - self.main_thread = get_thread_id() - self.request = None - self.tag = "" - self.scope: "_LogContextScope" | None = None - - # keep track of whether we have hit the __exit__ block for this context - # (suggesting that the the thing that created the context thinks it should - # be finished, and that re-activating it would suggest an error). - self.finished = False - - self.parent_context = parent_context - - # Inherit some fields from the parent context - if self.parent_context is not None: - # which request this corresponds to - self.request = self.parent_context.request - - # we also track the current scope: - self.scope = self.parent_context.scope - - if request is not None: - # the request param overrides the request from the parent context - self.request = request - - def __str__(self) -> str: - return self.name - - def __enter__(self) -> "LoggingContext": - """Enters this logging context into thread local storage""" - logcontext_debug_logger.debug("LoggingContext(%s).__enter__", self.name) - old_context = set_current_context(self) - if self.previous_context != old_context: - logcontext_error( - "Expected previous context %r, found %r" - % ( - self.previous_context, - old_context, - ) - ) - return self - - def __exit__( - self, - type: type[BaseException] | None, - value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - """Restore the logging context in thread local storage to the state it - was before this context was entered. - Returns: - None to avoid suppressing any exceptions that were thrown. - """ - logcontext_debug_logger.debug( - "LoggingContext(%s).__exit__ --> %s", self.name, self.previous_context - ) - current = set_current_context(self.previous_context) - if current is not self: - if current is SENTINEL_CONTEXT: - logcontext_error("Expected logging context %s was lost" % (self,)) - else: - logcontext_error( - "Expected logging context %s but found %s" % (self, current) - ) - - # the fact that we are here suggests that the caller thinks that everything - # is done and dusted for this logcontext, and further activity will not get - # recorded against the correct metrics. - self.finished = True - - def start(self, rusage: "resource.struct_rusage | None") -> None: - """ - Record that this logcontext is currently running. - - This should not be called directly: use set_current_context - - Args: - rusage: the resources used by the current thread, at the point of - switching to this logcontext. May be None if this platform doesn't - support getrusuage. - """ - if get_thread_id() != self.main_thread: - logcontext_error("Started logcontext %s on different thread" % (self,)) - return - - if self.finished: - logcontext_error("Re-starting finished log context %s" % (self,)) - - # If we haven't already started record the thread resource usage so - # far - if self.usage_start: - logcontext_error("Re-starting already-active log context %s" % (self,)) - else: - self.usage_start = rusage - - def stop(self, rusage: "resource.struct_rusage | None") -> None: - """ - Record that this logcontext is no longer running. - - This should not be called directly: use set_current_context - - Args: - rusage: the resources used by the current thread, at the point of - switching away from this logcontext. May be None if this platform - doesn't support getrusuage. - """ - - try: - if get_thread_id() != self.main_thread: - logcontext_error("Stopped logcontext %s on different thread" % (self,)) - return - - if not rusage: - return - - # Record the cpu used since we started - if not self.usage_start: - logcontext_error( - "Called stop on logcontext %s without recording a start rusage" - % (self,) - ) - return - - utime_delta, stime_delta = self._get_cputime(rusage) - self.add_cputime(utime_delta, stime_delta) - finally: - self.usage_start = None - - def get_resource_usage(self) -> ContextResourceUsage: - """Get resources used by this logcontext so far. - - Returns: - A *copy* of the object tracking resource usage so far - """ - # we always return a copy, for consistency - res = self._resource_usage.copy() - - # If we are on the correct thread and we're currently running then we - # can include resource usage so far. - is_main_thread = get_thread_id() == self.main_thread - if self.usage_start and is_main_thread: - rusage = get_thread_resource_usage() - assert rusage is not None - utime_delta, stime_delta = self._get_cputime(rusage) - res.ru_utime += utime_delta - res.ru_stime += stime_delta - - return res - - def _get_cputime(self, current: "resource.struct_rusage") -> tuple[float, float]: - """Get the cpu usage time between start() and the given rusage - - Args: - rusage: the current resource usage - - Returns: tuple[float, float]: seconds in user mode, seconds in system mode - """ - assert self.usage_start is not None - - utime_delta = current.ru_utime - self.usage_start.ru_utime - stime_delta = current.ru_stime - self.usage_start.ru_stime - - # sanity check - if utime_delta < 0: - logger.error( - "utime went backwards! %f < %f", - current.ru_utime, - self.usage_start.ru_utime, - ) - utime_delta = 0 - - if stime_delta < 0: - logger.error( - "stime went backwards! %f < %f", - current.ru_stime, - self.usage_start.ru_stime, - ) - stime_delta = 0 - - return utime_delta, stime_delta - - def add_cputime(self, utime_delta: float, stime_delta: float) -> None: - """Update the CPU time usage of this context (and any parents, recursively). - - Args: - utime_delta: additional user time, in seconds, spent in this context. - stime_delta: additional system time, in seconds, spent in this context. - """ - self._resource_usage.ru_utime += utime_delta - self._resource_usage.ru_stime += stime_delta - if self.parent_context: - self.parent_context.add_cputime(utime_delta, stime_delta) - - def add_database_transaction(self, duration_sec: float) -> None: - """Record the use of a database transaction and the length of time it took. - - Args: - duration_sec: The number of seconds the database transaction took. - """ - if duration_sec < 0: - raise ValueError("DB txn time can only be non-negative") - self._resource_usage.db_txn_count += 1 - self._resource_usage.db_txn_duration_sec += duration_sec - if self.parent_context: - self.parent_context.add_database_transaction(duration_sec) - - def add_database_scheduled(self, sched_sec: float) -> None: - """Record a use of the database pool - - Args: - sched_sec: number of seconds it took us to get a connection - """ - if sched_sec < 0: - raise ValueError("DB scheduling time can only be non-negative") - self._resource_usage.db_sched_duration_sec += sched_sec - if self.parent_context: - self.parent_context.add_database_scheduled(sched_sec) - - def record_event_fetch(self, event_count: int) -> None: - """Record a number of events being fetched from the db - - Args: - event_count: number of events being fetched - """ - self._resource_usage.evt_db_fetch_count += event_count - if self.parent_context: - self.parent_context.record_event_fetch(event_count) + # The cast is needed because mypy cannot narrow the Union via the + # `is SENTINEL_CONTEXT` identity check; Rust enforces the type at runtime. + previous = _rust_set_current_context( + None if context is SENTINEL_CONTEXT else cast(LoggingContext, context) + ) + return SENTINEL_CONTEXT if previous is None else previous class LoggingContextFilter(logging.Filter): @@ -733,39 +359,6 @@ class PreserveLoggingContext: ) -_thread_local = threading.local() -_thread_local.current_context = SENTINEL_CONTEXT - - -def current_context() -> LoggingContextOrSentinel: - """Get the current logging context from thread local storage""" - return getattr(_thread_local, "current_context", SENTINEL_CONTEXT) - - -def set_current_context(context: LoggingContextOrSentinel) -> LoggingContextOrSentinel: - """Set the current logging context in thread local storage - Args: - context: The context to activate. - - Returns: - The context that was previously active - """ - # everything blows up if we allow current_context to be set to None, so sanity-check - # that now. - if context is None: - raise TypeError("'context' argument may not be None") - - current = current_context() - - if current is not context: - rusage = get_thread_resource_usage() - current.stop(rusage) - _thread_local.current_context = context - context.start(rusage) - - return current - - def nested_logging_context(suffix: str) -> LoggingContext: """Creates a new logging context as a child of another. diff --git a/synapse/media/media_storage.py b/synapse/media/media_storage.py index 4c942f07f8..6f4b63741c 100644 --- a/synapse/media/media_storage.py +++ b/synapse/media/media_storage.py @@ -507,6 +507,12 @@ class SpamMediaException(NotFoundError): """ +class QuarantinedMediaException(NotFoundError): + """The media matched the hash of existing quarantined media, so we 404 the + request (in the same way as if the media itself had been quarantined). + """ + + @attr.s(slots=True, auto_attribs=True) class ReadableFileWrapper: """Wrapper that allows reading a file in chunks, yielding to the reactor, diff --git a/synapse/media/oembed.py b/synapse/media/oembed.py index 7e44072130..03e215530c 100644 --- a/synapse/media/oembed.py +++ b/synapse/media/oembed.py @@ -21,16 +21,21 @@ import html import logging import urllib.parse -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING import attr -from synapse.media.preview_html import parse_html_description +from synapse.media.preview_html import ( + NON_BLANK, + decode_body, + get_attribute, + parse_html_description, +) from synapse.types import JsonDict from synapse.util.json import json_decoder if TYPE_CHECKING: - from lxml import etree + from bs4 import BeautifulSoup from synapse.server import HomeServer @@ -105,35 +110,25 @@ class OEmbedProvider: # No match. return None - def autodiscover_from_html(self, tree: "etree._Element") -> str | None: + def autodiscover_from_html(self, soup: "BeautifulSoup") -> str | None: """ Search an HTML document for oEmbed autodiscovery information. Args: - tree: The parsed HTML body. + soup: The parsed HTML body. Returns: The URL to use for oEmbed information, or None if no URL was found. """ # Search for link elements with the proper rel and type attributes. - # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. - for tag in cast( - list["etree._Element"], - tree.xpath("//link[@rel='alternate'][@type='application/json+oembed']"), - ): - if "href" in tag.attrib: - return cast(str, tag.attrib["href"]) - - # Some providers (e.g. Flickr) use alternative instead of alternate. - # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. - for tag in cast( - list["etree._Element"], - tree.xpath("//link[@rel='alternative'][@type='application/json+oembed']"), - ): - if "href" in tag.attrib: - return cast(str, tag.attrib["href"]) - - return None + # Some providers (e.g. Flickr) use `alternative` instead of `alternate`. + tag = soup.find( + "link", + rel=("alternate", "alternative"), + type="application/json+oembed", + href=NON_BLANK, + ) + return get_attribute(tag, "href") if tag else None def parse_oembed_response(self, url: str, raw_body: bytes) -> OEmbedResult: """ @@ -196,7 +191,7 @@ class OEmbedProvider: if oembed_type == "rich": html_str = oembed.get("html") if isinstance(html_str, str): - calc_description_and_urls(open_graph_response, html_str) + calc_description_and_urls(open_graph_response, html_str, url) elif oembed_type == "photo": # If this is a photo, use the full image, not the thumbnail. @@ -208,7 +203,7 @@ class OEmbedProvider: open_graph_response["og:type"] = "video.other" html_str = oembed.get("html") if html_str and isinstance(html_str, str): - calc_description_and_urls(open_graph_response, oembed["html"]) + calc_description_and_urls(open_graph_response, oembed["html"], url) for size in ("width", "height"): val = oembed.get(size) if type(val) is int: # noqa: E721 @@ -223,55 +218,45 @@ class OEmbedProvider: return OEmbedResult(open_graph_response, author_name, cache_age) -def _fetch_urls(tree: "etree._Element", tag_name: str) -> list[str]: - results = [] - # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. - for tag in cast(list["etree._Element"], tree.xpath("//*/" + tag_name)): - if "src" in tag.attrib: - results.append(cast(str, tag.attrib["src"])) - return results +def _fetch_url(soup: "BeautifulSoup", tag_name: str) -> str | None: + tag = soup.find(tag_name, src=NON_BLANK) + return get_attribute(tag, "src") if tag else None -def calc_description_and_urls(open_graph_response: JsonDict, html_body: str) -> None: +def calc_description_and_urls( + open_graph_response: JsonDict, html_body: str, url: str +) -> None: """ Calculate description for an HTML document. - This uses lxml to convert the HTML document into plaintext. If errors + This uses BeautifulSoup to convert the HTML document into plaintext. If errors occur during processing of the document, an empty response is returned. Args: open_graph_response: The current Open Graph summary. This is updated with additional fields. html_body: The HTML document, as bytes. - - Returns: - The summary + url: The URL which is being previewed (not the one which was requested). """ + soup = decode_body(html_body, url) + # If there's no body, nothing useful is going to be found. - if not html_body: - return - - from lxml import etree - - # Create an HTML parser. If this fails, log and return no metadata. - parser = etree.HTMLParser(recover=True, encoding="utf-8") - - # Attempt to parse the body. If this fails, log and return no metadata. - tree = etree.fromstring(html_body, parser) - - # The data was successfully parsed, but no tree was found. - if tree is None: + if not soup: return # Attempt to find interesting URLs (images, videos, embeds). if "og:image" not in open_graph_response: - image_urls = _fetch_urls(tree, "img") - if image_urls: - open_graph_response["og:image"] = image_urls[0] + image_url = _fetch_url(soup, "img") + if image_url: + open_graph_response["og:image"] = image_url - video_urls = _fetch_urls(tree, "video") + _fetch_urls(tree, "embed") - if video_urls: - open_graph_response["og:video"] = video_urls[0] + video_url = _fetch_url(soup, "video") + if video_url: + open_graph_response["og:video"] = video_url + else: + embed_url = _fetch_url(soup, "embed") + if embed_url: + open_graph_response["og:video"] = embed_url - description = parse_html_description(tree) + description = parse_html_description(soup) if description: open_graph_response["og:description"] = description diff --git a/synapse/media/preview_html.py b/synapse/media/preview_html.py index 4f9315f4c9..b48df36d28 100644 --- a/synapse/media/preview_html.py +++ b/synapse/media/preview_html.py @@ -18,108 +18,27 @@ # [This file includes modifications made by New Vector Limited] # # -import codecs import logging import re -from typing import ( - TYPE_CHECKING, - Callable, - Generator, - Iterable, - Optional, - cast, -) +from typing import TYPE_CHECKING, Callable, Generator, Iterable, Optional if TYPE_CHECKING: - from lxml import etree + from bs4 import BeautifulSoup + from bs4.element import PageElement, Tag logger = logging.getLogger(__name__) -_charset_match = re.compile( - rb'<\s*meta[^>]*charset\s*=\s*"?([a-z0-9_-]+)"?', flags=re.I -) -_xml_encoding_match = re.compile( - rb'\s*<\s*\?\s*xml[^>]*encoding="([a-z0-9_-]+)"', flags=re.I -) _content_type_match = re.compile(r'.*; *charset="?(.*?)"?(;|$)', flags=re.I) # Certain elements aren't meant for display. ARIA_ROLES_TO_IGNORE = {"directory", "menu", "menubar", "toolbar"} - -def _normalise_encoding(encoding: str) -> str | None: - """Use the Python codec's name as the normalised entry.""" - try: - return codecs.lookup(encoding).name - except LookupError: - return None +NON_BLANK = re.compile(".+") -def _get_html_media_encodings(body: bytes, content_type: str | None) -> Iterable[str]: +def decode_body(body: bytes | str, uri: str) -> Optional["BeautifulSoup"]: """ - Get potential encoding of the body based on the (presumably) HTML body or the content-type header. - - The precedence used for finding a character encoding is: - - 1. tag with a charset declared. - 2. The XML document's character encoding attribute. - 3. The Content-Type header. - 4. Fallback to utf-8. - 5. Fallback to windows-1252. - - This roughly follows the algorithm used by BeautifulSoup's bs4.dammit.EncodingDetector. - - Args: - body: The HTML document, as bytes. - content_type: The Content-Type header. - - Returns: - The character encoding of the body, as a string. - """ - # There's no point in returning an encoding more than once. - attempted_encodings: set[str] = set() - - # Limit searches to the first 1kb, since it ought to be at the top. - body_start = body[:1024] - - # Check if it has an encoding set in a meta tag. - match = _charset_match.search(body_start) - if match: - encoding = _normalise_encoding(match.group(1).decode("ascii")) - if encoding: - attempted_encodings.add(encoding) - yield encoding - - # TODO Support - - # Check if it has an XML document with an encoding. - match = _xml_encoding_match.match(body_start) - if match: - encoding = _normalise_encoding(match.group(1).decode("ascii")) - if encoding and encoding not in attempted_encodings: - attempted_encodings.add(encoding) - yield encoding - - # Check the HTTP Content-Type header for a character set. - if content_type: - content_match = _content_type_match.match(content_type) - if content_match: - encoding = _normalise_encoding(content_match.group(1)) - if encoding and encoding not in attempted_encodings: - attempted_encodings.add(encoding) - yield encoding - - # Finally, fallback to UTF-8, then windows-1252. - for fallback in ("utf-8", "cp1252"): - if fallback not in attempted_encodings: - yield fallback - - -def decode_body( - body: bytes, uri: str, content_type: str | None = None -) -> Optional["etree._Element"]: - """ - This uses lxml to parse the HTML document. + This uses BeautifulSoup to parse the HTML document. Args: body: The HTML document, as bytes. @@ -133,54 +52,65 @@ def decode_body( if not body: return None - # The idea here is that multiple encodings are tried until one works. - # Unfortunately the result is never used and then LXML will decode the string - # again with the found encoding. - for encoding in _get_html_media_encodings(body, content_type): - try: - body.decode(encoding) - except Exception: - pass - else: - break - else: + from bs4 import BeautifulSoup + from bs4.builder import ParserRejectedMarkup + + try: + soup = BeautifulSoup(body, "html.parser") + # If an empty document is returned, convert to None. + if not len(soup): + return None + return soup + except ParserRejectedMarkup: logger.warning("Unable to decode HTML body for %s", uri) return None - from lxml import etree - # Create an HTML parser. - parser = etree.HTMLParser(recover=True, encoding=encoding) +def get_attribute(tag: "Tag", attribute_name: str) -> str: + """ + Get an attribute from a beautifulsoup tag. - # Attempt to parse the body. With `lxml` 6.0.0+, this will be an empty HTML - # tree if the body was successfully parsed, but no tree was found. In - # previous `lxml` versions, `etree.fromstring` would return `None` in that - # case. - html_tree = etree.fromstring(body, parser) + Fetching an attribute may return either a string or list of strings depending + on if the attribute is a "multi-valued" attribute. - # Account for the above referenced case where `html_tree` is an HTML tree - # with an empty body. If so, return None. - if html_tree is not None and html_tree.tag == "html": - # If the tree has only a single element and it's empty, then - # return None. - body_el = html_tree.find("body") - if body_el is not None and len(html_tree) == 1: - # Extract the content of the body tag as text. - body_text = "".join(cast(Iterable[str], body_el.itertext())) + The multi-valued attributes are never used in the HTML preview code, but this + function helps enforce type safety without casts. - # Strip any undecodable Unicode characters and whitespace. - body_text = body_text.strip("\ufffd").strip() + Args: + tag: The Tag object to get the attribute from. + attribute_name: The name of the attribute to get. - # If there's no text left, and there were no child tags, - # then we consider the tag empty. - if not body_text and len(body_el) == 0: - return None + Returns: + The attribute value as a string. + """ + attribute = tag[attribute_name] + assert isinstance(attribute, str), ( + f"Expected attribute {attribute_name} to have a string value" + ) + return attribute - return html_tree + +def get_float_attribute(tag: "Tag", attribute_name: str) -> float: + """ + Get an attribute from a beautifulsoup tag and parses it as a float, if it cannot be + parsed then return 0.. + + Args: + tag: The Tag object to get the attribute from. + attribute_name: The name of the attribute to get. + + Returns: + The attribute value as a float or 0 if it cannot be parsed. + """ + attribute = get_attribute(tag, attribute_name) + try: + return float(attribute) + except ValueError: + return 0 def _get_meta_tags( - tree: "etree._Element", + soup: "BeautifulSoup", property: str, prefix: str, property_mapper: Callable[[str], str | None] | None = None, @@ -189,7 +119,7 @@ def _get_meta_tags( Search for meta tags prefixed with a particular string. Args: - tree: The parsed HTML document. + soup: The parsed HTML document. property: The name of the property which contains the tag name, e.g. "property" for Open Graph. prefix: The prefix on the property to search for, e.g. "og" for Open Graph. @@ -199,15 +129,10 @@ def _get_meta_tags( Returns: A map of tag name to value. """ - # This actually returns dict[str, str], but the caller sets this as a variable - # which is dict[str, str | None]. results: dict[str, str | None] = {} # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. - for tag in cast( - list["etree._Element"], - tree.xpath( - f"//*/meta[starts-with(@{property}, '{prefix}:')][@content][not(@content='')]" - ), + for tag in soup.find_all( + "meta", attrs={property: re.compile(rf"^{prefix}:")}, content=NON_BLANK ): # if we've got more than 50 tags, someone is taking the piss if len(results) >= 50: @@ -217,7 +142,7 @@ def _get_meta_tags( ) return {} - key = cast(str, tag.attrib[property]) + key = get_attribute(tag, property) if property_mapper: new_key = property_mapper(key) # None is a special value used to ignore a value. @@ -225,7 +150,7 @@ def _get_meta_tags( continue key = new_key - results[key] = cast(str, tag.attrib["content"]) + results[key] = get_attribute(tag, "content") return results @@ -250,15 +175,14 @@ def _map_twitter_to_open_graph(key: str) -> str | None: return "og" + key[7:] -def parse_html_to_open_graph(tree: "etree._Element") -> dict[str, str | None]: +def parse_html_to_open_graph(soup: "BeautifulSoup") -> dict[str, str | None]: """ - Parse the HTML document into an Open Graph response. + Calculate metadata for an HTML document. - This uses lxml to search the HTML document for Open Graph data (or - synthesizes it from the document). + This uses BeautifulSoup to search the HTML document for Open Graph data. Args: - tree: The parsed HTML document. + soup: The parsed HTML document. Returns: The Open Graph response as a dictionary. @@ -278,12 +202,13 @@ def parse_html_to_open_graph(tree: "etree._Element") -> dict[str, str | None]: # "og:video:height" : "720", # "og:video:secure_url": "https://www.youtube.com/v/LXDBoHyjmtw?version=3", - ogRoot = _get_meta_tags(tree, "property", "og") + # TODO: grab article: meta tags too, e.g.: + ogRoot = _get_meta_tags(soup, "property", "og") # https://ogp.me/#type_article - ogArticle = _get_meta_tags(tree, "property", "article") + ogArticle = _get_meta_tags(soup, "property", "article") # https://ogp.me/#type_profile - ogProfile = _get_meta_tags(tree, "property", "profile") + ogProfile = _get_meta_tags(soup, "property", "profile") # Merge as-is og = ogRoot | ogArticle | ogProfile @@ -296,7 +221,7 @@ def parse_html_to_open_graph(tree: "etree._Element") -> dict[str, str | None]: # Twitter cards tags also duplicate Open Graph tags. # # See https://developer.twitter.com/en/docs/twitter-for-websites/cards/guides/getting-started - twitter = _get_meta_tags(tree, "name", "twitter", _map_twitter_to_open_graph) + twitter = _get_meta_tags(soup, "name", "twitter", _map_twitter_to_open_graph) # Merge the Twitter values with the Open Graph values, but do not overwrite # information from Open Graph tags. for key, value in twitter.items(): @@ -305,73 +230,67 @@ def parse_html_to_open_graph(tree: "etree._Element") -> dict[str, str | None]: if "og:title" not in og: # Attempt to find a title from the title tag, or the biggest header on the page. - # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. - title = cast( - list["etree._ElementUnicodeResult"], - tree.xpath("((//title)[1] | (//h1)[1] | (//h2)[1] | (//h3)[1])/text()"), - ) - if title: - og["og:title"] = title[0].strip() + # + # mypy doesn't like passing both name and string, but it is used to ignore + # empty elements. + title = soup.find(("title", "h1", "h2", "h3"), string=True) + if title and title.string: + og["og:title"] = title.string.strip() else: og["og:title"] = None if "og:image" not in og: - # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. - meta_image = cast( - list["etree._ElementUnicodeResult"], - tree.xpath( - "//*/meta[translate(@itemprop, 'IMAGE', 'image')='image'][not(@content='')]/@content[1]" - ), + # Check microdata for an image. + meta_image = soup.find( + "meta", itemprop=re.compile("image", re.I), content=NON_BLANK ) # If a meta image is found, use it. if meta_image: - og["og:image"] = meta_image[0] + og["og:image"] = get_attribute(meta_image, "content") else: # Try to find images which are larger than 10px by 10px. - # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. # # TODO: consider inlined CSS styles as well as width & height attribs - images = cast( - list["etree._Element"], - tree.xpath("//img[@src][number(@width)>10][number(@height)>10]"), + raw_images = soup.find_all( + "img", src=NON_BLANK, width=NON_BLANK, height=NON_BLANK ) images = sorted( - images, + filter( + lambda tag: get_float_attribute(tag, "width") > 10 + and get_float_attribute(tag, "height") > 10, + raw_images, + ), key=lambda i: ( - -1 * float(i.attrib["width"]) * float(i.attrib["height"]) + -1 + * get_float_attribute(i, "width") + * get_float_attribute(i, "height") ), ) # If no images were found, try to find *any* images. if not images: - # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. - images = cast(list["etree._Element"], tree.xpath("//img[@src][1]")) + images = soup.find_all("img", src=NON_BLANK, limit=1) if images: - og["og:image"] = cast(str, images[0].attrib["src"]) + og["og:image"] = get_attribute(images[0], "src") # Finally, fallback to the favicon if nothing else. else: - # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. - favicons = cast( - list["etree._ElementUnicodeResult"], - tree.xpath("//link[@href][contains(@rel, 'icon')]/@href[1]"), - ) - if favicons: - og["og:image"] = favicons[0] + favicon = soup.find("link", href=NON_BLANK, rel="icon") + if favicon: + og["og:image"] = get_attribute(favicon, "href") if "og:description" not in og: # Check the first meta description tag for content. - # Cast: the type returned by xpath depends on the xpath expression: mypy can't deduce this. - meta_description = cast( - list["etree._ElementUnicodeResult"], - tree.xpath( - "//*/meta[translate(@name, 'DESCRIPTION', 'description')='description'][not(@content='')]/@content[1]" - ), + meta_description = soup.find( + "meta", + attrs={"name": re.compile("description", re.I)}, + content=NON_BLANK, ) + # If a meta description is found with content, use it. if meta_description: - og["og:description"] = meta_description[0] + og["og:description"] = get_attribute(meta_description, "content") else: - og["og:description"] = parse_html_description(tree) + og["og:description"] = parse_html_description(soup) elif og["og:description"]: # This must be a non-empty string at this point. assert isinstance(og["og:description"], str) @@ -382,7 +301,7 @@ def parse_html_to_open_graph(tree: "etree._Element") -> dict[str, str | None]: return og -def parse_html_description(tree: "etree._Element") -> str | None: +def parse_html_description(soup: "BeautifulSoup") -> str | None: """ Calculate a text description based on an HTML document. @@ -395,16 +314,14 @@ def parse_html_description(tree: "etree._Element") -> str | None: This is a very very very coarse approximation to a plain text render of the page. Args: - tree: The parsed HTML document. + soup: The parsed HTML document. Returns: The plain text description, or None if one cannot be generated. """ - # We don't just use XPATH here as that is slow on some machines. - - from lxml import etree TAGS_TO_REMOVE = { + "head", "header", "nav", "aside", @@ -418,74 +335,60 @@ def parse_html_description(tree: "etree._Element") -> str | None: "canvas", "img", "picture", - # etree.Comment is a function which creates an etree._Comment element. - # The "tag" attribute of an etree._Comment instance is confusingly the - # etree.Comment function instead of a string. - etree.Comment, } # Split all the text nodes into paragraphs (by splitting on new # lines) text_nodes = ( re.sub(r"\s+", "\n", el).strip() - for el in _iterate_over_text(tree.find("body"), TAGS_TO_REMOVE) + for el in _iterate_over_text(soup, TAGS_TO_REMOVE) ) return summarize_paragraphs(text_nodes) def _iterate_over_text( - tree: Optional["etree._Element"], - tags_to_ignore: set[object], + soup: "Tag", + tags_to_ignore: Iterable[str], stack_limit: int = 1024, ) -> Generator[str, None, None]: - """Iterate over the tree returning text nodes in a depth first fashion, + """Iterate over the document returning text nodes in a depth first fashion, skipping text nodes inside certain tags. Args: - tree: The parent element to iterate. Can be None if there isn't one. + soup: The parent element to iterate. tags_to_ignore: Set of tags to ignore stack_limit: Maximum stack size limit for depth-first traversal. Nodes will be dropped if this limit is hit, which may truncate the textual result. Intended to limit the maximum working memory when generating a preview. """ + from bs4.element import NavigableString, Tag - if tree is None: - return - - # This is a stack whose items are elements to iterate over *or* strings + # This is basically a stack that we extend using itertools.chain. + # This will either consist of an element to iterate over *or* a string # to be returned. - elements: list[str | "etree._Element"] = [tree] + elements: list["PageElement"] = [soup] while elements: el = elements.pop() - if isinstance(el, str): - yield el - elif el.tag not in tags_to_ignore: + # Do not consider sub-classes of NavigableString since those represent + # stylesheets, etc. + if type(el) == NavigableString: # noqa: E721 + yield str(el) + elif isinstance(el, Tag) and el.name not in tags_to_ignore: # If the element isn't meant for display, ignore it. if el.get("role") in ARIA_ROLES_TO_IGNORE: continue - # el.text is the text before the first child, so we can immediately - # return it if the text exists. - if el.text: - yield el.text - - # We add to the stack all the element's children, interspersed with - # each child's tail text (if it exists). + # We add to the stack all the element's children. # # We iterate in reverse order so that earlier pieces of text appear # closer to the top of the stack. - for child in el.iterchildren(reversed=True): + for child in reversed(el.contents): if len(elements) > stack_limit: # We've hit our limit for working memory break - if child.tail: - # The tail text of a node is text that comes *after* the node, - # so we always include it even if we ignore the child node. - elements.append(child.tail) - elements.append(child) diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index a75a420750..8e90aeb8de 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -31,6 +31,7 @@ from synapse.api.errors import Codes, NotFoundError, SynapseError, cs_error from synapse.config.repository import THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP from synapse.http.server import respond_with_json from synapse.http.site import SynapseRequest +from synapse.logging.context import defer_to_thread from synapse.logging.opentracing import trace from synapse.media._base import ( FileInfo, @@ -533,10 +534,11 @@ class ThumbnailProvider: assert file_info.thumbnail is not None if for_federation: + f = await defer_to_thread(self.hs.get_reactor(), open, file_path, "rb") await respond_with_multipart_responder( self.hs.get_clock(), request, - FileResponder(self.hs, open(file_path, "rb")), + FileResponder(self.hs, f), file_info.thumbnail.type, file_info.thumbnail.length, None, diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 7782905a7a..4463034678 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -41,7 +41,11 @@ from synapse.api.errors import Codes, SynapseError from synapse.http.client import SimpleHttpClient from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.media._base import FileInfo, get_filename_from_headers -from synapse.media.media_storage import MediaStorage, SHA256TransparentIOWriter +from synapse.media.media_storage import ( + MediaStorage, + QuarantinedMediaException, + SHA256TransparentIOWriter, +) from synapse.media.oembed import OEmbedProvider from synapse.media.preview_html import decode_body, parse_html_to_open_graph from synapse.types import JsonDict, UserID @@ -65,6 +69,30 @@ ONE_DAY = 24 * ONE_HOUR IMAGE_CACHE_EXPIRY_MS = 2 * ONE_DAY +def _try_remove_parent_dirs(dirs: Iterable[str]) -> None: + """Attempt to remove the given chain of parent directories + + Args: + dirs: The list of directory paths to delete, with children appearing + before their parents. + """ + for dir in dirs: + try: + os.rmdir(dir) + except FileNotFoundError: + # Already deleted, continue with deleting the rest + pass + except OSError as e: + # Failed, skip deleting the rest of the parent dirs + if e.errno != errno.ENOTEMPTY: + logger.warning( + "Failed to remove media directory while clearing url preview cache: %r: %s", + dir, + e, + ) + break + + @attr.s(slots=True, frozen=True, auto_attribs=True) class DownloadResult: length: int @@ -294,16 +322,16 @@ class UrlPreviewer: # define our OG response for this media elif _is_html(media_info.media_type): - # TODO: somehow stop a big HTML tree from exploding synapse's RAM + # TODO: somehow stop a big HTML document from exploding synapse's RAM with open(media_info.filename, "rb") as file: body = file.read() - tree = decode_body(body, media_info.uri, media_info.media_type) - if tree is not None: + soup = decode_body(body, media_info.uri) + if soup is not None: # Check if this HTML document points to oEmbed information and # defer to that. - oembed_url = self._oembed.autodiscover_from_html(tree) + oembed_url = self._oembed.autodiscover_from_html(soup) og_from_oembed: JsonDict = {} # Only download to the oEmbed URL if it is allowed. if oembed_url: @@ -329,7 +357,7 @@ class UrlPreviewer: # Parse Open Graph information from the HTML in case the oEmbed # response failed or is incomplete. - og_from_html = parse_html_to_open_graph(tree) + og_from_html = parse_html_to_open_graph(soup) # Compile an Open Graph response by combining the oEmbed response # and the information from the HTML, with information in the HTML @@ -613,15 +641,20 @@ class UrlPreviewer: else: download_result = await self._download_url(url, sha256writer.wrap()) + sha256 = sha256writer.hexdigest() + if await self.store.get_is_hash_quarantined(sha256): + # The media matches something that has been quarantined. Rather than + # storing it (and handing the client an mxc:// URI that will always + # 404), throw the download away entirely and let the caller decide + # how to cope without it. + logger.warning( + "Discarding %s from URL preview as it matched existing quarantined media", + url, + ) + self._delete_url_cache_file(file_id) + raise QuarantinedMediaException() + try: - sha256 = sha256writer.hexdigest() - should_quarantine = await self.store.get_is_hash_quarantined(sha256) - - if should_quarantine: - logger.warning( - "Media has been automatically quarantined as it matched existing quarantined media" - ) - time_now_ms = self.clock.time_msec() await self.store.store_local_media( @@ -633,14 +666,13 @@ class UrlPreviewer: user_id=user, url_cache=url, sha256=sha256, - quarantined_by="system" if should_quarantine else None, ) except Exception as e: logger.error("Error handling downloaded %s: %r", url, e) - # TODO: we really ought to delete the downloaded file in this - # case, since we won't have recorded it in the db, and will - # therefore not expire it. + # We won't have recorded the file in the db, and will + # therefore not expire it; delete it now. + self._delete_url_cache_file(file_id) raise return MediaInfo( @@ -656,6 +688,28 @@ class UrlPreviewer: etag=download_result.etag, ) + def _delete_url_cache_file(self, file_id: str) -> None: + """Remove a file from the url cache directory, along with any parent + directories left empty by its removal. + + Note that url cache files are never handed to the storage providers (see + `StorageProviderWrapper.store_file`), so the local file is the only copy. + """ + fname = self.filepaths.url_cache_filepath(file_id) + try: + os.remove(fname) + except FileNotFoundError: + pass # If the path doesn't exist, meh + except OSError as e: + logger.warning( + "Failed to remove media from url preview cache: %r: %s", fname, e + ) + return + + _try_remove_parent_dirs( + self.filepaths.url_cache_filepath_dirs_to_delete(file_id) + ) + async def _precache_image_url( self, user: UserID, media_info: MediaInfo, og: JsonDict ) -> None: @@ -760,29 +814,6 @@ class UrlPreviewer: logger.debug("Running url preview cache expiry") - def try_remove_parent_dirs(dirs: Iterable[str]) -> None: - """Attempt to remove the given chain of parent directories - - Args: - dirs: The list of directory paths to delete, with children appearing - before their parents. - """ - for dir in dirs: - try: - os.rmdir(dir) - except FileNotFoundError: - # Already deleted, continue with deleting the rest - pass - except OSError as e: - # Failed, skip deleting the rest of the parent dirs - if e.errno != errno.ENOTEMPTY: - logger.warning( - "Failed to remove media directory while clearing url preview cache: %r: %s", - dir, - e, - ) - break - # First we delete expired url cache entries media_ids = await self.store.get_expired_url_cache(now) @@ -804,7 +835,7 @@ class UrlPreviewer: removed_media.append(media_id) dirs = self.filepaths.url_cache_filepath_dirs_to_delete(media_id) - try_remove_parent_dirs(dirs) + _try_remove_parent_dirs(dirs) await self.store.delete_url_cache(removed_media) @@ -836,7 +867,7 @@ class UrlPreviewer: continue dirs = self.filepaths.url_cache_filepath_dirs_to_delete(media_id) - try_remove_parent_dirs(dirs) + _try_remove_parent_dirs(dirs) thumbnail_dir = self.filepaths.url_cache_thumbnail_directory(media_id) try: @@ -854,7 +885,7 @@ class UrlPreviewer: dirs = self.filepaths.url_cache_thumbnail_dirs_to_delete(media_id) # Note that one of the directories to be deleted has already been # removed by the `rmtree` above. - try_remove_parent_dirs(dirs) + _try_remove_parent_dirs(dirs) await self.store.delete_url_cache_media(removed_media) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 8ff2803455..d51e08aae5 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -57,8 +57,6 @@ from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics._types import Collector if TYPE_CHECKING: - import resource - # Old versions don't have `LiteralString` from typing_extensions import LiteralString @@ -506,7 +504,7 @@ class BackgroundProcessLoggingContext(LoggingContext): desc=name, server_name=server_name, ctx=self ) - def start(self, rusage: "resource.struct_rusage | None") -> None: + def start(self, rusage: "tuple[float, float] | None") -> None: """Log context has started running (again).""" super().start(rusage) diff --git a/synapse/notifier.py b/synapse/notifier.py index e24d0ef5a2..3ee0fc60d6 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -82,6 +82,14 @@ users_woken_by_stream_counter = Counter( labelnames=["stream", SERVER_NAME_LABEL], ) +wait_for_stream_token_timeout_counter = Counter( + "synapse_notifier_wait_for_stream_token_timeouts", + "Number of times we gave up waiting to catch up to a stream token, counted " + "once per lagging stream. `stream_key` is a `StreamToken` field name, which " + "is not always the replication stream name", + labelnames=["stream_key", SERVER_NAME_LABEL], +) + notifier_listeners_gauge = LaterGauge( name="synapse_notifier_listeners", @@ -103,6 +111,22 @@ notifier_users_gauge = LaterGauge( T = TypeVar("T") +def _describe_lagging_streams( + target_token: StreamToken, + current_token: StreamToken, + lagging_stream_keys: Collection[StreamKeyType], +) -> str: + """Describe how far each lagging stream has to go, for logging. + + e.g. `quarantined_media_key (at 4, waiting for 6)` + """ + return ", ".join( + f"{key.value} (at {current_token.get_field(key)}, " + f"waiting for {target_token.get_field(key)})" + for key in lagging_stream_keys + ) + + # TODO(paul): Should be shared somewhere def count(func: Callable[[T], bool], it: Iterable[T]) -> int: """Return the number of items in it for which func returns true.""" @@ -879,21 +903,40 @@ class Notifier: start = self.clock.time_msec() logged = False while True: + # Check if we are caught up to the stream token, if so return early. + # Equivalent to `is_before_or_eq`, except we can get the lagging + # stream keys for logging. current_token = self.event_sources.get_current_token() - if stream_token.is_before_or_eq(current_token): + lagging_stream_keys = stream_token.fields_strictly_after_token( + current_token + ) + if not lagging_stream_keys: return True now = self.clock.time_msec() # Timed out if now - start > 10_000: + for stream_key in lagging_stream_keys: + wait_for_stream_token_timeout_counter.labels( + stream_key=stream_key.value, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc() + + logger.warning( + "Timed out waiting for this worker's streams to advance to given token: %s", + _describe_lagging_streams( + stream_token, current_token, lagging_stream_keys + ), + ) return False if not logged: logger.info( - "Waiting for current token to reach %s; currently at %s", - stream_token, - current_token, + "Waiting for this worker's streams to advance to given token: %s", + _describe_lagging_streams( + stream_token, current_token, lagging_stream_keys + ), ) logged = True diff --git a/synapse/replication/http/delayed_events.py b/synapse/replication/http/delayed_events.py index 26eaf68dae..461c3e044c 100644 --- a/synapse/replication/http/delayed_events.py +++ b/synapse/replication/http/delayed_events.py @@ -58,5 +58,41 @@ class ReplicationAddedDelayedEventRestServlet(ReplicationEndpoint): return 200, {} +class ReplicationCancelDelayedEventsForUserRestServlet(ReplicationEndpoint): + """Cancel all of a local user's delayed events, on behalf of another worker. + + Request format: + + POST /_synapse/replication/cancel_delayed_events_for_user/:user_localpart + + {} + """ + + NAME = "cancel_delayed_events_for_user" + PATH_ARGS = ("user_localpart",) + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.handler = hs.get_delayed_events_handler() + + @staticmethod + async def _serialize_payload(user_localpart: str) -> JsonDict: # type: ignore[override] + """ + Args: + user_localpart: The localpart of the local user whose delayed events to cancel. + """ + return {} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_localpart: str + ) -> tuple[int, JsonDict]: + await self.handler.cancel_all_for_user(user_localpart) + + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ReplicationAddedDelayedEventRestServlet(hs).register(http_server) + ReplicationCancelDelayedEventsForUserRestServlet(hs).register(http_server) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index c0896b83e7..8a60f8df6f 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -50,6 +50,8 @@ from synapse.replication.tcp.streams._base import ( ) from synapse.replication.tcp.streams.events import ( EventsStream, + EventsStreamAllStateRow, + EventsStreamCurrentStateRow, EventsStreamEventRow, EventsStreamRow, ) @@ -191,6 +193,20 @@ class ReplicationDataHandler: # We shouldn't get multiple rows per token for events stream, so # we don't need to optimise this for multiple rows. for row in rows: + # If this is a server ACL event, clear the cache in the storage controller. + if row.type in ( + EventsStreamEventRow.TypeId, + EventsStreamCurrentStateRow.TypeId, + ): + if row.data.type == EventTypes.ServerACL: + self._state_storage_controller.get_server_acl_for_room.invalidate( + (row.data.room_id,) + ) + elif row.type == EventsStreamAllStateRow.TypeId: + self._state_storage_controller.get_server_acl_for_room.invalidate( + (row.data.room_id,) + ) + if row.type != EventsStreamEventRow.TypeId: # The row's data is an `EventsStreamCurrentStateRow`. # When we recompute the current state of a room based on forward @@ -238,11 +254,6 @@ class ReplicationDataHandler: row.data.event_id, row.data.room_id ) - # If this is a server ACL event, clear the cache in the storage controller. - if row.data.type == EventTypes.ServerACL: - self._state_storage_controller.get_server_acl_for_room.invalidate( - (row.data.room_id,) - ) elif stream_name == UnPartialStatedRoomStream.NAME: for row in rows: assert isinstance(row, UnPartialStatedRoomStreamRow) diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 705ea4172c..ec559ffe59 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -21,7 +21,7 @@ import logging from inspect import isawaitable -from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, cast import attr from txredisapi import ( @@ -404,7 +404,7 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory): self.synapse_outbound_redis_connection = outbound_redis_connection - def buildProtocol(self, addr: IAddress) -> RedisSubscriber: + def buildProtocol(self, addr: Optional[IAddress]) -> RedisSubscriber: p = super().buildProtocol(addr) p = cast(RedisSubscriber, p) diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 7d6886e0b8..545bc0b811 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -22,7 +22,7 @@ import logging import random -from typing import TYPE_CHECKING, Sequence +from typing import TYPE_CHECKING, Optional, Sequence from prometheus_client import Counter @@ -68,7 +68,9 @@ class ReplicationStreamProtocolFactory(ServerFactory): # listener config again or always starting a `ReplicationStreamer`.) hs.get_replication_streamer() - def buildProtocol(self, addr: IAddress) -> ServerReplicationStreamProtocol: + def buildProtocol( + self, addr: Optional[IAddress] + ) -> ServerReplicationStreamProtocol: return ServerReplicationStreamProtocol( self.hs, self.server_name, self.clock, self.command_handler ) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 43dab16598..ecf7fc45f4 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -237,6 +237,7 @@ class UserRestServletV2Get(RestServlet): self.pusher_pool = hs.get_pusherpool() self._msc3866_enabled = hs.config.experimental.msc3866.enabled self._all_user_types = hs.config.user_types.all_user_types + self._auth_delegation_enabled = hs.config.mas.enabled async def on_GET( self, request: SynapseRequest, user_id: str @@ -479,6 +480,15 @@ class UserRestServletV2(UserRestServletV2Get): return HTTPStatus.OK, user else: # create user + if self._auth_delegation_enabled: + raise SynapseError( + HTTPStatus.FORBIDDEN, + "User creation via the Admin API is not available when " + "Synapse is delegating authentication to MAS. Create the " + "user via MAS instead.", + errcode=Codes.FORBIDDEN, + ) + displayname = body.get("displayname", None) password_hash = None diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 7ffde7ac04..45e94a25b4 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -48,6 +48,7 @@ from synapse.logging.opentracing import set_tag from synapse.rest.client._base import client_patterns, interactive_auth_handler from synapse.types import JsonDict, StreamToken from synapse.types.rest import RequestBodyModel +from synapse.types.rest.client import UserInteractiveAuthBody from synapse.util.cancellation import cancellable if TYPE_CHECKING: @@ -518,6 +519,9 @@ class SigningKeyUploadServlet(RestServlet): requester = await self.auth.get_user_by_req(request) user_id = requester.user.to_string() body = parse_json_object_from_request(request) + # Only the `auth` field is validated here. The raw body is kept because + # `check_ui_auth` stores it when it creates a new UIA session. + validate_json_object(body, UserInteractiveAuthBody) ( is_cross_signing_setup, diff --git a/synapse/rest/client/login_token_request.py b/synapse/rest/client/login_token_request.py index f455e9c0b7..17cfc29f17 100644 --- a/synapse/rest/client/login_token_request.py +++ b/synapse/rest/client/login_token_request.py @@ -25,10 +25,15 @@ from typing import TYPE_CHECKING from synapse.api.ratelimiting import Ratelimiter from synapse.config.ratelimiting import RatelimitSettings from synapse.http.server import HttpServer -from synapse.http.servlet import RestServlet, parse_json_object_from_request +from synapse.http.servlet import ( + RestServlet, + parse_json_object_from_request, + validate_json_object, +) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns, interactive_auth_handler from synapse.types import JsonDict +from synapse.types.rest.client import UserInteractiveAuthBody if TYPE_CHECKING: from synapse.server import HomeServer @@ -92,6 +97,9 @@ class LoginTokenRequestServlet(RestServlet): async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) body = parse_json_object_from_request(request) + # Only the `auth` field is validated here. The raw body is kept because + # `check_ui_auth` stores it when it creates a new UIA session. + validate_json_object(body, UserInteractiveAuthBody) if self._require_ui_auth: await self.auth_handler.validate_user_via_ui_auth( diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 58a6c7c4e2..aadda870fd 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -36,7 +36,7 @@ from synapse.http.servlet import ( ) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, Requester, UserID from synapse.util.stringutils import is_namedspaced_grammar if TYPE_CHECKING: @@ -57,6 +57,32 @@ def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: return propagate +async def _auth_and_ratelimit_profile_lookup( + hs: "HomeServer", request: SynapseRequest +) -> Requester | None: + """Authenticate a profile lookup request and apply the `rc_profile` rate + limit to it. + + Authentication is optional unless `require_auth_for_profile_requests` is + set. The rate limit is applied per user if credentials were supplied, else + per client IP address. + + Returns: + The requester if the request was authenticated, else None. + """ + auth = hs.get_auth() + requester: Requester | None + if hs.config.server.require_auth_for_profile_requests: + requester = await auth.get_user_by_req(request) + else: + requester = await auth.get_optional_user_by_req(request, allow_guest=True) + + await hs.get_profile_lookup_ratelimiter().ratelimit( + requester, key=None if requester else request.getClientAddress().host + ) + return requester + + class ProfileRestServlet(RestServlet): PATTERNS = client_patterns("/profile/(?P[^/]*)$", v1=True) CATEGORY = "Event sending requests" @@ -70,11 +96,8 @@ class ProfileRestServlet(RestServlet): async def on_GET( self, request: SynapseRequest, user_id: str ) -> tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user + requester = await _auth_and_ratelimit_profile_lookup(self.hs, request) + requester_user = requester.user if requester else None if not UserID.is_valid(user_id): raise SynapseError( @@ -119,11 +142,8 @@ class ProfileFieldRestServlet(RestServlet): async def on_GET( self, request: SynapseRequest, user_id: str, field_name: str ) -> tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user + requester = await _auth_and_ratelimit_profile_lookup(self.hs, request) + requester_user = requester.user if requester else None if not UserID.is_valid(user_id): raise SynapseError( diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 8355d0c571..6e4bb400af 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -54,11 +54,13 @@ from synapse.http.servlet import ( assert_params_in_dict, parse_json_object_from_request, parse_string, + validate_json_object, ) from synapse.http.site import SynapseRequest from synapse.metrics import SERVER_NAME_LABEL, threepid_send_requests from synapse.push.mailer import Mailer from synapse.types import JsonDict +from synapse.types.rest.client import UserInteractiveAuthBody from synapse.util.duration import Duration from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.ratelimitutils import FederationRateLimiter @@ -469,6 +471,9 @@ class RegisterRestServlet(RestServlet): @interactive_auth_handler async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: body = parse_json_object_from_request(request) + # Only the `auth` field is validated here. The raw body is kept because + # `check_ui_auth` stores it when it creates a new UIA session. + validate_json_object(body, UserInteractiveAuthBody) client_addr = request.getClientAddress().host diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 1b8774fde3..4ffb6a89b3 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -217,7 +217,6 @@ class RoomStateEventRestServlet(RestServlet): self.clock = hs.get_clock() self._event_serializer = hs.get_event_client_serializer() self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker - self._msc4354_enabled = hs.config.experimental.msc4354_enabled def register(self, http_server: HttpServer) -> None: # /rooms/$roomid/state/$eventtype @@ -339,10 +338,6 @@ class RoomStateEventRestServlet(RestServlet): if requester.app_service_id: origin_server_ts = parse_integer(request, "ts") - sticky_duration_ms: int | None = None - if self._msc4354_enabled: - sticky_duration_ms = parse_integer(request, StickyEvent.QUERY_PARAM_NAME) - delay = _parse_request_for_delayed_event_delay(request) if delay is not None: delay_id = await self.delayed_events_handler.add( @@ -353,7 +348,7 @@ class RoomStateEventRestServlet(RestServlet): origin_server_ts=origin_server_ts, content=content, delay=delay, - sticky_duration_ms=sticky_duration_ms, + sticky_duration_ms=None, ) set_tag("delay_id", delay_id) @@ -381,10 +376,6 @@ class RoomStateEventRestServlet(RestServlet): "room_id": room_id, "sender": requester.user.to_string(), } - if sticky_duration_ms is not None: - event_dict[StickyEvent.EVENT_FIELD_NAME] = StickyEventField( - duration_ms=sticky_duration_ms - ) if state_key is not None: event_dict["state_key"] = state_key diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 08002a6708..83aa00d0b5 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -33,6 +33,7 @@ from synapse.events.utils import ( EventFormat, FilteredEvent, SerializeEventConfig, + strip_event, ) from synapse.handlers.presence import format_user_presence_state from synapse.handlers.sliding_sync import SlidingSyncConfig, SlidingSyncResult @@ -477,7 +478,8 @@ class SyncRestServlet(RestServlet): invited_state = [] invited_state = list(invited_state) - invited_state.append(invite) + # MSC4319: Add the invite itself + invited_state.append(strip_event(room.invite)) invited[room.room_id] = {"invite_state": {"events": invited_state}} return invited @@ -523,12 +525,11 @@ class SyncRestServlet(RestServlet): knocked_state = [] knocked_state = list(knocked_state) - # Append the actual knock membership event itself as well. This provides - # the client with: + # MSC4319: Append the actual knock membership event itself as well. This + # provides the client with: # # * A knock state event that they can use for easier internal tracking - # * The rough timestamp of when the knock occurred contained within the event - knocked_state.append(knock) + knocked_state.append(strip_event(room.knock)) # Build the `knock_state` dictionary, which will contain the state of the # room that the client has knocked on diff --git a/synapse/server.py b/synapse/server.py index 9422882167..266440534c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -177,6 +177,7 @@ from synapse.streams.events import EventSources from synapse.synapse_rust.handlers import RustHandlers from synapse.synapse_rust.msc4388_rendezvous import MSC4388RendezvousHandler from synapse.synapse_rust.rendezvous import RendezvousHandler +from synapse.synapse_rust.runtime import RustRuntime from synapse.types import DomainSpecificString, ISynapseReactor from synapse.util import SYNAPSE_VERSION from synapse.util.caches import CACHE_METRIC_REGISTRY @@ -722,6 +723,14 @@ class HomeServer(metaclass=abc.ABCMeta): cfg=self.config.ratelimiting.rc_registration, ) + @cache_in_self + def get_profile_lookup_ratelimiter(self) -> Ratelimiter: + return Ratelimiter( + store=self.get_datastores().main, + clock=self.get_clock(), + cfg=self.config.ratelimiting.rc_profile, + ) + @cache_in_self def get_federation_client(self) -> FederationClient: return FederationClient(self) @@ -964,6 +973,18 @@ class HomeServer(metaclass=abc.ABCMeta): def get_rust_handlers(self) -> RustHandlers: return RustHandlers(self) + @cache_in_self + def get_rust_runtime(self) -> RustRuntime: + """The per-homeserver state for the Rust side of Synapse: the tokio + thread pool, plus anything else Rust code keeps for the lifetime of the + homeserver. + + The tokio runtime is started lazily on first use, and shut down when + this homeserver is shut down. + """ + # TODO: make the number of worker threads configurable + return RustRuntime(hs=self, worker_threads=4) + @cache_in_self def get_event_sources(self) -> EventSources: return EventSources(self) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index b22f91b37c..e5cd386213 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -32,7 +32,6 @@ from typing import ( ) import attr -from immutabledict import immutabledict from prometheus_client import Counter, Histogram from synapse.api.constants import EventTypes @@ -53,6 +52,7 @@ from synapse.storage.databases.main.event_federation import StateDifference from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import StateMap, StrCollection from synapse.types.state import StateFilter +from synapse.util import MutableOverlayMapping from synapse.util.async_helpers import Linearizer from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.duration import Duration @@ -113,18 +113,13 @@ class _StateCacheEntry: # # This can be None if we have a `state_group` (as then we can fetch the # state from the DB.) - self._state: StateMap[str] | None = ( - immutabledict(state) if state is not None else None - ) - + self._state = state # the ID of a state group if one and only one is involved. # otherwise, None otherwise? self.state_group = state_group self.prev_group = prev_group - self.delta_ids: StateMap[str] | None = ( - immutabledict(delta_ids) if delta_ids is not None else None - ) + self.delta_ids = delta_ids async def get_state( self, @@ -174,14 +169,29 @@ class _StateCacheEntry: length = 0 if self._state: - length += len(self._state) + length += _state_map_size(self._state) if self.delta_ids: - length += len(self.delta_ids) + length += _state_map_size(self.delta_ids) return length or 1 # Make sure its not 0. +def _state_map_size(state_map: Mapping[Any, Any]) -> int: + """Estimate a proxy for the memory a state map holds, for sizing caches. + + Since state maps are often combinations of `ChainMap` and + `MutableOverlayMapping`, we look at the total number of entries across all + layers rather than just the number of distinct keys. This is both faster and + a more accurate proxy for memory usage. + """ + if isinstance(state_map, ChainMap): + return sum(_state_map_size(layer) for layer in state_map.maps) + if isinstance(state_map, MutableOverlayMapping): + return state_map.total_entries() + return len(state_map) + + class StateHandler: """Fetches bits of state from the stores, and does state resolution where necessary @@ -636,6 +646,31 @@ class StateResolutionHandler: ) ) + # The result of resolving a conflicted set of state, keyed on a digest + # of the inputs to `_resolve_conflicted_set`. See + # `v2._conflict_cache_key`. + # + # This is different to `_state_cache` above, which caches the resolved + # state based on the state groups. This cache aims to address the case + # where resolving across different state groups often produces the same + # conflicted set, which we can then cache. + # + # We bound the size of the cache based on the size calculated by + # `_state_map_size`, which calculates a proxy for a rough estimate of + # the memory footprint of a state map. + self._conflict_resolution_cache: ExpiringCache[bytes, StateMap[str]] = ( + ExpiringCache( + cache_name="state_conflict_resolution_cache", + server_name=self.server_name, + hs=hs, + clock=self.clock, + max_len=100000, + expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000, + size_callback=_state_map_size, + reset_expiry_on_get=True, + ) + ) + # # stuff for tracking time spent on state-res by room # @@ -798,6 +833,7 @@ class StateResolutionHandler: state_sets, event_map, state_res_store, + conflict_cache=self._conflict_resolution_cache, ) finally: self._record_state_res_metrics(room_id, m.get_resource_usage()) diff --git a/synapse/state/v2.py b/synapse/state/v2.py index 1241a4d66e..b384964bac 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -18,18 +18,24 @@ # # +import hashlib import heapq import itertools +import json import logging +from collections import ChainMap from typing import ( + AbstractSet, Any, Awaitable, Callable, Generator, Iterable, Literal, + MutableMapping, Protocol, Sequence, + cast, overload, ) @@ -39,7 +45,8 @@ from synapse.api.errors import AuthError from synapse.api.room_versions import RoomVersion, StateResolutionVersions from synapse.events import EventBase, is_creator from synapse.storage.databases.main.event_federation import StateDifference -from synapse.types import MutableStateMap, StateMap, StrCollection +from synapse.types import MutableStateMap, StateKey, StateMap, StrCollection +from synapse.util import MutableOverlayMapping from synapse.util.duration import Duration logger = logging.getLogger(__name__) @@ -86,6 +93,7 @@ async def resolve_events_with_store( state_sets: Sequence[StateMap[str]], event_map: dict[str, EventBase] | None, state_res_store: StateResolutionStore, + conflict_cache: "ConflictCache | None" = None, ) -> StateMap[str]: """Resolves the state using the v2 state resolution algorithm @@ -104,6 +112,10 @@ async def resolve_events_with_store( If None, all events will be fetched via state_res_store. state_res_store: + conflict_cache: + if given, the resolution of the conflicted set is looked up here + before it is computed, and stored here afterwards. See + `_conflict_cache_key` for what the key covers. Returns: A map from (type, state_key) to event_id. @@ -163,6 +175,145 @@ async def resolve_events_with_store( logger.debug("%d full_conflicted_set entries", len(full_conflicted_set)) + # Calculate the base state. + # + # v2 uses the unconflicted state as the base state, but v2.1 uses the empty + # set. + base_state: StateMap[str] = {} + if room_version.state_res != StateResolutionVersions.V2_1: + # Resolving conflicted sets requires the following types from the base + # state: + # - the `auth_types_for_event(..)` of the conflicted events for + # `_iterative_auth_checks` + # - the power levels for `_mainline_sort` + # + # We can therefore safely restrict the base state to those keys, which + # keeps keys that cannot affect the outcome out of the cache key. The + # rest of the unconflicted state is layered back on below. + base_state_keys = {(EventTypes.PowerLevels, "")} + for event_id in full_conflicted_set: + base_state_keys.update( + event_auth.auth_types_for_event(room_version, event_map[event_id]) + ) + + base_state = { + key: unconflicted_state[key] + for key in base_state_keys + if key in unconflicted_state + } + + resolved_state: StateMap[str] | None = None + if conflict_cache is not None: + cache_key = _conflict_cache_key( + room_id, room_version, full_conflicted_set, base_state + ) + resolved_state = conflict_cache.get(cache_key) + + if resolved_state is None: + resolved_state = await _resolve_conflicted_set( + clock, + room_id, + room_version, + full_conflicted_set, + base_state, + event_map, + state_res_store, + ) + if conflict_cache is not None: + conflict_cache[cache_key] = resolved_state + else: + logger.debug( + "Reusing the resolution of %d conflicted events", + len(full_conflicted_set), + ) + + logger.debug("done") + + # Finally, we copy the unconflicted state over the resolved state. + # + # We use a `ChainMap` here to avoid a copy. + # + # `ChainMap` expects mutable mappings as it is a mutable mapping. However, + # the return type of this function is an immutable mapping so it is safe to + # cast the underlying mappings to mutable mappings to satify `ChainMap`'s + # signature. + return ChainMap( + cast(MutableMapping[StateKey, str], unconflicted_state), + cast(MutableMapping[StateKey, str], resolved_state), + ) + + +class ConflictCache(Protocol): + """What `resolve_events_with_store` needs from a `conflict_cache`. + + Satisfied by a plain `dict` and by `ExpiringCache`. + + Mainly used to allow unit tests to pass a `dict` rather than building a full + cache. + """ + + def get(self, key: bytes) -> StateMap[str] | None: ... + + def __setitem__(self, key: bytes, value: StateMap[str]) -> None: ... + + +def _conflict_cache_key( + room_id: str, + room_version: RoomVersion, + full_conflicted_set: AbstractSet[str], + base_state: StateMap[str], +) -> bytes: + """Create a key for the conflict cache. Incorporates everything + that would affect the result of `_resolve_conflicted_set`. + + We use a digest as the key as the inputs can be very large. + """ + + # We use JSON as the serialization format for ease, we could use a + # hand-rolled format but one has to be careful to ensure that you can't get + # collisions (given in some room versions the room ID is arbitrary bytes + # rather than a hash). + key_material = json.dumps( + { + "room_id": room_id, + "room_version": room_version.identifier, + "conflicted_set": sorted(full_conflicted_set), + "base_state": sorted(base_state.values()), + } + ) + return hashlib.sha256(key_material.encode("utf-8")).digest() + + +async def _resolve_conflicted_set( + clock: Clock, + room_id: str, + room_version: RoomVersion, + full_conflicted_set: set[str], + base_state: StateMap[str], + event_map: dict[str, EventBase], + state_res_store: StateResolutionStore, +) -> MutableStateMap[str]: + """Apply the conflicted events to the base state. + + This is the expensive part of `resolve_events_with_store`: the reverse + topological power sort, the two iterative auth check passes and the + mainline sort. + + Args: + clock + room_id: the room we are working in + room_version: the room version + full_conflicted_set: the conflicted events plus the auth chain + difference. All must be present in `event_map`. + base_state: the state to apply the conflicted events to + event_map: updated in place with the events fetched along the way + state_res_store + + Returns: + The base state with the conflicted events that passed auth applied + over it. The unconflicted state is not merged in. + """ + # Get and sort all the power events (kicks/bans/etc) power_events = ( eid for eid in full_conflicted_set if _is_power_event(event_map[eid]) @@ -174,17 +325,6 @@ async def resolve_events_with_store( logger.debug("sorted %d power events", len(sorted_power_events)) - # v2.1 starts iterative auth checks from the empty set and not the unconflicted state. - # It relies on IAC behaviour which populates the base state with the events from auth_events - # if the state tuple is missing from the base state. This ensures the base state is only - # populated from auth_events rather than whatever the unconflicted state is (which could be - # completely bogus). - base_state = ( - {} - if room_version.state_res == StateResolutionVersions.V2_1 - else unconflicted_state - ) - # Now sequentially auth each one resolved_state = await _iterative_auth_checks( clock, @@ -227,11 +367,6 @@ async def resolve_events_with_store( logger.debug("resolved") - # We make sure that unconflicted state always still applies. - resolved_state.update(unconflicted_state) - - logger.debug("done") - return resolved_state @@ -691,7 +826,7 @@ async def _iterative_auth_checks( Returns: Returns the final updated state """ - resolved_state = dict(base_state) + resolved_state = MutableOverlayMapping(base_state) for idx, event_id in enumerate(event_ids, start=1): event = event_map[event_id] diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1df7f70b71..bc13a54dd1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -137,6 +137,7 @@ class SQLBaseStore(metaclass=ABCMeta): # Purge other caches based on room state. self._attempt_to_invalidate_cache("get_room_summary", (room_id,)) self._attempt_to_invalidate_cache("get_partial_current_state_ids", (room_id,)) + self._attempt_to_invalidate_cache("_get_current_state_event_id", (room_id,)) self._attempt_to_invalidate_cache("get_room_type", (room_id,)) self._attempt_to_invalidate_cache("get_room_encryption", (room_id,)) self._attempt_to_invalidate_cache( @@ -154,6 +155,7 @@ class SQLBaseStore(metaclass=ABCMeta): room_id: Room where state changed """ self._attempt_to_invalidate_cache("get_partial_current_state_ids", (room_id,)) + self._attempt_to_invalidate_cache("_get_current_state_event_id", (room_id,)) self._attempt_to_invalidate_cache("get_users_in_room", (room_id,)) self._attempt_to_invalidate_cache("is_host_invited", None) self._attempt_to_invalidate_cache("is_host_joined", None) diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index d29b116c84..9f6e9a9fab 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -654,7 +654,54 @@ class EventsPersistenceStorageController: state_delta_for_room = None new_state_dag_extrems = None - if not backfilled: + if all_supports_msc4242_state_dag(chunk) and backfilled: + # Backfilled events do not update the room DAG extremities or the + # current state, but a state event which is new to us still has to move + # the state DAG forward extremities. + # + # In practice the only thing which gets here are state events from /send_join. + # Those events are persisted as backfilled so that they do not appear down + # /sync, but the state DAG forward extremities must be updated otherwise when + # we hit the join event we won't calculate the current state correctly due to having + # stale extremities. + # + # Catching up on a room we are already in does not come through here: the + # state DAG walk in /get_missing_events fetches `prev_state_events` at + # backfilled=False, and we refuse to persist any event whose + # `prev_state_events` we do not already hold, so an event cannot reach us + # ahead of the prev_state_events it references. + # + # There is one edge case here: when prev_events disagrees with prev_state_events. E.g: + # + # prev_events graph We know S1, S2, S3 + # S1 <- S2 <- S3 Sx = state event, M = message + # ^-- S4 <- S5 <- M Backfilling M returns S4, S5 alongside it + # + # Suppose M incorrectly has prev_state_events=[S3] but prev_events=[S5] + # then /get_missing_events will NOT fetch S4, S5 and only when you /backfill + # the prev_events graph will you suddenly become aware of [S4, S5], + # which will hit this code path. In this case, we should be recalculating the current + # state to take S4,S5 into account but this code path does not. + # We rely on the caller (_process_pulled_events) to handle this case by checking if + # _backfilled_ events changed the state DAG forward extremities, and in the extremely + # unlikely case it did, warn in the logs and recalculate the room state. + seen_event_ids = await self.main_store.have_seen_events( + room_id, [event.event_id for event, _ in chunk] + ) + unseen_chunk = [ + (event, context) + for event, context in chunk + if event.event_id not in seen_event_ids + ] + if unseen_chunk: + new_state_dag_extrems = ( + await self._calculate_new_state_dag_extremities( + room_id, + await self.main_store.get_state_dag_extremities(room_id), + unseen_chunk, + ) + ) + elif not backfilled: if all_supports_msc4242_state_dag(chunk): with Measure( self._clock, diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 4885268305..4b3c978ccd 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -533,7 +533,12 @@ class StateStorageController: Raises: RuntimeError if we don't have a state group for one or more of the events - (ie. they are outliers or unknown) + (ie. they are outliers or unknown) AND the lookup reached the database. + + Callers MUST NOT rely on this to determine missing events, as cached responses + return None for the state group instead of raising. + + TODO: audit every caller, as some of them may be relying on the current behaviour. """ if await_full_state: await self._partial_state_events_tracker.await_full_state(event_ids) @@ -580,8 +585,8 @@ class StateStorageController: """Get the current state event ids for a room based on the current_state_events table. - If a state filter is given (that is not `StateFilter.all()`) the query - result is *not* cached. + If a wildcard state filter is given (that is not `StateFilter.all()`) + the query result is *not* cached. Args: room_id: The room to get the state IDs of. state_filter: The state diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 7cd3667a2b..f9e36ea8be 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -438,7 +438,23 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke ) if hs.config.worker.run_background_tasks and self.user_ips_max_age: - self.clock.looping_call(self._prune_old_user_ips, Duration(seconds=5)) + self.clock.looping_call( + self._prune_old_user_ips, + # Based on a measured value of ~9.17 average deletes/second on the + # `user_ips` table on `matrix.org`. If we prune every 60 seconds, this + # query will pick-up ~550 rows which is under the query `LIMIT` set + # (5000) with about an order of magnitude head-room to catch-up from + # downtime or peak/heavy traffic. On `matrix.org`, the peak delete + # activity on the `user_ips` table is ~2x the daily low. + # + # Running this more often seems good as that means we pick up a smaller + # number of rows each time (less work), therefore less disruptive the + # queries will be. But in practice, the query still has a fixed cost as + # it has to scan through dead tuples left behind by all of the previous + # prunes until the table is vacuumed which isn't free so just run this a + # reasonable amount. + Duration(seconds=60), + ) if self._update_on_this_worker: # This is the designated worker that can write to the client IP diff --git a/synapse/storage/databases/main/delayed_events.py b/synapse/storage/databases/main/delayed_events.py index 35f78e3f97..1cd06dc14e 100644 --- a/synapse/storage/databases/main/delayed_events.py +++ b/synapse/storage/databases/main/delayed_events.py @@ -621,6 +621,35 @@ class DelayedEventsStore(SQLBaseStore): "cancel_delayed_state_events", cancel_delayed_state_events_txn ) + async def cancel_delayed_events_for_user( + self, user_localpart: str + ) -> Timestamp | None: + """ + Cancels all delayed events owned by the given local user, i.e. remove them as long as they haven't been processed. + + Args: + user_localpart: The localpart of the user whose delayed events to cancel. + + Returns: The send time of the next delayed event to be sent, if any. + """ + + def cancel_delayed_events_for_user_txn( + txn: LoggingTransaction, + ) -> Timestamp | None: + self.db_pool.simple_delete_txn( + txn, + table="delayed_events", + keyvalues={ + "user_localpart": user_localpart, + "is_processed": False, + }, + ) + return self._get_next_delayed_event_send_ts_txn(txn) + + return await self.db_pool.runInteraction( + "cancel_delayed_events_for_user", cancel_delayed_events_for_user_txn + ) + async def delete_processed_delayed_event(self, delay_id: DelayID) -> None: """ Delete the matching delayed event, as long as it has been marked as processed. diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index 9c5fd35906..afbbe6b61a 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -435,12 +435,13 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas """ txn.execute(sql, args) - seen_thread_ids = set() + # The (room ID, thread ID) pairs we found an up-to-date summary for. + seen_room_thread_ids = set() room_to_count: dict[str, int] = defaultdict(int) for room_id, thread_id, notif_count in txn: room_to_count[room_id] += notif_count - seen_thread_ids.add(thread_id) + seen_room_thread_ids.add((room_id, thread_id)) # Now get any event push actions that haven't been rotated using the same OR # join and filter by receipt and event push summary rotated up to stream ordering. @@ -460,35 +461,31 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas for room_id, thread_id, notif_count in txn: # Note: only count push actions we have valid summaries for with up to date receipt. - if thread_id not in seen_thread_ids: + if (room_id, thread_id) not in seen_room_thread_ids: continue room_to_count[room_id] += notif_count - thread_id_clause, thread_ids_args = make_in_list_sql_clause( - self.database_engine, "epa.thread_id", seen_thread_ids - ) - - # Finally re-check event_push_actions for any rooms not in the summary, ignoring - # the rotated up-to position. This handles the case where a read receipt has arrived - # but not been rotated meaning the summary table is out of date, so we go back to - # the push actions table. + # Finally re-check event_push_actions for any room/threads not in the summary, + # ignoring the rotated up-to position. This handles the case where a read receipt + # has arrived but not been rotated meaning the summary table is out of date, so we + # go back to the push actions table. sql = f""" {receipts_cte} - SELECT epa.room_id, COUNT(CASE WHEN epa.notif = 1 THEN 1 END) AS notif_count + SELECT epa.room_id, epa.thread_id, COUNT(CASE WHEN epa.notif = 1 THEN 1 END) AS notif_count FROM event_push_actions AS epa {receipts_joins} WHERE user_id = ? - AND NOT {thread_id_clause} AND epa.notif = 1 AND (threaded_receipt_stream_ordering IS NULL OR stream_ordering > threaded_receipt_stream_ordering) AND (unthreaded_receipt_stream_ordering IS NULL OR stream_ordering > unthreaded_receipt_stream_ordering) - GROUP BY epa.room_id + GROUP BY epa.room_id, epa.thread_id """ - args.extend(thread_ids_args) txn.execute(sql, args) - for room_id, notif_count in txn: + for room_id, thread_id, notif_count in txn: + if (room_id, thread_id) in seen_room_thread_ids: + continue room_to_count[room_id] += notif_count return room_to_count diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index cfe7e7a592..f0080d2d63 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -61,7 +61,6 @@ from synapse.events.py_protocol import ( supports_msc4242_state_dag, ) from synapse.events.snapshot import EventPersistencePair -from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import trace from synapse.metrics import SERVER_NAME_LABEL from synapse.storage._base import db_to_json, make_in_list_sql_clause @@ -77,12 +76,15 @@ from synapse.storage.databases.main.search import SearchEntry from synapse.storage.engines import PostgresEngine from synapse.storage.util.id_generators import AbstractStreamIdGenerator from synapse.storage.util.sequence import SequenceGenerator +from synapse.synapse_rust.events import redact_event +from synapse.synapse_rust.room_versions import EventFormatVersions, RoomVersion from synapse.types import ( JsonDict, MutableStateMap, StateMap, StrCollection, UserID, + get_domain_from_id, ) from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.state import StateFilter @@ -1119,6 +1121,7 @@ class PersistEventsStore: min_stream_order = events_and_contexts[0][0].internal_metadata.stream_ordering max_stream_order = events_and_contexts[-1][0].internal_metadata.stream_ordering + room_version = events_and_contexts[0][0].room_version # We check that the room still exists for events we're trying to # persist. This is to protect against races with deleting a room. @@ -1180,6 +1183,10 @@ class PersistEventsStore: # From this point onwards the events are only events that we haven't # seen before. + events_and_contexts = self._apply_existing_redaction_txn( + txn, room_id, room_version, events_and_contexts=events_and_contexts + ) + self._store_event_txn(txn, events_and_contexts=events_and_contexts) if new_forward_extremities: @@ -2235,7 +2242,13 @@ class PersistEventsStore: """ Record updates into the profile updates stream for when a user leaves a room. - If this was the last shared room with a set of users, clear all old rows from + This handles two distinct cases when a user leaves a room: + 1) we find users in the the room who no longer share rooms with the user that + left the room, and record a `LEFT_ROOM` action for them. + 2) we check for the user who left the room if they no longer share rooms with + some users of the room that was left, and do the same in reverse. + + In both cases, when recording a `LEFT_ROOM` action, we clear all old rows from the `profile_updates_per_user` table relating to those users, to avoid exposing any profile field changes past the point of not being in any common rooms with the user. @@ -2287,15 +2300,39 @@ class PersistEventsStore: (*user_args, user_id.to_string()), ) - # Now record the "left room" action in the stream + # Now record the "left room" action in the stream for each user + # in the room that no longer shares a room with the user who left the room. self.store.record_profile_updates_txn( txn=txn, - user_id=user_id, + users={user_id.to_string()}, action=ProfileUpdateAction.LEFT_ROOM, field_names=[], target_users=users_no_longer_sharing_rooms, ) + # We also need to record things in reverse. The user, who left the + # room, needs to get profile update rows for every user they no longer + # share a room with. + # First clear old rows between these users. + txn.execute( + f""" + DELETE FROM profile_updates_per_user + WHERE user_id = ? + AND stream_id IN ( + SELECT stream_id FROM profile_updates WHERE {user_clause} + ) + """, + (user_id.to_string(), *user_args), + ) + # Then add the left room action rows in the stream. + self.store.record_profile_updates_txn( + txn=txn, + users=users_no_longer_sharing_rooms, + action=ProfileUpdateAction.LEFT_ROOM, + field_names=[], + target_users={user_id.to_string()}, + ) + @classmethod def _get_relevant_sliding_sync_current_state_event_ids_txn( cls, txn: LoggingTransaction, room_id: str @@ -2430,7 +2467,7 @@ class PersistEventsStore: stripped_state_map: MutableStateMap[StrippedStateEvent] = {} if isinstance(unsigned_stripped_state_events, list): for raw_stripped_event in unsigned_stripped_state_events: - stripped_state_event = parse_stripped_state_event( + stripped_state_event = StrippedStateEvent.from_json_dict( raw_stripped_event ) if stripped_state_event is not None: @@ -2898,6 +2935,118 @@ class PersistEventsStore: ec for ec in events_and_contexts if ec[0] not in to_remove ], gained_state + def _apply_existing_redaction_txn( + self, + txn: LoggingTransaction, + room_id: str, + room_version: RoomVersion, + events_and_contexts: list[EventPersistencePair], + ) -> list[EventPersistencePair]: + """ + If we have any redactions for the events we're about to persist, + pre-applies those if they are appropriate. + + Redaction events themselves won't be redacted immediately, to avoid + breaking circular redactions (which are tested in `test_circular_redaction`). + + In general, applying redactions prior to persistence (like this method does) + is not required for correctness as `_maybe_redact_event_row` will do it on + the read path (and the event will eventually be 'censored' in the background). + + However sticky events are an exception: redaction removes the field that makes + an event sticky (`msc4354_sticky`) and applying this prior to persistence + allows us to skip inserting the event into the sticky events stream. + + Returns a copy of the `events_and_contexts` lists with redactions applied. + """ + + # Get all the event IDs, whilst also filtering out redaction events + # which we don't want to redact immediately anyway + event_ids = [ + ev.event_id + for ev, _ in events_and_contexts + if ev.type != EventTypes.Redaction + ] + + if not event_ids: + # nothing to do here + return events_and_contexts + + events_by_id = {ev.event_id: ev for ev, _ in events_and_contexts} + event_id_in_list_clause, event_id_in_list_args = make_in_list_sql_clause( + txn.database_engine, + "redactions.redacts", + event_ids, + ) + txn.execute( + f""" + SELECT redactions.event_id, redactions.redacts, redaction_events.sender, redactions.recheck + FROM redactions + INNER JOIN events AS redaction_events + ON redactions.event_id = redaction_events.event_id + AND redaction_events.room_id = ? + WHERE {event_id_in_list_clause} + """, + (room_id, *event_id_in_list_args), + ) + + # map from redacted event ID to the event ID that redacts it + redacted_event_id_map = {} + for ( + redaction_event_id, + redaction_redacts_event_id, + redaction_event_sender, + redaction_event_needs_v3_recheck, + ) in txn: + if redaction_redacts_event_id in redacted_event_id_map: + # Already redacted + continue + + event = events_by_id[redaction_redacts_event_id] + if ( + redaction_event_needs_v3_recheck + # Normally v1/v2 don't ever need rechecks because the checks are part of auth rules. + # However the `recheck` column was only recently introduced as a retrofit, + # with a database-level default of `true`. + # For that reason, we can't `assert` on this condition as even v1/v2 rooms can appear + # with a `recheck` value of true, even though they don't need a recheck. + and room_version.event_format != EventFormatVersions.ROOM_V1_V2 + ): + # Apply the same logic as `_maybe_redact_event_row` + if get_domain_from_id(redaction_event_sender) != get_domain_from_id( + event.sender + ): + # Sender servers don't match, so the event isn't actually redacted + logger.debug( + "redaction of %s by %s skipped as it required a v3 recheck and sender servers differ: %r != %r", + redaction_redacts_event_id, + redaction_event_id, + redaction_event_sender, + event.sender, + ) + continue + + # Unlike `_maybe_redact_event_row`, we _don't_ mutate the cached instance of the redaction event to set + # `recheck_redaction` to False here, as we don't actually load the redaction out of the database. + + redacted_event_id_map[redaction_redacts_event_id] = redaction_event_id + logger.debug( + "%r redacted at persistence time by %r", + redaction_redacts_event_id, + redaction_event_id, + ) + + out: list[EventPersistencePair] = [] + for event, context in events_and_contexts: + if redacted_by_event_id := redacted_event_id_map.get(event.event_id): + redacted_event = redact_event(event) + redacted_event.internal_metadata.redacted_by = redacted_by_event_id + out.append((redacted_event, context)) + else: + out.append((event, context)) + + return out + def _store_event_txn( self, txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 215c8e44e3..873d355ea1 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -41,7 +41,7 @@ from typing_extensions import assert_never from twisted.internet import defer -from synapse.api.constants import Direction, EventTypes +from synapse.api.constants import Direction, EventTypes, Membership from synapse.api.errors import NotFoundError, SynapseError from synapse.api.room_versions import ( KNOWN_ROOM_VERSIONS, @@ -390,6 +390,8 @@ class EventsWorkerStore(SQLBaseStore): finished (so we don't have to keep querying it every time) """ + self._room_prejoin_state_types = hs.config.api.room_prejoin_state + def get_un_partial_stated_events_token(self, instance_name: str) -> int: return ( self._un_partial_stated_events_stream_id_gen.get_current_token_for_writer( @@ -1141,14 +1143,12 @@ class EventsWorkerStore(SQLBaseStore): async def get_stripped_room_state_from_event_context( self, + event: EventBase, context: EventContext, - state_keys_to_include: StateFilter, - membership_user_id: str | None = None, ) -> list[JsonDict]: """ Retrieve the stripped state from a room, given an event context to retrieve state - from as well as the state types to include. Optionally, include the membership - events from a specific user. + from as well as the state types to include. "Stripped" state means that only the `type`, `state_key`, `content` and `sender` keys are included from each state event. @@ -1156,35 +1156,60 @@ class EventsWorkerStore(SQLBaseStore): Args: context: The event context to retrieve state of the room from. state_keys_to_include: The state events to include, for each event type. - membership_user_id: An optional user ID to include the stripped membership state - events of. This is useful when generating the stripped state of a room for - invites. We want to send membership events of the inviter, so that the - invitee can display the inviter's profile information if the room lacks any. Returns: A list of dictionaries, each representing a stripped state event from the room. """ - if membership_user_id: + selected_state_ids = await self.get_stripped_room_state_ids_from_event_context( + event, context + ) + + state_to_include = await self.get_events(selected_state_ids) + + return [strip_event(e) for e in state_to_include.values()] + + async def get_stripped_room_state_ids_from_event_context( + self, + event: EventBase, + context: EventContext, + ) -> list[str]: + """ + Retrieve the stripped state IDs for an event, given an event context to retrieve state + from as well as the state types to include. + + Args: + context: The event context to retrieve state of the room from. + + Returns: + A list of event_ids, each representing the stripped state event to include for this event + """ + # Start with the configured default set of stripped state to include + state_filter = self._room_prejoin_state_types + + # MSC4319: We want to send membership events of the inviter, so that the invitee + # can display the inviter's profile information if the room lacks any. + is_invite_event = ( + event.type == EventTypes.Member and event.membership == Membership.INVITE + ) + if is_invite_event: types = chain( - state_keys_to_include.to_types(), - [(EventTypes.Member, membership_user_id)], + self._room_prejoin_state_types.to_types(), + [(EventTypes.Member, event.sender)], ) - filter = StateFilter.from_types(types) - else: - filter = state_keys_to_include - selected_state_ids = await context.get_current_state_ids(filter) + state_filter = StateFilter.from_types(types) + + # Get the relevant state + selected_state_ids = await context.get_current_state_ids(state_filter) # We know this event is not an outlier, so this must be # non-None. assert selected_state_ids is not None - # Confusingly, get_current_state_events may return events that are discarded by - # the filter, if they're in context._state_delta_due_to_event. Strip these away. - selected_state_ids = filter.filter_state(selected_state_ids) + # Confusingly, `get_current_state_ids` may return events that are discarded by + # the filter, if they're in `context._state_delta_due_to_event`. Strip these away. + selected_state_ids = state_filter.filter_state(selected_state_ids) - state_to_include = await self.get_events(selected_state_ids.values()) - - return [strip_event(e) for e in state_to_include.values()] + return list(selected_state_ids.values()) def _maybe_start_fetch_thread(self) -> None: """Starts an event fetch thread if we are not yet at the maximum number.""" @@ -1482,7 +1507,7 @@ class EventsWorkerStore(SQLBaseStore): # if d["type"] != EventTypes.Member: raise InvalidEventError( - "Room %s for event %s is unknown" % (d["room_id"], event_id) + "Room %s for event %s is unknown" % (d.get("room_id"), event_id) ) # so, assuming this is an out-of-band-invite that arrived before @@ -1513,7 +1538,7 @@ class EventsWorkerStore(SQLBaseStore): logger.warning( "Event %s in room %s has unknown room version %s", event_id, - d["room_id"], + d.get("room_id"), room_version_id, ) continue @@ -1523,7 +1548,7 @@ class EventsWorkerStore(SQLBaseStore): "Event %s in room %s with version %s has wrong format: " "expected %s, was %s", event_id, - d["room_id"], + d.get("room_id"), room_version_id, room_version.event_format, format_version, @@ -1552,7 +1577,7 @@ class EventsWorkerStore(SQLBaseStore): # it's difficult to see what to do here. Pretty much all bets are off # if Synapse cannot rely on the consistency of its database. raise DatabaseCorruptionError( - d["room_id"], event_id, original_ev.event_id + d.get("room_id"), event_id, original_ev.event_id ) event_map[event_id] = original_ev @@ -1785,13 +1810,14 @@ class EventsWorkerStore(SQLBaseStore): # Starting in room version v3, some redactions need to be # rechecked if we didn't have the redacted event at the # time, so we recheck on read instead. + # NOTE: If this logic changes, need to update `_apply_existing_redaction_txn` if redaction_event.internal_metadata.need_to_check_redaction(): expected_domain = get_domain_from_id(original_ev.sender) if get_domain_from_id(redaction_event.sender) == expected_domain: # This redaction event is allowed. Mark as not needing a recheck. redaction_event.internal_metadata.recheck_redaction = False else: - # Senders don't match, so the event isn't actually redacted + # Sender servers don't match, so the event isn't actually redacted logger.debug( "%s was redacted by %s but the senders don't match", original_ev.event_id, diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 7f860258ee..8ed0d645bc 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -854,7 +854,7 @@ class ProfileWorkerStore(SQLBaseStore): # Record updates in the profile updates stream stream_id = self.record_profile_updates_txn( txn=txn, - user_id=user_id, + users={user_id.to_string()}, action=ProfileUpdateAction.UPDATE, field_names=[field_name], ) @@ -885,42 +885,57 @@ class ProfileWorkerStore(SQLBaseStore): # Ensure we're working with local users only users = {user_id for user_id in joined_users if self.hs.is_mine_id(user_id)} + # Get the members of the room + rows = self.db_pool.simple_select_list_txn( + txn=txn, + table="local_current_membership", + keyvalues={ + "room_id": room_id, + "membership": Membership.JOIN, + }, + retcols=("user_id",), + ) + target_users = {row[0] for row in rows} + # Record the profile updates for each user - for user_id in users: - self.record_profile_updates_txn( - txn=txn, - user_id=UserID.from_string(user_id), - action=ProfileUpdateAction.JOINED_ROOM, - field_names=None, - user_rooms={room_id}, - ) + self.record_profile_updates_txn( + txn=txn, + users=users, + action=ProfileUpdateAction.JOINED_ROOM, + target_users=target_users, + field_names=None, + ) def record_profile_updates_txn( self, *, txn: LoggingTransaction, - user_id: UserID, + users: set[str], action: ProfileUpdateAction, field_names: Collection[str] | None, - user_rooms: set[str] | None = None, target_users: set[str] | None = None, + user_rooms: set[str] | None = None, ) -> int | None: """ Record updates into the profile updates stream tables. + If `target`_users` is not given as a parameter, `users` must be a single user. + Currently, updates are only recorded for local users. Args: txn: Transaction to use - user_id: User ID that made the profile update + users: A set of user IDs to write the profile updates for. action: The profile update action, either `update`, `left_room` or `joined_room`. field_names: A list of fields that were set, if ProfileUpdateAction.UPDATE + target_users: Optionally, set of users to create per user profile update + stream rows for. If not given, and the length of `users` is only + a single user, a database lookup will be done based on + `user_rooms`, or if that is not set, the result of the rooms lookup. user_rooms: Optionally, a set of rooms that the update concerns. If not given, a database lookup will be done to fetch all the users rooms. - target_users: Optionally, set of users to create profile update stream rows - for. If not given, a database lookup will be done based on `user_rooms`, - or if that is not set, the result of the rooms lookup. + Only used if `target_users` is not given. Returns: The latest stream ID created in this transaction @@ -933,15 +948,20 @@ class ProfileWorkerStore(SQLBaseStore): else: assert not field_names - if not target_users: - if not user_rooms: + if target_users is None: + # This function must be called with one user only if it needs to + # compute the target users. This restriction mainly exists as a + # fail safe to ensure we don't abuse the loop of membership fetches here + # and ensure calling code makes the necessary optimizations. + assert len(users) == 1 + if user_rooms is None: rows = self.db_pool.simple_select_onecol_txn( txn=txn, table="current_state_events", keyvalues={ "type": EventTypes.Member, "membership": Membership.JOIN, - "state_key": user_id.to_string(), + "state_key": list(users)[0], }, retcol="room_id", ) @@ -959,40 +979,97 @@ class ProfileWorkerStore(SQLBaseStore): ) target_users = {row[0] for row in rows} + if action == ProfileUpdateAction.UPDATE: + # Always include ourselves when updating field values. + # We need to do this as the users updating their profile may not be + # in any rooms, and thus wont be collected above, but should still get the + # update pushed to their other devices. + target_users = target_users.union(users) + # Ensure we only write updates for local users - users = {user for user in target_users if self.hs.is_mine_id(user)} + target_users = {user for user in target_users if self.hs.is_mine_id(user)} - if action in (ProfileUpdateAction.JOINED_ROOM, ProfileUpdateAction.LEFT_ROOM): - users.discard(user_id.to_string()) - if not users: - # No point writing an update for ourselves, if a membership change and no - # other users interested - return None - elif action == ProfileUpdateAction.UPDATE: - # Always include ourselves when updating field values - users.add(user_id.to_string()) + if not target_users: + return None - # Record the profile update inserted_ts = self.clock.time_msec() - stream_id = self._profile_updates_id_gen.get_next_txn(txn) + profile_update_values = {} + sorted_field_names = ( + json_encoder.encode(sorted(field_names)) if field_names else None + ) - self.db_pool.simple_insert_txn( + # Collect profile updates to add + # We don't have stream ID's at this point, so just use a counter, and add in + # the stream ID's later. We do this here to avoid generating stream ID's we're + # not going to use, because here we'll be dropping any updates which only + # contain the user themselves, in a "joined room" or "left room" situation. If + # this call contains multiple users for "joined room" or "left room" situations, + # we'll filter the user out later in the per user updates. + # We also need to maintain a new users list, as it may shring. + final_users = [] + for counter, user_id in enumerate(users): + targets = ( + target_users - {user_id} + if action + in (ProfileUpdateAction.JOINED_ROOM, ProfileUpdateAction.LEFT_ROOM) + else target_users + ) + if not len(targets): + # No point writing a joined or left to the user themselves, skip. + continue + final_users.append(user_id) + profile_update_values[counter] = ( + self._instance_name, + user_id, + action.value, + sorted_field_names, + inserted_ts, + ) + + if not len(profile_update_values): + # We found nothing to update, abort. + return None + + # Now generate the stream ID's we want to use and add them to the list of + # updates. + stream_ids = self._profile_updates_id_gen.get_next_mult_txn( + txn, len(profile_update_values.keys()) + ) + profile_updates = [ + (stream_id, *values) + for stream_id, values in zip(stream_ids, profile_update_values.values()) + ] + # Maintain a map of stream_id to user_id, so we can later filter out rows + # when inserting into the per user updates table. + stream_ids_to_user_id = dict(list(zip(stream_ids, final_users))) + + self.db_pool.simple_insert_many_txn( txn, table="profile_updates", - values={ - "stream_id": stream_id, - "instance_name": self._instance_name, - "user_id": user_id.to_string(), - "action": action.value, - "affected_fields": json_encoder.encode(sorted(field_names)) - if field_names - else None, - "inserted_ts": inserted_ts, - }, + keys=[ + "stream_id", + "instance_name", + "user_id", + "action", + "affected_fields", + "inserted_ts", + ], + values=profile_updates, ) # Add per user tracking rows for each generated stream ID - per_user_values = [(stream_id, user_id, inserted_ts) for user_id in users] + per_user_values = [] + for stream_id in stream_ids: + per_user_values.extend( + # Filter out JOINED_ROOM/LEFT_ROOM updates to ourselves. + [ + (stream_id, user_id, inserted_ts) + for user_id in target_users + if stream_ids_to_user_id[stream_id] != user_id + or action == ProfileUpdateAction.UPDATE + ] + ) + self.db_pool.simple_insert_many_txn( txn, table="profile_updates_per_user", @@ -1003,7 +1080,7 @@ class ProfileWorkerStore(SQLBaseStore): ], values=per_user_values, ) - return stream_id + return stream_ids[-1] async def set_profile_field( self, @@ -1067,7 +1144,7 @@ class ProfileWorkerStore(SQLBaseStore): stream_id = self.record_profile_updates_txn( txn=txn, - user_id=user_id, + users={user_id.to_string()}, action=ProfileUpdateAction.UPDATE, field_names=[field_name], ) diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index 5086f4639b..b06e36149e 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -106,6 +106,7 @@ def _load_rules( msc3664_enabled=experimental_config.msc3664_enabled, msc3381_polls_enabled=experimental_config.msc3381_polls_enabled, msc4028_push_encrypted_events=experimental_config.msc4028_push_encrypted_events, + msc4075_enabled=experimental_config.msc4075_enabled, msc4210_enabled=experimental_config.msc4210_enabled, msc4306_enabled=experimental_config.msc4306_enabled, ) diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index ba5e07a051..3720d25c1d 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -32,7 +32,7 @@ from typing import ( import attr -from synapse.api.constants import EduTypes +from synapse.api.constants import Direction, EduTypes from synapse.replication.tcp.streams import ReceiptsStream from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( @@ -42,7 +42,11 @@ from synapse.storage.database import ( make_tuple_in_list_sql_clause, ) from synapse.storage.engines._base import IsolationLevel -from synapse.storage.util.id_generators import MultiWriterIdGenerator +from synapse.storage.util.id_generators import ( + MultiWriterIdGenerator, + advance_multiwriter_sharded_token_after_partial_read, + make_multiwriter_sharded_token_bounds_sql, +) from synapse.types import ( JsonDict, JsonMapping, @@ -578,54 +582,81 @@ class ReceiptsWorkerStore(SQLBaseStore): self, to_key: MultiWriterStreamToken, from_key: MultiWriterStreamToken | None = None, - ) -> Mapping[str, JsonMapping]: - """Get receipts for all rooms between two stream_ids, up - to a limit of the latest 100 read receipts. + limit: int = 100, + order: Direction = Direction.FORWARDS, + ) -> tuple[Mapping[str, JsonMapping], MultiWriterStreamToken]: + """Get receipts for all rooms between two stream_ids, up to a limit of + `limit` read receipts in that range. Args: to_key: Max stream id to fetch receipts up to. from_key: Min stream id to fetch receipts from. None fetches from the start. + limit: The maximum number of receipts to fetch. + order: `Direction.FORWARDS` fetches the oldest `limit` receipts in + the range. `Direction.BACKWARDS` fetches the newest `limit` + instead; anything older is deliberately skipped, and the + returned stream token is always `to_key`. Returns: - A dictionary of roomids to a list of receipts. + A two-tuple containing the following: + * A dictionary of roomids to receipt EDUs. + * The stream token up to which receipts were actually fetched. + With `Direction.FORWARDS`, this is earlier than `to_key` (per + writer) if the limit was hit; callers must call this method + again from the returned token to fetch the remaining + receipts. With `Direction.BACKWARDS`, it is always `to_key`, + even if the limit was hit: older receipts are skipped, not + left for a later call. """ + sql_order = "DESC" if order == Direction.BACKWARDS else "ASC" - def f(txn: LoggingTransaction) -> list[tuple[str, str, str, str, str]]: - if from_key: - sql = """ - SELECT stream_id, instance_name, room_id, receipt_type, user_id, event_id, data - FROM receipts_linearized WHERE - stream_id > ? AND stream_id <= ? - ORDER BY stream_id DESC - LIMIT 100 - """ - txn.execute(sql, [from_key.stream, to_key.get_max_stream_pos()]) - else: - sql = """ - SELECT stream_id, instance_name, room_id, receipt_type, user_id, event_id, data - FROM receipts_linearized WHERE - stream_id <= ? - ORDER BY stream_id DESC - LIMIT 100 - """ + # Bound each row on its own writer's positions in the tokens, so that + # `limit` applies after the bounds and a truncated page never leaves + # in-range rows behind. + from_token = ( + from_key if from_key is not None else MultiWriterStreamToken(stream=0) + ) + bounds_clause, bounds_values = make_multiwriter_sharded_token_bounds_sql( + self.database_engine, + stream_id_column="stream_id", + instance_name_column="instance_name", + from_token_exclusive=from_token, + to_token_inclusive=to_key, + ) - txn.execute(sql, [to_key.get_max_stream_pos()]) + def f(txn: LoggingTransaction) -> list[tuple[int, str, str, str, str, str]]: + sql = f""" + SELECT stream_id, room_id, receipt_type, user_id, event_id, data + FROM receipts_linearized + WHERE {bounds_clause} + ORDER BY stream_id {sql_order} + LIMIT ? + """ + txn.execute(sql, [*bounds_values, limit]) - return [ - (room_id, receipt_type, user_id, event_id, data) - for stream_id, instance_name, room_id, receipt_type, user_id, event_id, data in txn - if MultiWriterStreamToken.is_stream_position_in_range( - from_key, to_key, instance_name, stream_id - ) - ] + return cast(list[tuple[int, str, str, str, str, str]], txn.fetchall()) txn_results = await self.db_pool.runInteraction( "get_linearized_receipts_for_all_rooms", f ) + if order == Direction.FORWARDS and len(txn_results) == limit: + # We hit the limit, so there may be more receipts in the range. + # Report how far we actually got so that the caller can fetch the + # rest, claiming each writer's position only up to the last row we + # fetched so that receipts from writers that are behind it are not + # skipped. + reached_token = advance_multiwriter_sharded_token_after_partial_read( + from_token_exclusive=from_token, + to_token_inclusive=to_key, + last_read_stream_id=txn_results[-1][0], + ) + else: + reached_token = to_key + results: JsonDict = {} - for room_id, receipt_type, user_id, event_id, data in txn_results: + for _stream_id, room_id, receipt_type, user_id, event_id, data in txn_results: # We want a single event per room, since we want to batch the # receipts by room, event and type. room_event = results.setdefault( @@ -640,7 +671,7 @@ class ReceiptsWorkerStore(SQLBaseStore): receipt_type_dict[user_id] = db_to_json(data) - return results + return results, reached_token async def get_linearized_receipts_for_user_in_rooms( self, user_id: str, room_ids: StrCollection, to_key: MultiWriterStreamToken diff --git a/synapse/storage/databases/main/rejections.py b/synapse/storage/databases/main/rejections.py index c73c3d761d..1e0e87ae28 100644 --- a/synapse/storage/databases/main/rejections.py +++ b/synapse/storage/databases/main/rejections.py @@ -35,3 +35,17 @@ class RejectionsStore(SQLBaseStore): allow_none=True, desc="get_rejection_reason", ) + + async def has_rejected_event_ids(self, event_ids: set[str]) -> set[str]: + """Filter the provided event IDs to only return rejected events. + This does not check if the event IDs are persisted (missing events would appear as non-rejected). + """ + rows = await self.db_pool.simple_select_many_batch( + table="rejections", + column="event_id", + iterable=event_ids, + retcols=("event_id",), + keyvalues={}, + desc="has_rejected_event_ids", + ) + return {r[0] for r in rows} diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 3da0332ee9..9be37064c4 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -49,6 +49,7 @@ from synapse.storage.database import ( LoggingDatabaseConnection, LoggingTransaction, make_in_list_sql_clause, + make_tuple_in_list_sql_clause, ) from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.databases.main.roommember import RoomMemberWorkerStore @@ -536,7 +537,84 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): return frozenset(event_id for (event_id,) in rows) - # FIXME: how should this be cached? + @cached(max_entries=100000, tree=True) + async def _get_current_state_event_id( + self, room_id: str, event_type_and_state_key: tuple[str, str] + ) -> str | None: + """Get the event ID of the given piece of current state in the room. + + Returns None if there is no such event in the current state. + """ + return await self.db_pool.simple_select_one_onecol( + table="current_state_events", + keyvalues={ + "room_id": room_id, + "type": event_type_and_state_key[0], + "state_key": event_type_and_state_key[1], + }, + retcol="event_id", + allow_none=True, + desc="_get_current_state_event_id", + ) + + @cachedList( + cached_method_name="_get_current_state_event_id", + list_name="event_types_and_state_keys", + num_args=2, + ) + async def _get_current_state_event_ids( + self, room_id: str, event_types_and_state_keys: Collection[tuple[str, str]] + ) -> Mapping[tuple[str, str], str | None]: + """Bulk version of `_get_current_state_event_id`. + + Types/state keys that aren't in the room's current state map to None, so + that their absence gets cached too. + """ + if not event_types_and_state_keys: + return {} + + # Check if the room_id is in `get_partial_current_state_ids` cache, if + # so, we can use that to avoid a DB query. + room_state = self.get_partial_current_state_ids.cache.get_immediate( + room_id, None, update_metrics=False + ) + if room_state is not None: + return { + (intern_string(typ), intern_string(state_key)): room_state.get( + (typ, state_key) + ) + for typ, state_key in event_types_and_state_keys + } + + def _get_current_state_event_ids_txn( + txn: LoggingTransaction, + ) -> dict[tuple[str, str], str | None]: + results: dict[tuple[str, str], str | None] = { + (intern_string(typ), intern_string(state_key)): None + for typ, state_key in event_types_and_state_keys + } + + for batch in batch_iter(event_types_and_state_keys, 500): + clause, args = make_tuple_in_list_sql_clause( + self.database_engine, ("type", "state_key"), batch + ) + + sql = f""" + SELECT type, state_key, event_id FROM current_state_events + WHERE room_id = ? AND {clause} + """ + + txn.execute(sql, [room_id, *args]) + + for typ, state_key, event_id in txn: + results[(intern_string(typ), intern_string(state_key))] = event_id + + return results + + return await self.db_pool.runInteraction( + "_get_current_state_event_ids", _get_current_state_event_ids_txn + ) + @cancellable async def get_partial_filtered_current_state_ids( self, room_id: str, state_filter: StateFilter | None = None @@ -555,15 +633,28 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: Map from type/state_key to event ID. """ - if state_filter is None: - state_filter = StateFilter.all() + # First we check if we can delegate to one of the cached functions. + if state_filter is None or state_filter.is_full(): + return await self.get_partial_current_state_ids(room_id) + + if not state_filter.has_wildcards(): + results = StateMapWrapper(state_filter=state_filter) + + concrete_types = state_filter.concrete_types() + if not concrete_types: + # The filter matches nothing. + return results + + ids = await self._get_current_state_event_ids(room_id, concrete_types) + results.update( + (type_and_state_key, event_id) + for type_and_state_key, event_id in ids.items() + if event_id is not None + ) + return results where_clause, where_args = (state_filter).make_sql_filter_clause() - if not where_clause: - # We delegate to the cached version - return await self.get_partial_current_state_ids(room_id) - def _get_filtered_current_state_ids_txn( txn: LoggingTransaction, ) -> StateMap[str]: diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index d294119bb2..d370f2f815 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -36,9 +36,12 @@ from typing import ( ) import attr +from immutabledict import immutabledict +from prometheus_client import Gauge from sortedcontainers import SortedList, SortedSet from synapse.logging import issue9533_logger +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.database import ( DatabasePool, @@ -46,15 +49,26 @@ from synapse.storage.database import ( LoggingTransaction, make_in_list_sql_clause, ) -from synapse.storage.engines import PostgresEngine +from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine from synapse.storage.types import Cursor from synapse.storage.util.sequence import build_sequence_generator +from synapse.types import MultiWriterStreamToken if TYPE_CHECKING: from synapse.notifier import ReplicationNotifier logger = logging.getLogger(__name__) +# We report the calculated current position as seen by this worker. Note that +# this is always positive (even for negative streams like backfill) to make +# monitoring/alerting easier. +stream_current_position_gauge = Gauge( + "synapse_storage_stream_current_position", + "The stream's current position as this process sees it." + "Note, these are always positive, even for negative streams.", + labelnames=["stream_name", SERVER_NAME_LABEL], +) + T = TypeVar("T") @@ -232,6 +246,15 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): self._writers = writers self._return_factor = 1 if positive else -1 + # The `instance_name` label is what separates one worker's metrics from + # another's when there are multiple Synapse workers for the same homeserver + # running in the same process. In production the scrape labels would do that + # too, but tests run several homeservers in one process. + self._current_position_gauge = stream_current_position_gauge.labels( + stream_name=stream_name, + **{SERVER_NAME_LABEL: server_name}, + ) + # We lock as some functions may be called from DB threads. self._lock = threading.Lock() @@ -340,6 +363,20 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): # position with the current minimum. self._current_positions[self._instance_name] = self._persisted_upto_position + with self._lock: + self._report_current_position() + + def _report_current_position(self) -> None: + """Report the current position as a metric. + + Must be called with `_lock` held. + """ + assert self._lock.locked() + + # Note that we always report this as a positive value, so we don't + # multiply by the `_return_factor`. + self._current_position_gauge.set(self._persisted_upto_position) + def _load_current_ids( self, db_conn: LoggingDatabaseConnection, @@ -828,6 +865,8 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): # do. break + self._report_current_position() + # Hacky debug logging to attempt to trace https://github.com/element-hq/synapse/issues/19795. # If this is the to-device stream, and we are a writer for that stream, log some stats if ( @@ -982,3 +1021,198 @@ class _MultiWriterCtxManager: ) return False + + +def make_multiwriter_sharded_token_bounds_sql( + db_engine: BaseDatabaseEngine, + *, + stream_id_column: str, + instance_name_column: str, + from_token_exclusive: MultiWriterStreamToken, + to_token_inclusive: MultiWriterStreamToken, +) -> tuple[str, Sequence[str | int]]: + """ + Build an SQL clause that, in a multi-writer stream table, + matches the rows within the bounds of the `MultiWriterStreamToken` tokens provided. + + Bounds: from_token_exclusive < ... <= to_token_inclusive + + Arguments: + stream_id_column: the name of the `stream_id` column + Should be prefixed with the table name or alias + when used in a multi-table SELECT statement. + instance_name_column: the name of the `instance_name` column. + Should be prefixed with the table name or alias + when used in a multi-table SELECT statement. + from_token_exclusive: + MultiWriterStreamToken representing the highest position that has 'already been seen'. + Exclusive lower bound + to_token_inclusive: + MultiWriterStreamToken representing the highest position that has visible data. + Inclusive upper bound + + Example: + + make_multiwriter_sharded_token_bounds_sql( + stream_id_column="se.stream_id", + instance_name_column="se.instance_name", + # m5~1.8 + from_token_exclusive=MultiWriterStreamToken( + stream=5, instance_map=immutabledict({"worker1": 8}) + ), + # m10~2.14 + to_token_inclusive=MultiWriterStreamToken( + stream=10, instance_map=immutabledict({"worker2": 14}) + ), + ) + + gives: + + ( + 5 < se.stream_id AND se.stream_id <= 14 + AND NOT (se.instance_name = 'worker1' AND se.stream_id <= 8) + AND ( + se.stream_id <= 10 + OR (se.instance_name = 'worker2' AND se.stream_id <= 14) + ) + ) + + which matches, for example: + - a `worker1` row at position 9 (past the 8 we had already read from + `worker1`, and within the 10 that is visible for every writer); + - a `worker2` row at position 12 (past 5, and `worker2` is + explicitly visible up to 14); + but not: + - a `worker1` row at position 7, which the `from` token says we have + already read; + - a `worker3` row at position 12, which is beyond the position that + is visible for writers other than `worker2`. + + See also: + - `advance_multiwriter_sharded_token_after_partial_read` to create the next + `from_token_exclusive` after the result of reading a limited set of rows. + """ + + is_not_distinct_from = "IS NOT DISTINCT FROM" + if isinstance(db_engine, Sqlite3Engine): + # TODO(SQLite 3.39.0): Drop this compatibility code + # SQL standard `IS NOT DISTINCT FROM` was not introduced until 3.39.0 + # https://sqlite.org/releaselog/3_39_0.html + is_not_distinct_from = "IS" + + # The SQL we build will fundamentally consist of many clauses ANDed together. + # + # We start with an envelope delimited by the two outermost, writer-independent, bounds. + # We know everything before the lowest position of `from` and everything after the highest + # position of `to` must be out-of-bounds. + # This pair of bounds is the most likely to be index-friendly. + clauses = [f"? < {stream_id_column}", f"{stream_id_column} <= ?"] + values: list[int | str] = [ + from_token_exclusive.stream, + to_token_inclusive.get_max_stream_pos(), + ] + + # Now for every writer where we had already read further ahead than the baseline (lowest) position of `from`, + # we whittle away what we have already seen from the lower end of the envelope. + for instance_name, pos in from_token_exclusive.instance_map.items(): + # We need `IS NOT DISTINCT FROM` (analogous to `=` but treats `NULL` as a known value) + # for legacy rows where `instance_name` is `NULL`, such as before the stream was sharded. + clauses.append( + f"NOT ({instance_name_column} {is_not_distinct_from} ? AND {stream_id_column} <= ?)" + ) + values.extend((instance_name, pos)) + + # Now we need to restrict the upper end of the envelope to only include those rows where either: + # - the row is in a region where all writers' rows are visible; or + upper_or_clauses = [f"{stream_id_column} <= ?"] + upper_or_values: list[int | str] = [to_token_inclusive.stream] + # - the row's writer is explicitly visible ahead of the baseline (minimum) position + for instance_name, pos in to_token_inclusive.instance_map.items(): + upper_or_clauses.append( + f"({instance_name_column} {is_not_distinct_from} ? AND {stream_id_column} <= ?)" + ) + upper_or_values.extend((instance_name, pos)) + + clauses.append("( " + " OR ".join(upper_or_clauses) + " )") + values.extend(upper_or_values) + + return "( " + " AND ".join(clauses) + " )", values + + +def advance_multiwriter_sharded_token_after_partial_read( + *, + from_token_exclusive: MultiWriterStreamToken, + to_token_inclusive: MultiWriterStreamToken, + last_read_stream_id: int, +) -> MultiWriterStreamToken: + """ + Calculates the 'read up to' token after reading a limited number of rows + between two `MultiWriterStreamToken` bounds. + Pairs with `make_multiwriter_sharded_token_bounds_sql`. + + The read operation MUST have been ordered by the stream ID, e.g. + using `ORDER BY stream_id`. + + Caution: + For streams where one fact is represented by multiple rows with + the same `stream_id`, the ENTIRE fact corresponding to `last_read_stream_id` + MUST have been processed because this function won't help you 'pause' + in the middle of a fact. + + Arguments: + from_token_exclusive: the exclusive lower bound the rows were read with. + to_token_inclusive: the inclusive upper bound the rows were read with. + last_read_stream_id: the `stream_id` of the last row that was read. + + Returns: + A token that could be used as the `from_token_exclusive` for a subsequent + read, in order to continue reading rows in range. + + The token represents the maximum 'read up to' position. + Rows with positions strictly above the token have not yet been read. + """ + assert from_token_exclusive.stream < last_read_stream_id, "read was out of bounds" + + # Calculate the new baseline position that we have read up to, + # which applies across all workers. + new_baseline = max( + # Must be at least as far as it was before (doesn't go backwards) + from_token_exclusive.stream, + min( + # Simple case: this is just `last_read_stream_id`, + # the `stream_id` of the last row we read. + last_read_stream_id, + # Complicated case: some writers may not have advanced to `last_read_stream_id` yet, + # so we have to cap off at the baseline 'to' position (`to_token_inclusive.stream`) + to_token_inclusive.stream, + ), + ) + + # Now consider whether any workers are ahead of the baseline. + # We do this by calculating the position of each relevant worker. + # (Relevant by virtue of being in either the `from` or `to` token.) + advanced_instance_map = {} + for instance_name in ( + from_token_exclusive.instance_map.keys() + | to_token_inclusive.instance_map.keys() + ): + # Calculate the worker's position + worker_pos = max( + # Must be at least as far as it was before (doesn't go backwards) + from_token_exclusive.get_stream_pos_for_instance(instance_name), + # Then: + # - up to `last_read_stream_id` + # - unless its rows aren't visible yet, so cap off at the `to` position + # Either one of these cases will be at least as high as `new_baseline`. + min( + last_read_stream_id, + to_token_inclusive.get_stream_pos_for_instance(instance_name), + ), + ) + + if worker_pos > new_baseline: + advanced_instance_map[instance_name] = worker_pos + + return MultiWriterStreamToken( + stream=new_baseline, instance_map=immutabledict(advanced_instance_map) + ) diff --git a/synapse/synapse_rust/http_client.pyi b/synapse/synapse_rust/http_client.pyi index 1814daec73..e6468bc46b 100644 --- a/synapse/synapse_rust/http_client.pyi +++ b/synapse/synapse_rust/http_client.pyi @@ -14,7 +14,7 @@ from typing import Mapping from twisted.internet.defer import Deferred -from synapse.types import ISynapseReactor +from synapse.synapse_rust.runtime import RustRuntime class HttpClient: """ @@ -23,7 +23,7 @@ class HttpClient: def __init__( self, - reactor: ISynapseReactor, + runtime: RustRuntime, user_agent: str, http2_only: bool = False, ) -> None: @@ -31,7 +31,7 @@ class HttpClient: Create a new HTTP client backed by reqwest. Args: - reactor: The Twisted reactor to coordinate with + runtime: The per-homeserver Rust state (`hs.get_rust_runtime()`) user_agent: The user agent to use for requests http2_only: Whether to use HTTP/2 only, even on unencrypted connections. By default, it will always use HTTP/1.1 over unencrypted connections, and diff --git a/synapse/synapse_rust/logcontext.pyi b/synapse/synapse_rust/logcontext.pyi new file mode 100644 index 0000000000..7e10128e3e --- /dev/null +++ b/synapse/synapse_rust/logcontext.pyi @@ -0,0 +1,177 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from types import TracebackType +from typing import TYPE_CHECKING, Optional + +from synapse.logging.context import ContextRequest + +if TYPE_CHECKING: + from synapse.logging.scopecontextmanager import _LogContextScope + +DEBUG_LOGGER_NAME: str +"""Name of the opt-in logger for debugging when the logcontext switches +(`synapse.logging.context.debug`). Shared with the Rust `debug!` target so the +names stay in sync.""" + +class ContextResourceUsage: + """Tracks the resources used by a log context. + + The attributes, operators and `repr` format are relied on by Python callers + (`Measure`, request and background-process metrics, the task scheduler, + etc). Implemented in Rust so that `set_current_context` can update the + counters without allocating a Python object each time. + """ + + ru_stime: float + """System CPU time, in seconds.""" + ru_utime: float + """User CPU time, in seconds.""" + db_txn_count: int + """Number of database transactions done.""" + db_txn_duration_sec: float + """Time spent doing database transactions (excluding scheduling), in seconds.""" + db_sched_duration_sec: float + """Time spent waiting for a database connection, in seconds.""" + evt_db_fetch_count: int + """Number of events requested from the database.""" + + def __init__(self, copy_from: "Optional[ContextResourceUsage]" = None) -> None: + """If `copy_from` is given, copy its stats; otherwise start at zero.""" + + def copy(self) -> "ContextResourceUsage": + """Return a copy of this object.""" + + def reset(self) -> None: + """Reset all stats to zero.""" + + def __iadd__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": + """`self += other`, mutating in place.""" + + def __isub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": + """`self -= other`, mutating in place.""" + + def __add__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": + """`self + other`, returning a new object.""" + + def __sub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": + """`self - other`, returning a new object.""" + +class LoggingContext: + """Additional context for log formatting. Contexts are scoped within a + "with" block. + + If a parent is given when creating a new context, then: + - logging fields are copied from the parent to the new context on entry + - when the new context exits, the cpu usage stats are copied from the + child to the parent + """ + + # The context that was current when this one was created. None means the + # sentinel. + previous_context: Optional[LoggingContext] + name: str + server_name: str + parent_context: "Optional[LoggingContext]" + main_thread: int + finished: bool + request: Optional[ContextRequest] + # Narrower than the runtime: the setter also accepts None (see the Rust + # field docs for why), but all in-tree code treats this as str. + tag: str + scope: "Optional[_LogContextScope]" + + def __init__( + self, + *, + name: str, + server_name: str, + parent_context: "Optional[LoggingContext]" = None, + request: Optional[ContextRequest] = None, + ) -> None: + """ + Args: + name: Name for the context for logging. + server_name: The name of the server this context is associated with + (`config.server.server_name` or `hs.hostname`). + parent_context: The parent of the new context. + request: Synapse Request Context object. Useful to associate all the + logs happening to a given request. + """ + + def __str__(self) -> str: ... + def __enter__(self) -> "LoggingContext": ... + def __exit__( + self, + type: Optional[type[BaseException]], + value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: ... + def start(self, rusage: "Optional[tuple[float, float]]") -> None: + """Record that this logcontext is currently running. + + Should not be called directly: use `set_current_context`. + + Args: + rusage: The thread CPU usage `(ru_utime, ru_stime)` at the point of + switching to this context, or None if the platform doesn't + track it. + """ + + def stop(self, rusage: "Optional[tuple[float, float]]") -> None: + """Record that this logcontext is no longer running. + + Should not be called directly: use `set_current_context`. + + Args: + rusage: The thread CPU usage `(ru_utime, ru_stime)` at the point of + switching away from this context, or None if the platform + doesn't track it. + """ + + def get_resource_usage(self) -> ContextResourceUsage: + """Get the resources used by this logcontext so far. + + Returns: + A *copy* of the object tracking resource usage so far. + """ + + def add_cputime(self, utime_delta: float, stime_delta: float) -> None: + """Update the CPU time usage of this context (and any parents, recursively).""" + + def add_database_transaction(self, duration_sec: float) -> None: + """Record the use of a database transaction and how long it took.""" + + def add_database_scheduled(self, sched_sec: float) -> None: + """Record a use of the database pool (the time taken to get a connection).""" + + def record_event_fetch(self, event_count: int) -> None: + """Record a number of events being fetched from the db.""" + +def current_context() -> Optional[LoggingContext]: + """Get the current logging context, or None for the sentinel. + + This is not the Python-facing API. + `synapse.logging.context.current_context` wraps this and returns + `SENTINEL_CONTEXT` instead of `None`. + """ + +def set_current_context( + context: Optional[LoggingContext], +) -> Optional[LoggingContext]: + """Set the current logging context, returning the context that was + previously current. `None` means the sentinel, in both directions. + + This is not the Python-facing API. + `synapse.logging.context.set_current_context` wraps this, converting between + `SENTINEL_CONTEXT` and `None`. + """ diff --git a/synapse/synapse_rust/push.pyi b/synapse/synapse_rust/push.pyi index ef0d5f94f4..eede2c5c37 100644 --- a/synapse/synapse_rust/push.pyi +++ b/synapse/synapse_rust/push.pyi @@ -48,6 +48,7 @@ class FilteredPushRules: msc3381_polls_enabled: bool, msc3664_enabled: bool, msc4028_push_encrypted_events: bool, + msc4075_enabled: bool, msc4210_enabled: bool, msc4306_enabled: bool, ): ... diff --git a/synapse/synapse_rust/room_versions.pyi b/synapse/synapse_rust/room_versions.pyi index 9bbb538f18..1f99f6d29b 100644 --- a/synapse/synapse_rust/room_versions.pyi +++ b/synapse/synapse_rust/room_versions.pyi @@ -123,6 +123,28 @@ class RoomVersion: to the create event every time we insert an event would be prohibitively expensive. This is similar to how doubly-linked lists can potentially not refer to previous items correctly without verifying the list's integrity, but doing it on every insert is too expensive.""" + msc4311_stripped_state: bool + """ + Determines whether a room version *SHOULD* rather than *MAY* reject invites/knocks + with invalid stripped state events. + + According to Matrix v1.18 (introduced in MSC4311) for the `PUT + /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + > The `invite_room_state` has additional validation, which servers MAY apply to + > room versions 1 through 11 and SHOULD apply to all other room versions. As with + > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + > - The `m.room.create` event is missing from `invite_room_state`. + > - One or more entries in `invite_room_state` are not formatted according to the room's version. + > - One or more events fails a signature check. + > - One or more events does not reside in the same room as the invite. + + Regardless of room version (we should always do these things): + 1. The `m.room.create` event *MUST* be included in + `invite_room_state`/`knock_room_state` when sending invites/knocks over the + federation API's. + 2. Use full PDU's in the `invite_room_state`/`knock_room_state` in the federation + API. The client API still uses stripped state. + """ class RoomVersions: V1: RoomVersion diff --git a/synapse/synapse_rust/runtime.pyi b/synapse/synapse_rust/runtime.pyi new file mode 100644 index 0000000000..32ae77ec28 --- /dev/null +++ b/synapse/synapse_rust/runtime.pyi @@ -0,0 +1,15 @@ +from synapse.server import HomeServer + +class RustRuntime: + """The per-homeserver state for the Rust side of Synapse. + + Holds the tokio thread pool (started lazily on first use, shut down with the + Homeserver). Rust classes that need them take this as a constructor + argument. Get it from `hs.get_rust_runtime()`. + """ + + def __init__( + self, + hs: HomeServer, + worker_threads: int = 4, + ) -> None: ... diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 7516847303..90d6c9f32f 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -31,6 +31,7 @@ from typing import ( Any, ClassVar, Final, + Iterator, Literal, Mapping, Match, @@ -1392,20 +1393,22 @@ class StreamToken: """Returns the stream ID for the given key.""" return getattr(self, key.value) - def is_before_or_eq(self, other_token: "StreamToken") -> bool: - """Wether this token is before the other token, i.e. every constituent - part is before the other. + def _iter_fields_strictly_after_token( + self, other_token: "StreamToken" + ) -> Iterator[StreamKeyType]: + """The keys where `other_token` is behind this token. - Essentially it is `self <= other`. + A generator so that `is_before_or_eq` can stop at the first one. - Note: if `self.is_before_or_eq(other_token) is False` then that does not - imply that the reverse is True. + The typing key is never yielded. That stream is allowed to "reset", and + so comparisons don't really make sense as is. + TODO: Figure out a better way of tracking resets. """ for _, key in StreamKeyType.__members__.items(): if key == StreamKeyType.TYPING: - # Typing stream is allowed to "reset", and so comparisons don't - # really make sense as is. + # The typing key is never yielded. That stream is allowed to + # "reset", and so comparisons don't really make sense as is. # TODO: Figure out a better way of tracking resets. continue @@ -1415,17 +1418,34 @@ class StreamToken: if isinstance(self_value, RoomStreamToken): assert isinstance(other_value, RoomStreamToken) if not self_value.is_before_or_eq(other_value): - return False + yield key elif isinstance(self_value, MultiWriterStreamToken): assert isinstance(other_value, MultiWriterStreamToken) if not self_value.is_before_or_eq(other_value): - return False + yield key else: assert isinstance(other_value, int) if self_value > other_value: - return False + yield key - return True + def fields_strictly_after_token( + self, other_token: "StreamToken" + ) -> list[StreamKeyType]: + """The keys where `other_token` is behind this token, i.e. the keys that + stop `self.is_before_or_eq(other_token)` from being True. + """ + return list(self._iter_fields_strictly_after_token(other_token)) + + def is_before_or_eq(self, other_token: "StreamToken") -> bool: + """Whether this token is before the other token, i.e. every constituent + part is before the other. + + Essentially it is `self <= other`. + + Note: if `self.is_before_or_eq(other_token) is False` then that does not + imply that the reverse is True. + """ + return next(self._iter_fields_strictly_after_token(other_token), None) is None def __str__(self) -> str: return ( diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index dbd4455b82..cd1bd1f057 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -58,6 +58,17 @@ class AuthenticationData(RequestBodyModel): type: StrictStr | None = None +class UserInteractiveAuthBody(RequestBodyModel): + """ + A request body that may carry user-interactive authentication data. Other keys + are endpoint-specific and are left for the endpoint to handle. + """ + + model_config = ConfigDict(extra="allow") + + auth: AuthenticationData | None = None + + # See also assert_valid_client_secret() ClientSecretStr = Annotated[ str, diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 977a662d7a..38e57e572e 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -151,19 +151,35 @@ class MutableOverlayMapping(collections.abc.MutableMapping[K, V]): yield key def __len__(self) -> int: - count = len(self._underlying_map) - for key in self._deletions: - if key in self._underlying_map: - count -= 1 + # The distinct keys can be calculated via `(underlying ∪ mutable) − + # deletions`. A key is never in both `_mutable_map` and `_deletions`, so + # the only deletions to subtract are those of keys in the underlying + # map: + # + # |underlying| + |mutable| − |mutable ∩ underlying| − |underlying ∩ deletions| + # + # The intersections run at C speed and iterates over the smaller + # operand, so this is much cheaper than a Python loop over every key. + underlying_keys = self._underlying_map.keys() + return ( + len(self._underlying_map) + + len(self._mutable_map) + - len(self._mutable_map.keys() & underlying_keys) + - len(underlying_keys & self._deletions) + ) - for key in self._mutable_map: - # `key` should not be in both _mutable_map and _deletions - assert key not in self._deletions + def total_entries(self) -> int: + """The number of entries held across the underlying map, the + overrides and the deletions, following nested overlays down. - if key not in self._underlying_map: - count += 1 - - return count + Useful for estimating the memory usage of the overlay mapping. + """ + underlying = self._underlying_map + if isinstance(underlying, MutableOverlayMapping): + underlying_size = underlying.total_entries() + else: + underlying_size = len(underlying) + return underlying_size + len(self._mutable_map) + len(self._deletions) def clear(self) -> None: self._underlying_map = {} diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 87870f4223..367ec6b413 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -24,8 +24,8 @@ from collections import OrderedDict from typing import ( TYPE_CHECKING, Any, + Callable, Generic, - Iterable, Literal, TypeVar, overload, @@ -66,6 +66,7 @@ class ExpiringCache(Generic[KT, VT]): expiry_ms: int = 0, reset_expiry_on_get: bool = False, iterable: bool = False, + size_callback: Callable[[VT], int] | None = None, ): """ Args: @@ -83,7 +84,18 @@ class ExpiringCache(Generic[KT, VT]): an item on access. Defaults to False. iterable: If true, the size is calculated by summing the sizes of all entries, rather than the number of entries. + Shorthand for `size_callback=len`. + size_callback: If given, the size of the cache is the sum of this + function over all values, rather than the number of entries. """ + if iterable and size_callback is not None: + raise ValueError("`iterable` and `size_callback` are exclusive") + if iterable: + # type-ignore: if `iterable` is true, then the value type VT should + # be Sized (i.e. have a `__len__` method). We don't enforce this via + # the type system at present. + size_callback = len # type: ignore[assignment] + self._cache_name = cache_name self._original_max_size = max_len @@ -97,7 +109,7 @@ class ExpiringCache(Generic[KT, VT]): self._cache: OrderedDict[KT, _CacheEntry[VT]] = OrderedDict() - self.iterable = iterable + self._size_callback = size_callback self.metrics = register_cache( cache_type="expiring", @@ -124,13 +136,7 @@ class ExpiringCache(Generic[KT, VT]): # Evict if there are now too many items while self._max_size and len(self) > self._max_size: _key, value = self._cache.popitem(last=False) - if self.iterable: - # type-ignore, here and below: if self.iterable is true, then the value - # type VT should be Sized (i.e. have a __len__ method). We don't enforce - # this via the type system at present. - self.metrics.inc_evictions(EvictionReason.size, len(value.value)) # type: ignore[arg-type] - else: - self.metrics.inc_evictions(EvictionReason.size) + self.metrics.inc_evictions(EvictionReason.size, self._size_of(value.value)) def __getitem__(self, key: KT) -> VT: try: @@ -161,10 +167,9 @@ class ExpiringCache(Generic[KT, VT]): raise KeyError(key) return default - if self.iterable: - self.metrics.inc_evictions(EvictionReason.invalidation, len(value.value)) - else: - self.metrics.inc_evictions(EvictionReason.invalidation) + self.metrics.inc_evictions( + EvictionReason.invalidation, self._size_of(value.value) + ) return value.value @@ -207,10 +212,7 @@ class ExpiringCache(Generic[KT, VT]): for k in keys_to_delete: value = self._cache.pop(k) - if self.iterable: - self.metrics.inc_evictions(EvictionReason.time, len(value.value)) # type: ignore[arg-type] - else: - self.metrics.inc_evictions(EvictionReason.time) + self.metrics.inc_evictions(EvictionReason.time, self._size_of(value.value)) logger.debug( "[%s] _prune_cache before: %d, after len: %d", @@ -219,12 +221,16 @@ class ExpiringCache(Generic[KT, VT]): len(self), ) + def _size_of(self, value: VT) -> int: + """How much a value counts towards `max_len`.""" + if self._size_callback is None: + return 1 + return self._size_callback(value) + def __len__(self) -> int: - if self.iterable: - g: Iterable[int] = (len(entry.value) for entry in self._cache.values()) # type: ignore[arg-type] - return sum(g) - else: + if self._size_callback is None: return len(self._cache) + return sum(self._size_callback(entry.value) for entry in self._cache.values()) def set_cache_factor(self, factor: float) -> bool: """ diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index c01ddf7339..aef3eb681c 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -83,8 +83,10 @@ class TaskScheduler: - The reconciliation loop runs every minute, so this is not a high-precision scheduler. - - Only 10 tasks can run at the same time. If the pool is full, tasks may be - delayed. Make sure your scheduled tasks can actually finish. + - Only a limited number of tasks can run at the same time (configured by + the `task_scheduler.max_concurrent_tasks` setting, defaulting to 5). If the + pool is full, tasks may be delayed. Make sure your scheduled tasks can + actually finish. - Currently, there's no way to stop a task if it gets stuck. @@ -100,8 +102,9 @@ class TaskScheduler: CLEANUP_INTERVAL_MS = 30 * 60 * 1000 # Time before a complete or failed task is deleted from the DB KEEP_TASKS_FOR_MS = 7 * 24 * 60 * 60 * 1000 # 1 week - # Maximum number of tasks that can run at the same time - MAX_CONCURRENT_RUNNING_TASKS = 5 + # Default maximum number of tasks that can run at the same time + DEFAULT_MAX_CONCURRENT_RUNNING_TASKS = 5 + MAX_CONCURRENT_RUNNING_TASKS = DEFAULT_MAX_CONCURRENT_RUNNING_TASKS # Time from the last task update after which we will log a warning LAST_UPDATE_BEFORE_WARNING_MS = 24 * 60 * 60 * 1000 # 24hrs # Report a running task's status and usage every so often. @@ -112,6 +115,8 @@ class TaskScheduler: self.server_name = hs.hostname self._store = hs.get_datastores().main self._clock = hs.get_clock() + self._max_concurrent_tasks = hs.config.task_scheduler.max_concurrent_tasks + self.MAX_CONCURRENT_RUNNING_TASKS = self._max_concurrent_tasks # A map between a task's ID and a deferred linked to the task self._running_tasks: dict[str, defer.Deferred] = {} # A map between action names and their registered function @@ -366,7 +371,7 @@ class TaskScheduler: def _launch_scheduled_tasks(self) -> None: """Retrieve and launch scheduled tasks that should be running at this time.""" # Don't bother trying to launch new tasks if we're already at capacity. - if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: + if len(self._running_tasks) >= self._max_concurrent_tasks: return if self._launching_new_tasks: @@ -378,7 +383,7 @@ class TaskScheduler: try: for task in await self.get_tasks( statuses=[TaskStatus.ACTIVE], - limit=self.MAX_CONCURRENT_RUNNING_TASKS, + limit=self._max_concurrent_tasks, ): # _launch_task will ignore tasks that we're already running, and # will also do nothing if we're already at the maximum capacity. @@ -386,7 +391,7 @@ class TaskScheduler: for task in await self.get_tasks( statuses=[TaskStatus.SCHEDULED], max_timestamp=self._clock.time_msec(), - limit=self.MAX_CONCURRENT_RUNNING_TASKS, + limit=self._max_concurrent_tasks, ): await self._launch_task(task) @@ -509,7 +514,7 @@ class TaskScheduler: self._launch_scheduled_tasks, ) - if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: + if len(self._running_tasks) >= self._max_concurrent_tasks: return if ( diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index e7ee7eec91..5604f41437 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -106,6 +106,41 @@ class AuthTestCase(unittest.HomeserverTestCase): self.assertEqual(f.code, 401) self.assertEqual(f.errcode, "M_MISSING_TOKEN") + def test_get_optional_user_by_req_valid_token(self) -> None: + user_info = TokenLookupResult( + user_id=self.test_user_id.to_string(), token_id=5, device_id="device" + ) + self.store.get_user_by_access_token = AsyncMock(return_value=user_info) + self.store.mark_access_token_as_used = AsyncMock(return_value=None) + self.store.get_user_locked_status = AsyncMock(return_value=False) + + request = Mock(args={}) + request.args[b"access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + requester = self.get_success(self.auth.get_optional_user_by_req(request)) + assert requester is not None + self.assertEqual(requester.user, self.test_user_id) + + def test_get_optional_user_by_req_bad_token(self) -> None: + """A token that is present but invalid is still rejected.""" + self.store.get_user_by_access_token = AsyncMock(return_value=None) + + request = Mock(args={}) + request.args[b"access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + f = self.get_failure( + self.auth.get_optional_user_by_req(request), InvalidClientTokenError + ).value + self.assertEqual(f.code, 401) + self.assertEqual(f.errcode, Codes.UNKNOWN_TOKEN) + + def test_get_optional_user_by_req_missing_token(self) -> None: + """A request without any token yields no requester rather than an error.""" + request = Mock(args={}) + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + requester = self.get_success(self.auth.get_optional_user_by_req(request)) + self.assertIsNone(requester) + def test_get_user_by_req_appservice_valid_token(self) -> None: app_service = Mock( id="as_id", diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 6a1a630fe8..d2f5d69bca 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -22,6 +22,7 @@ from unittest.mock import Mock, patch from parameterized import parameterized from twisted.internet.testing import MemoryReactor +from twisted.web.server import Site from synapse.app.generic_worker import GenericWorkerServer from synapse.app.homeserver import SynapseHomeServer @@ -77,6 +78,7 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): # Grab the resource from the site that was told to listen site = self.reactor.tcpServers[0][1] + assert isinstance(site, Site) try: site.resource.children[b"_matrix"].children[b"federation"] except KeyError: @@ -125,6 +127,7 @@ class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): # Grab the resource from the site that was told to listen site = self.reactor.tcpServers[0][1] + assert isinstance(site, Site) try: site.resource.children[b"_matrix"].children[b"federation"] except KeyError: diff --git a/tests/config/test_server.py b/tests/config/test_server.py index f718e20a2a..12c57d33da 100644 --- a/tests/config/test_server.py +++ b/tests/config/test_server.py @@ -229,6 +229,30 @@ class ServerConfigTestCase(unittest.TestCase): with self.assertRaises(ConfigError): _read_config(generate_config(disallowed_value)) + def test_limit_profile_requests_requires_auth(self) -> None: + """ + Test that `limit_profile_requests_to_users_who_share_rooms` can only be + enabled together with `require_auth_for_profile_requests`, as the shared-room + check is only applied to authenticated requests + """ + + def generate_config(limit: bool, require_auth: bool) -> JsonDict: + return { + "limit_profile_requests_to_users_who_share_rooms": limit, + "require_auth_for_profile_requests": require_auth, + } + + _read_config(generate_config(limit=False, require_auth=False)) + _read_config(generate_config(limit=False, require_auth=True)) + _read_config(generate_config(limit=True, require_auth=True)) + + with self.assertRaises(ConfigError): + _read_config(generate_config(limit=True, require_auth=False)) + + # `require_auth_for_profile_requests` defaults to false + with self.assertRaises(ConfigError): + _read_config({"limit_profile_requests_to_users_who_share_rooms": True}) + @parameterized.expand( [ [ diff --git a/tests/config/test_task_scheduler_config.py b/tests/config/test_task_scheduler_config.py new file mode 100644 index 0000000000..804ac0aab7 --- /dev/null +++ b/tests/config/test_task_scheduler_config.py @@ -0,0 +1,54 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2026 Nevil Anson Dsouza +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +from synapse.config._base import ConfigError, RootConfig +from synapse.config.homeserver import HomeServerConfig +from synapse.config.task_scheduler import TaskSchedulerConfig + +from tests.unittest import TestCase +from tests.utils import default_config + + +class TaskSchedulerConfigTestCase(TestCase): + def test_default_configuration(self) -> None: + config_dict = default_config(server_name="test") + config = HomeServerConfig() + config.parse_config_dict(config_dict, "", "") + self.assertEqual(config.task_scheduler.max_concurrent_tasks, 5) + + def test_custom_configuration(self) -> None: + config_dict = default_config(server_name="test") + config_dict["task_scheduler"] = {"max_concurrent_tasks": 4} + config = HomeServerConfig() + config.parse_config_dict(config_dict, "", "") + self.assertEqual(config.task_scheduler.max_concurrent_tasks, 4) + + def test_invalid_configuration(self) -> None: + config = TaskSchedulerConfig(RootConfig()) + with self.assertRaises(ConfigError): + config.read_config({"task_scheduler": {"max_concurrent_tasks": 0}}) + + with self.assertRaises(ConfigError): + config.read_config({"task_scheduler": {"max_concurrent_tasks": -1}}) + + with self.assertRaises(ConfigError): + config.read_config( + {"task_scheduler": {"max_concurrent_tasks": "not-an-int"}} + ) + + with self.assertRaises(ConfigError): + config.read_config({"task_scheduler": {"max_concurrent_tasks": True}}) + + with self.assertRaises(ConfigError): + config.read_config({"task_scheduler": "not-a-dict"}) diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py index 5c2cefaf5b..c98d90bfa0 100644 --- a/tests/config/test_tls.py +++ b/tests/config/test_tls.py @@ -34,6 +34,7 @@ from synapse.crypto.context_factory import ( ) from synapse.types import JsonDict +from tests.http import get_test_ca_cert_file from tests.unittest import TestCase @@ -49,6 +50,13 @@ class TestConfig(RootConfig): class TLSConfigTests(TestCase): + def test_custom_ca_certificate(self) -> None: + """Custom CA files are loaded as PEM bytes for Twisted.""" + config: JsonDict = {"federation_custom_ca_list": [get_test_ca_cert_file()]} + t = TestConfig() + t.tls.read_config(config, config_dir_path="", data_dir_path="") + self.assertIsNotNone(t.tls.federation_ca_trust_root) + def test_tls_client_minimum_default(self) -> None: """ The default client TLS version is 1.0. diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index 632d1dc4f6..c44b4e011e 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -198,9 +198,9 @@ class AutoAcceptInvitesTestCase(FederatingHomeserverTestCase): ) self.get_success( self.handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ) ) @@ -324,9 +324,9 @@ class AutoAcceptInvitesTestCase(FederatingHomeserverTestCase): ) self.get_success( self.handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ) ) else: diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 9fd85fc542..d018568798 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -35,7 +35,6 @@ from synapse.events.utils import ( format_event_for_client_v2_without_room_id, format_event_raw, maybe_upsert_event_field, - parse_stripped_state_event, prune_event, ) from synapse.types import JsonDict @@ -1004,7 +1003,7 @@ class CopyPowerLevelsContentTestCase(stdlib_unittest.TestCase): copy_and_fixup_power_levels_contents({"a": {"b": {"c": 1}}}) # type: ignore[dict-item] -class TestParseStrippedStateEvent(stdlib_unittest.TestCase): +class TestStrippedStateEvent(stdlib_unittest.TestCase): def test_valid_dict(self) -> None: """A valid dict should be parsed into a StrippedStateEvent.""" raw = { @@ -1013,7 +1012,7 @@ class TestParseStrippedStateEvent(stdlib_unittest.TestCase): "sender": "@alice:example.com", "content": {"membership": "join"}, } - result = parse_stripped_state_event(raw) + result = StrippedStateEvent.from_json_dict(raw) self.assertEqual( result, StrippedStateEvent( @@ -1024,17 +1023,10 @@ class TestParseStrippedStateEvent(stdlib_unittest.TestCase): ), ) - def test_invalid_type(self) -> None: - """Non-dict inputs should return None.""" - self.assertIsNone(parse_stripped_state_event("string")) - self.assertIsNone(parse_stripped_state_event(123)) - self.assertIsNone(parse_stripped_state_event([])) - self.assertIsNone(parse_stripped_state_event(None)) - def test_missing_fields(self) -> None: """Dicts with missing required fields should return None.""" self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "state_key": "@alice:example.com", "sender": "@alice:example.com", @@ -1043,7 +1035,7 @@ class TestParseStrippedStateEvent(stdlib_unittest.TestCase): ) ) self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "sender": "@alice:example.com", @@ -1052,7 +1044,7 @@ class TestParseStrippedStateEvent(stdlib_unittest.TestCase): ) ) self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": "@alice:example.com", @@ -1061,7 +1053,7 @@ class TestParseStrippedStateEvent(stdlib_unittest.TestCase): ) ) self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": "@alice:example.com", @@ -1074,7 +1066,7 @@ class TestParseStrippedStateEvent(stdlib_unittest.TestCase): """Dicts with invalid field types should return None.""" # Type must be string self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": 123, "state_key": "@alice:example.com", @@ -1085,7 +1077,7 @@ class TestParseStrippedStateEvent(stdlib_unittest.TestCase): ) # State key must be string self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": 123, @@ -1096,7 +1088,7 @@ class TestParseStrippedStateEvent(stdlib_unittest.TestCase): ) # Sender must be string self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": "@alice:example.com", @@ -1107,7 +1099,7 @@ class TestParseStrippedStateEvent(stdlib_unittest.TestCase): ) # Content must be dict self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": "@alice:example.com", diff --git a/tests/federation/test_federation_client.py b/tests/federation/test_federation_client.py index 656665cb42..47a4df942d 100644 --- a/tests/federation/test_federation_client.py +++ b/tests/federation/test_federation_client.py @@ -18,14 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # - +import urllib from unittest import mock import twisted.web.client from twisted.internet import defer from twisted.internet.testing import MemoryReactor -from synapse.api.room_versions import RoomVersions from synapse.events import EventBase from synapse.rest import admin from synapse.rest.client import login, room @@ -33,6 +32,7 @@ from synapse.server import HomeServer from synapse.util.clock import Clock from tests.test_utils import FakeResponse, event_injection +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import FederatingHomeserverTestCase @@ -58,28 +58,42 @@ class FederationClientTest(FederatingHomeserverTestCase): self.reactor.advance(1000000000) self.creator = f"@creator:{self.OTHER_SERVER_NAME}" - self.test_room_id = "!room_id" + self.room_version = self.hs.config.server.default_room_version def test_get_room_state(self) -> None: # mock up some events to use in the response. - # In real life, these would have things in `prev_events` and `auth_events`, but that's - # a bit annoying to mock up, and the code under test doesn't care, so we don't bother. + # In real life, these would have things in `prev_events` and `auth_events`, but + # that's a bit annoying to mock up, and the code under test doesn't care, so we + # don't bother. We do bother with the room_id though, since it has to be + # generated by the hash of the creation event. + create_event_source = { + "type": "m.room.create", + "state_key": "", + "sender": self.creator, + "content": {"room_version": self.room_version.identifier}, + "prev_events": [], + "auth_events": [], + "depth": 1, + "origin_server_ts": 500, + } + if not self.room_version.msc4291_room_ids_as_hashes: + # Do not bother saving this now, we will pull it out of the calculated event + # later. + create_event_source["room_id"] = f"!room:{self.OTHER_SERVER_NAME}" + create_event_dict = self.add_hashes_and_signatures_from_other_server( - { - "room_id": self.test_room_id, - "type": "m.room.create", - "state_key": "", - "sender": self.creator, - "content": {"creator": self.creator}, - "prev_events": [], - "auth_events": [], - "depth": 1, - "origin_server_ts": 500, - } + create_event_source, + self.room_version, ) + create_event = make_test_pdu_event( + create_event_source, + self.room_version, + ) + room_id = create_event.room_id + member_event_dict = self.add_hashes_and_signatures_from_other_server( { - "room_id": self.test_room_id, + "room_id": room_id, "type": "m.room.member", "sender": self.creator, "state_key": self.creator, @@ -88,11 +102,12 @@ class FederationClientTest(FederatingHomeserverTestCase): "auth_events": [], "depth": 2, "origin_server_ts": 600, - } + }, + self.room_version, ) pl_event_dict = self.add_hashes_and_signatures_from_other_server( { - "room_id": self.test_room_id, + "room_id": room_id, "type": "m.room.power_levels", "sender": self.creator, "state_key": "", @@ -101,7 +116,8 @@ class FederationClientTest(FederatingHomeserverTestCase): "auth_events": [], "depth": 3, "origin_server_ts": 700, - } + }, + self.room_version, ) # mock up the response, and have the agent return it @@ -125,16 +141,18 @@ class FederationClientTest(FederatingHomeserverTestCase): state_resp, auth_resp = self.get_success( self.hs.get_federation_client().get_room_state( "yet.another.server", - self.test_room_id, + room_id, "event_id", - RoomVersions.V9, + self.room_version, ) ) # check the right call got made to the agent self._mock_agent.request.assert_called_once_with( b"GET", - b"matrix-federation://yet.another.server/_matrix/federation/v1/state/%21room_id?event_id=event_id", + f"matrix-federation://yet.another.server/_matrix/federation/v1/state/{urllib.parse.quote(room_id, '')}?event_id=event_id".encode( + "ascii" + ), headers=mock.ANY, bodyProducer=None, ) @@ -158,7 +176,7 @@ class FederationClientTest(FederatingHomeserverTestCase): self.hs.get_federation_client().get_pdu( ["yet.another.server"], "event_should_not_exist", - RoomVersions.V9, + self.room_version, ) ) self.assertEqual(pulled_pdu_info, None) @@ -185,7 +203,7 @@ class FederationClientTest(FederatingHomeserverTestCase): self.hs.get_federation_client().get_pdu( ["yet.another.server"], remote_pdu.event_id, - RoomVersions.V9, + self.room_version, ) ) assert pulled_pdu_info2 is not None @@ -204,7 +222,7 @@ class FederationClientTest(FederatingHomeserverTestCase): """ message_event_dict = self.add_hashes_and_signatures_from_other_server( { - "room_id": self.test_room_id, + "room_id": f"!bogus_made_up_room_id:{self.OTHER_SERVER_NAME}", "type": "m.room.message", "sender": self.creator, "state_key": "", @@ -213,7 +231,8 @@ class FederationClientTest(FederatingHomeserverTestCase): "auth_events": [], "origin_server_ts": 700, "depth": 10, - } + }, + self.room_version, ) # mock up the response, and have the agent return it @@ -233,7 +252,7 @@ class FederationClientTest(FederatingHomeserverTestCase): self.hs.get_federation_client().get_pdu( ["yet.another.server"], "event_id", - RoomVersions.V9, + self.room_version, ) ) assert pulled_pdu_info is not None diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py index 85e221a64a..3adbf4e35b 100644 --- a/tests/federation/test_federation_out_of_band_membership.py +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -32,7 +32,7 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventContentFields, EventTypes, Membership -from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.api.room_versions import RoomVersion from synapse.events import EventBase from synapse.events.utils import strip_event from synapse.federation.transport.client import SendJoinResponse @@ -188,13 +188,11 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): # Create a remote room room_creator_user_id = f"@remote-user:{self.OTHER_SERVER_NAME}" - remote_room_id = f"!remote-room:{self.OTHER_SERVER_NAME}" - room_version = RoomVersions.V11 + room_version = self.hs.config.server.default_room_version room_create_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { - "room_id": remote_room_id, "sender": room_creator_user_id, "depth": 1, "origin_server_ts": 1, @@ -210,6 +208,8 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): room_version=room_version, ) + remote_room_id = room_create_event.room_id + creator_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { @@ -220,7 +220,7 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): "type": EventTypes.Member, "state_key": room_creator_user_id, "content": {"membership": Membership.JOIN}, - "auth_events": [room_create_event.event_id], + "auth_events": [], "prev_events": [room_create_event.event_id], } ), @@ -239,7 +239,6 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): "state_key": local_user1_id, "content": {"membership": Membership.INVITE}, "auth_events": [ - room_create_event.event_id, creator_membership_event.event_id, ], "prev_events": [creator_membership_event.event_id], @@ -300,7 +299,6 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): "state_key": local_user1_id, "content": {"membership": Membership.JOIN}, "auth_events": [ - room_create_event.event_id, user1_invite_membership_event.event_id, ], "prev_events": [user1_invite_membership_event.event_id], @@ -542,9 +540,6 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): "state_key": local_user2_id, "content": {"membership": Membership.INVITE}, "auth_events": [ - remote_room_join_result.state_map[ - (EventTypes.Create, "") - ].event_id, remote_room_join_result.state_map[ ( EventTypes.Member, diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index ec705676cc..cdaae5cfd7 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -184,14 +184,11 @@ class KnockingStrippedStateEventHelperMixin(HomeserverTestCase): expected_room_state[event_type]["content"], event["content"] ) - # Check the state key is correct + # Check the state_key is correct self.assertEqual( expected_room_state[event_type]["state_key"], event["state_key"] ) - # Ensure the event has been stripped - self.assertNotIn("signatures", event) - # Pop once we've found and processed a state event expected_room_state.pop(event_type) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 011d0552a5..43fac00296 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -29,6 +29,7 @@ from typing import ( ) from unittest.mock import AsyncMock, Mock +from immutabledict import immutabledict from parameterized import parameterized from twisted.internet import defer @@ -77,6 +78,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): self.reactor, self.clock = get_clock() hs = Mock() + hs.hostname = "test_server" def test_run_as_background_process( desc: "LiteralString", @@ -340,7 +342,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): event = Mock(event_id="event_1") self.event_source.sources.receipt.get_new_events_as = AsyncMock( - return_value=([event], None) + return_value=([event], MultiWriterStreamToken(stream=580)) ) self.handler.notify_interested_services_ephemeral( @@ -370,7 +372,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): event = Mock(event_id="event_1") self.event_source.sources.receipt.get_new_events_as = AsyncMock( - return_value=([event], None) + return_value=([event], MultiWriterStreamToken(stream=580)) ) self.handler.notify_interested_services_ephemeral( @@ -803,6 +805,276 @@ class ApplicationServicesHandlerSendEventsTestCase(unittest.HomeserverTestCase): }, ) + def test_sending_read_receipt_batches_with_single_token_to_application_services( + self, + ) -> None: + """Tests that a large batch of read receipts covered by a single stream + token notification (e.g. a burst arriving over federation, or an + application service catching up after downtime) is sent in full, rather + than being truncated by the per-fetch receipt limit. + """ + interested_appservice = self._register_application_service( + namespaces={ + ApplicationService.NS_USERS: [ + { + "regex": "@exclusive_as_user:.+", + "exclusive": True, + } + ], + ApplicationService.NS_ROOMS: [ + { + "regex": "!fakeroom_.*", + "exclusive": True, + } + ], + }, + ) + + # Deliver a first receipt to establish a stored read receipt stream + # position for this appservice, as an appservice without one is + # fast-forwarded to the most recent receipts instead of backfilling. + # note: stream tokens start at 2 + self.get_success( + self.hs.get_datastores().main.insert_receipt( + room_id="!fakeroom_bootstrap:test", + receipt_type="m.read", + user_id=self.local_user, + event_ids=["$eventid_bootstrap"], + thread_id=None, + data={}, + ) + ) + self.get_success( + self.hs.get_application_service_handler()._notify_interested_services_ephemeral( + services=[interested_appservice], + stream_key=StreamKeyType.RECEIPT, + new_token=MultiWriterStreamToken(stream=2), + users=[self.exclusive_as_user], + ) + ) + self.send_mock.reset_mock() + + # Insert a large burst of read receipts (300 total, past the per-fetch + # limit of 100), occupying stream IDs 3 to 302. + for i in range(300): + self.get_success( + self.hs.get_datastores().main.insert_receipt( + # We have to use unique room ID + user ID combinations here, as the db query + # is an upsert. + room_id=f"!fakeroom_{i}:test", + receipt_type="m.read", + user_id=self.local_user, + event_ids=[f"$eventid_{i}"], + thread_id=None, + data={}, + ) + ) + + # Now notify the appservice handler with a single token covering all 300 + # read receipts at once. + self.get_success( + self.hs.get_application_service_handler()._notify_interested_services_ephemeral( + services=[interested_appservice], + stream_key=StreamKeyType.RECEIPT, + new_token=MultiWriterStreamToken(stream=302), + users=[self.exclusive_as_user], + ) + ) + + # Using our txn send mock, we can see what the AS received. After iterating over every + # transaction, we'd like to see all 300 read receipts accounted for. + # No more, no less. + all_ephemeral_events = [] + for call in self.send_mock.call_args_list: + ephemeral_events = call[0][2] + all_ephemeral_events += ephemeral_events + + self.assertEqual(len(all_ephemeral_events), 300) + + # The stored stream position should have caught up with the notified token. + self.assertEqual( + self.get_success( + self.hs.get_datastores().main.get_type_stream_id_for_appservice( + interested_appservice, "read_receipt" + ) + ), + 302, + ) + + def test_read_receipts_from_lagging_writer_are_not_skipped(self) -> None: + """ + With several receipt writers, a worker can be notified with a token in + which one writer is ahead of the others, e.g. because its replication + rows arrived first. The application service handler must not treat the + leading writer's position as the point it has caught up to, or the + lagging writer's receipts would be skipped for good once they arrive. + + Scenario: writers `rw1` and `rw2` alternate stream IDs 1001 to 1200. + The worker has heard from `rw2` up to 1200 but from `rw1` only up to + 1000, so the receipt stream watermark is still 1000. + """ + interested_appservice = self._register_application_service( + namespaces={ + ApplicationService.NS_ROOMS: [ + { + "regex": "!fakeroom_.*", + "exclusive": True, + } + ], + }, + ) + + # The application service has already been sent everything up to 1000. + self.get_success( + self.hs.get_datastores().main.set_appservice_stream_type_pos( + interested_appservice, "read_receipt", 1000 + ) + ) + + # Insert the receipts as the two writers would have persisted them, each + # in its own room so they can be told apart in what the appservice gets. + self.get_success( + self.hs.get_datastores().main.db_pool.simple_insert_many( + desc="test_read_receipts_from_lagging_writer_are_not_skipped", + table="receipts_linearized", + keys=( + "stream_id", + "instance_name", + "room_id", + "receipt_type", + "user_id", + "event_id", + "data", + ), + values=[ + ( + stream_id, + "rw1" if stream_id % 2 else "rw2", + f"!fakeroom_{stream_id}:test", + "m.read", + self.local_user, + f"$eventid_{stream_id}", + "{}", + ) + for stream_id in range(1001, 1201) + ], + ) + ) + + def notify(new_token: MultiWriterStreamToken) -> None: + self.get_success( + self.hs.get_application_service_handler()._notify_interested_services_ephemeral( + services=[interested_appservice], + stream_key=StreamKeyType.RECEIPT, + new_token=new_token, + users=[], + ) + ) + + def received_stream_ids() -> list[int]: + return [ + int(event["room_id"].removeprefix("!fakeroom_").removesuffix(":test")) + for call in self.send_mock.call_args_list + for event in call[0][2] + ] + + def stored_position() -> int: + return self.get_success( + self.hs.get_datastores().main.get_type_stream_id_for_appservice( + interested_appservice, "read_receipt" + ) + ) + + # `rw2`'s rows have replicated to this worker, `rw1`'s have not. + notify( + MultiWriterStreamToken( + stream=1000, instance_map=immutabledict({"rw2": 1200}) + ) + ) + + # Nothing can be sent yet without risking `rw1`'s receipts being skipped, + # and the stored position must not move past them. + self.assertEqual(received_stream_ids(), []) + self.assertEqual(stored_position(), 1000) + + # `rw1`'s rows arrive and the watermark catches up. + notify(MultiWriterStreamToken(stream=1200)) + + # Every receipt from both writers is delivered exactly once, in order. + self.assertEqual(received_stream_ids(), list(range(1001, 1201))) + self.assertEqual(stored_position(), 1200) + + def test_application_services_are_fast_forwarded_on_first_read_receipt_delivery( + self, + ) -> None: + """Tests that an application service without a stored read receipt stream + position (i.e. one which has never been sent receipts before) is sent only + the most recent receipts, rather than the entire receipt history. + """ + interested_appservice = self._register_application_service( + namespaces={ + ApplicationService.NS_USERS: [ + { + "regex": "@exclusive_as_user:.+", + "exclusive": True, + } + ], + ApplicationService.NS_ROOMS: [ + { + "regex": "!fakeroom_.*", + "exclusive": True, + } + ], + }, + ) + + # Insert 300 read receipts, occupying stream IDs 2 to 301. + for i in range(300): + self.get_success( + self.hs.get_datastores().main.insert_receipt( + room_id=f"!fakeroom_{i}:test", + receipt_type="m.read", + user_id=self.local_user, + event_ids=[f"$eventid_{i}"], + thread_id=None, + data={}, + ) + ) + + # Notify the appservice handler, which has no stored stream position for + # this appservice yet. + self.get_success( + self.hs.get_application_service_handler()._notify_interested_services_ephemeral( + services=[interested_appservice], + stream_key=StreamKeyType.RECEIPT, + new_token=MultiWriterStreamToken(stream=301), + users=[self.exclusive_as_user], + ) + ) + + all_ephemeral_events = [] + for call in self.send_mock.call_args_list: + ephemeral_events = call[0][2] + all_ephemeral_events += ephemeral_events + + # Only the most recent 100 receipts (the per-fetch limit) should have been + # sent, not the whole history. + self.assertEqual(len(all_ephemeral_events), 100) + received_rooms = {event["room_id"] for event in all_ephemeral_events} + self.assertEqual( + received_rooms, {f"!fakeroom_{i}:test" for i in range(200, 300)} + ) + + # The stored stream position should nonetheless be at the notified token. + self.assertEqual( + self.get_success( + self.hs.get_datastores().main.get_type_stream_id_for_appservice( + interested_appservice, "read_receipt" + ) + ), + 301, + ) + @unittest.override_config( {"experimental_features": {"msc2409_to_device_messages_enabled": True}} ) diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index ce1bd06b6d..abc6c01d4e 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -33,10 +33,9 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, JoinRules, RoomEncryptionAlgorithms from synapse.api.errors import NotFoundError, SynapseError -from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.federation.federation_client import SendJoinResult from synapse.federation.transport.client import ( StateRequestResponse, @@ -60,6 +59,7 @@ from synapse.util.duration import Duration from synapse.util.task_scheduler import TaskScheduler from tests import unittest +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import override_config user1 = "@boris:aaa" @@ -671,8 +671,7 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): def _build_public_room(self) -> StateMap[EventBase]: """Build a public room DAG that has REMOTE1 in it""" - room_id = f"!room:{self.REMOTE1_SERVER_NAME}" - room_version = RoomVersions.V10 + room_version = self.hs.config.server.default_room_version events: list[EventBase] = [] @@ -680,13 +679,11 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): create_event_dict: JsonDict = { "auth_events": [], "content": { - "creator": self.REMOTE1_USER, "room_version": room_version.identifier, }, - "depth": 0, + "depth": 1, "origin_server_ts": 0, "prev_events": [], - "room_id": room_id, "sender": self.REMOTE1_USER, "state_key": "", "type": EventTypes.Create, @@ -699,16 +696,15 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): self.REMOTE1_SERVER_SIGNATURE_KEY, ) - create_event = make_event_from_dict(create_event_dict, room_version) + create_event = make_test_pdu_event(create_event_dict, room_version) + # This will always be the correct room_id + room_id = create_event.room_id events.append(create_event) - room_version = self.hs.config.server.default_room_version join_event_dict: JsonDict = { - "auth_events": [ - create_event.event_id, - ], + "auth_events": [], "content": {"membership": "join"}, - "depth": 1, + "depth": 2, "origin_server_ts": 100, "prev_events": [create_event.event_id], "sender": self.REMOTE1_USER, @@ -716,20 +712,21 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): "room_id": room_id, "type": EventTypes.Member, } + add_hashes_and_signatures( room_version, join_event_dict, self.hs.hostname, self.hs.signing_key, ) - join_event = make_event_from_dict(join_event_dict, room_version) + join_event = make_test_pdu_event(join_event_dict, room_version) events.append(join_event) # Then set the join rules to public join_rules_event_dict: JsonDict = { - "auth_events": [create_event.event_id, join_event.event_id], + "auth_events": [join_event.event_id], "content": {"join_rule": JoinRules.PUBLIC}, - "depth": 2, + "depth": 3, "origin_server_ts": 200, "prev_events": [join_event.event_id], "room_id": room_id, @@ -744,7 +741,7 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): self.REMOTE1_SERVER_NAME, self.REMOTE1_SERVER_SIGNATURE_KEY, ) - join_rules_event = make_event_from_dict(join_rules_event_dict, room_version) + join_rules_event = make_test_pdu_event(join_rules_event_dict, room_version) events.append(join_rules_event) return {(event.type, event.state_key): event for event in events} @@ -763,7 +760,6 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): room_version = self.hs.config.server.default_room_version join_event_dict: JsonDict = { "auth_events": [ - state[(EventTypes.Create, "")].event_id, state[(EventTypes.JoinRules, "")].event_id, ], "content": {"membership": "join"}, @@ -781,7 +777,7 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): get_domain_from_id(user), signing_key, ) - return make_event_from_dict(join_event_dict, room_version) + return make_test_pdu_event(join_event_dict, room_version) @parameterized.expand([("not_pruned", False), ("pruned", True)]) @patch( @@ -795,7 +791,7 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): partial state period should be sent to remote servers that were NOT known at the time of the partial join. - We do this by creating a room with one remote server, partialling + We do this by creating a room with one remote server, partially joining it, then receiving a join event from a second remote server. The second remote server should receive a device list update EDU for any local device changes that happened during the partial state period. diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 71f29be2ea..edb419744a 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -383,18 +383,18 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): event = create_invite() self.get_success( self.handler.on_invite_request( - other_server, - event, - event.room_version, + origin=other_server, + event=event, + room_version=event.room_version, ) ) event = create_invite() self.get_failure( self.handler.on_invite_request( - other_server, - event, - event.room_version, + origin=other_server, + event=event, + room_version=event.room_version, ), exc=LimitExceededError, ) diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 3d88085c11..c3f4be40ee 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -124,7 +124,6 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): ) auth_event_ids = [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], member_event.event_id, ] @@ -420,7 +419,6 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): ) auth_event_ids = [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], member_event.event_id, ] @@ -521,7 +519,6 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): ) auth_event_ids = [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], member_event.event_id, ] @@ -718,7 +715,6 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): ) auth_event_ids = [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], member_event.event_id, ] @@ -838,6 +834,12 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): ) room_version = self.get_success(main_store.get_room_version(room_id)) + # This test requires a user that is not the room creator. Infinite power levels + # from MSC4289 change the dynamics of the power level state resolution later to + # not fallback on `origin_server_ts` as described in more detail below. + ernie_user_id = self.register_user("ernie", "test") + ernie_tok = self.login("ernie", "test") + self.helper.join(room_id, user=ernie_user_id, tok=ernie_tok) # Add another local user to the room. This user is going to be kicked in a # rejected event. bert_user_id = self.register_user("bert", "test") @@ -846,13 +848,13 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # Allow the remote user to kick bert. # The remote user is going to send a rejected power levels event later on and we - # need state resolution to order it before another power levels event kermit is + # need state resolution to order it before another power levels event ernie is # going to send later on. Hence we give both users the same power level, so that # ties are broken by `origin_server_ts`. self.helper.send_state( room_id, "m.room.power_levels", - {"users": {kermit_user_id: 100, OTHER_USER: 100}}, + {"users": {ernie_user_id: 100, OTHER_USER: 100}}, tok=kermit_tok, ) @@ -891,7 +893,6 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): "sender": OTHER_USER, "prev_events": [other_member_event.event_id], "auth_events": [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], # The event will be rejected because of the duplicated auth # event. @@ -940,7 +941,6 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): "sender": OTHER_USER, "prev_events": [rejected_power_levels_event.event_id], "auth_events": [ - initial_state_map[("m.room.create", "")], rejected_power_levels_event.event_id, initial_state_map[("m.room.member", bert_user_id)], initial_state_map[("m.room.member", OTHER_USER)], @@ -1018,8 +1018,8 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): self.helper.send_state( room_id, "m.room.power_levels", - {"users": {kermit_user_id: 100, OTHER_USER: 100, bert_user_id: 1}}, - tok=kermit_tok, + {"users": {ernie_user_id: 100, OTHER_USER: 100, bert_user_id: 1}}, + tok=ernie_tok, )["event_id"] ) ) @@ -1054,7 +1054,6 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): "sender": OTHER_USER, "prev_events": [rejected_kick_event.event_id], "auth_events": [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], initial_state_map[("m.room.member", OTHER_USER)], ], @@ -1082,7 +1081,6 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): missing_event.event_id, ], "auth_events": [ - initial_state_map[("m.room.create", "")], new_power_levels_event.event_id, initial_state_map[("m.room.member", OTHER_USER)], ], @@ -1528,3 +1526,137 @@ class FetchMissingStateDagEventsTests(unittest.FederatingHomeserverTestCase): linear["FOREIGN"] = foreign self._prepare_handler(set(), linear, {("C",): ["A", "B", "FOREIGN"]}) self._assert_fetches(linear, "C", ["A", "B"]) + + +class BackfilledStateDagEventTests(unittest.FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + @unittest.override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_backfilled_state_events_update_current_state(self) -> None: + """State events we only learn about through backfill must reach the current state. + + An event's `prev_events` can reach state events which its `prev_state_events` do + not. This should not normally happen but the protocol allows it, so we have to handle it. + The state DAG walk in /get_missing_events only follows `prev_state_events`, + so those state events are never fetched that way, and we first see them when the + `prev_events` graph is backfilled. They are still state events which nothing in + our state DAG references, so they belong on the state DAG frontier and therefore + change the current state. + + prev_state_events graph We know MEMBER and TOPIC + Sx = state event + MEMBER <- TOPIC Backfill hands us S4 and S5, forking at MEMBER + `-- S4 <- S5 + + The frontier becomes {TOPIC, S5}, so S5 has to be merged into the current state. + """ + OTHER_USER = f"@user:{self.OTHER_SERVER_NAME}" + main_store = self.hs.get_datastores().main + federation_event_handler = self.hs.get_federation_event_handler() + + user_id = self.register_user("kermit", "test") + tok = self.login("kermit", "test") + room_id = self.helper.create_room_as( + room_creator=user_id, + tok=tok, + room_version=RoomVersions.MSC4242v12.identifier, + ) + room_version = self.get_success(main_store.get_room_version(room_id)) + self.assertTrue(room_version.msc4242_state_dags) + + # allow the remote user to send state events + self.helper.send_state( + room_id, + "m.room.power_levels", + {"events_default": 0, "state_default": 0}, + tok=tok, + ) + + # add the remote user to the room. The backfilled branch forks here. + member_event = self.get_success( + event_injection.inject_member_event(self.hs, room_id, OTHER_USER, "join") + ) + + # Move the state DAG locally, so that the backfill below is a genuine fork + # rather than an extension of the only branch we have. + topic_event_id = self.helper.send_state( + room_id, "m.room.topic", {"topic": "local"}, tok=tok + )["event_id"] + + extremities_before = self.get_success( + main_store.get_state_dag_extremities(room_id) + ) + self.assertIncludes({topic_event_id}, extremities_before, exact=True) + + # S4 and S5 are a branch of the state DAG which we have never been told about: + # nothing we hold references them via `prev_state_events`, so the state DAG walk + # would never have fetched them. + s4 = make_test_event( + self.add_hashes_and_signatures_from_other_server( + { + "type": "test_state_type", + "state_key": "", + "room_id": room_id, + "sender": OTHER_USER, + "prev_events": [member_event.event_id], + "prev_state_events": [member_event.event_id], + "origin_server_ts": 1, + "depth": 10, + "content": {"body": "s4"}, + }, + room_version, + ), + room_version, + ) + s5 = make_test_event( + self.add_hashes_and_signatures_from_other_server( + { + "type": "test_state_type", + "state_key": "", + "room_id": room_id, + "sender": OTHER_USER, + "prev_events": [s4.event_id], + "prev_state_events": [s4.event_id], + "origin_server_ts": 2, + "depth": 11, + "content": {"body": "s5"}, + }, + room_version, + ), + room_version, + ) + + self.get_success( + federation_event_handler._process_pulled_events( + self.OTHER_SERVER_NAME, [s4, s5], backfilled=True + ) + ) + + # The tip of the backfilled branch is now a state DAG forward extremity, and the + # branch we already had still is too: the fork is unmerged in the state DAG. + extremities_after = self.get_success( + main_store.get_state_dag_extremities(room_id) + ) + self.assertIncludes( + {topic_event_id, s5.event_id}, + extremities_after, + exact=True, + message="the backfilled state event did not become a state DAG forward extremity", + ) + + # ...and resolving that frontier is what the current state must reflect. + state_map = self.get_success(main_store.get_partial_current_state_ids(room_id)) + self.assertEqual( + state_map.get(("test_state_type", "")), + s5.event_id, + "the state event learned through backfill is not part of the current state", + ) + self.assertEqual( + state_map.get(("m.room.topic", "")), + topic_event_id, + "merging in the backfilled branch lost the state on the branch we had", + ) diff --git a/tests/handlers/test_message.py b/tests/handlers/test_message.py index 6450a90444..1160a66938 100644 --- a/tests/handlers/test_message.py +++ b/tests/handlers/test_message.py @@ -234,11 +234,6 @@ class EventCreationTestCase(unittest.HomeserverTestCase): ) def test_call_invite_event_creation_fails_in_public_room(self) -> None: - # get prev_events for room - prev_events = self.get_success( - self.store.get_prev_events_for_room(self.room_id) - ) - # the invite in a public room should fail self.get_failure( self.handler.create_event( @@ -248,8 +243,6 @@ class EventCreationTestCase(unittest.HomeserverTestCase): "room_id": self.room_id, "sender": self.requester.user.to_string(), }, - prev_event_ids=prev_events, - auth_event_ids=prev_events, ), SynapseError, ) @@ -263,8 +256,6 @@ class EventCreationTestCase(unittest.HomeserverTestCase): "room_id": self.private_room_id, "sender": self.requester.user.to_string(), }, - prev_event_ids=prev_events, - auth_event_ids=prev_events, ) ) diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 71f83607ff..d37a58949e 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -216,8 +216,6 @@ class MasAuthDelegation(HomeserverTestCase): def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: self.server = FakeMasServer() hs = self.setup_test_homeserver() - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() self._auth = checked_cast(MasDelegatedAuth, hs.get_auth()) return hs @@ -664,3 +662,43 @@ class DisabledEndpointsTestCase(HomeserverTestCase): self.expect_unrecognized("GET", "/_synapse/admin/v1/users/foo/admin") self.expect_unrecognized("PUT", "/_synapse/admin/v1/users/foo/admin") self.expect_unrecognized("POST", "/_synapse/admin/v1/account_validity/validity") + + def _mock_admin_requester(self) -> None: + self.hs.get_auth().get_user_by_req = AsyncMock( # type: ignore[method-assign] + return_value=create_requester(user_id=USER_ID, device_id=DEVICE) + ) + self.hs.get_auth().is_server_admin = AsyncMock(return_value=True) # type: ignore[method-assign] + + def test_admin_api_create_user_rejected(self) -> None: + """MAS never learns about a user created through the admin API, so the + account would be unusable.""" + self._mock_admin_requester() + + channel = self.make_request( + "PUT", + "/_synapse/admin/v2/users/@newuser:test", + {"password": "hunter2"}, + access_token="token", + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["errcode"], Codes.FORBIDDEN, channel.json_body + ) + + def test_admin_api_modify_existing_user_allowed(self) -> None: + """Only creation is rejected: modifying a user MAS already knows about + has to keep working, since MAS has no equivalent endpoint.""" + self._mock_admin_requester() + self.get_success( + self.hs.get_datastores().main.register_user(f"@existing:{SERVER_NAME}") + ) + + channel = self.make_request( + "PUT", + "/_synapse/admin/v2/users/@existing:test", + {"displayname": "Existing User"}, + access_token="token", + ) + + self.assertEqual(channel.code, 200, channel.json_body) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index a562050842..61cf78730f 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -2382,7 +2382,6 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): "origin_server_ts": 1, "content": {"membership": Membership.JOIN}, "auth_events": [ - state_map[(EventTypes.Create, "")].event_id, state_map[(EventTypes.JoinRules, "")].event_id, ], "prev_events": list(forward_extremity_event_ids), diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 7af0345c87..8ea932c46e 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -659,7 +659,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): affected_fields=None, ), ProfileUpdate( - stream_id=7, + stream_id=9, user_id="@gracie:test", action="left_room", affected_fields=None, @@ -688,6 +688,45 @@ class ProfileTestCase(unittest.HomeserverTestCase): ], ) + @override_config({"include_profile_updates_in_sync": True}) + def test_left_room_event_if_we_leave_the_last_shared_room( + self, + ) -> None: + """Test that when we leave a room, we get profile update rows with a "left room" + action for users we no longer share a room with. + """ + self.register_user("roger", "password") + roger_token = self.login("roger", "password") + room_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.helper.join(room_id, "@roger:test", tok=roger_token) + + # Make us leave the room + self.helper.leave(room_id, self.frank.to_string(), tok=self.frank_token) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id=self.frank.to_string(), + field_names={"m.status"}, + ) + ) + # We're no longer in any rooms with roger, and the profile update stream + # should have an update regarding that. + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id="@roger:test", + action="left_room", + affected_fields=None, + ), + ], + ) + @parameterized.expand( [ ["displayname", "Frank"], diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 182ff7a8fc..46cd2ed83b 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -477,6 +477,7 @@ class RegistrationTestCase(unittest.HomeserverTestCase): "auto_join_rooms": ["#room:test"], "autocreate_auto_join_room_preset": "private_chat", "auto_join_mxid_localpart": "support", + "default_room_version": "11", } ) def test_auto_create_auto_join_room_preset_invalid_permissions(self) -> None: @@ -485,6 +486,8 @@ class RegistrationTestCase(unittest.HomeserverTestCase): registration doesn't completely break if the inviter doesn't have proper permissions. """ + # This test is limited to room version 11, as after that the room creator will + # always have permission to invite users and make any other changes they wish. inviter = "@support:test" # Register an initial user to create the room and such (essentially this @@ -528,7 +531,7 @@ class RegistrationTestCase(unittest.HomeserverTestCase): ) ) - # Register a second user, which won't be be in the room (or even have an invite) + # Register a second user, which won't be in the room (or even have an invite) # since the inviter no longer has the proper permissions. user_id = self.get_success(self.handler.register_user(localpart="bob")) diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index a4a57b64d9..1b62830571 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -8,10 +8,9 @@ import synapse.rest.client.room from synapse.api.constants import AccountDataTypes, EventTypes, Membership from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer -from synapse.types import UserID, create_requester +from synapse.types import JsonDict, UserID, create_requester from synapse.util.clock import Clock from tests.replication._base import BaseMultiWorkerStreamTestCase @@ -43,10 +42,10 @@ class TestJoinsLimitedByPerRoomRateLimiter(FederatingHomeserverTestCase): self.chris_token = self.login("chris", "pass") # Create a room on this homeserver. Note that this counts as a join: it - # contributes to the rate limter's count of actions + # contributes to the rate limiter's count of actions self.room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) - self.intially_unjoined_room_id = f"!example:{self.OTHER_SERVER_NAME}" + self.room_version = self.hs.config.server.default_room_version @override_config({"rc_joins_per_room": {"per_second": 0.1, "burst_count": 2}}) def test_local_user_local_joins_contribute_to_limit_and_are_limited(self) -> None: @@ -105,60 +104,55 @@ class TestJoinsLimitedByPerRoomRateLimiter(FederatingHomeserverTestCase): # We also patch out a bunch of event checks on our end. All we're really # trying to check here is that remote joins will bump the rate limter when # they are persisted. - create_event_source = { + create_event_source: JsonDict = { "auth_events": [], "content": { - "creator": f"@creator:{self.OTHER_SERVER_NAME}", - "room_version": self.hs.config.server.default_room_version.identifier, + "room_version": self.room_version.identifier, }, - "depth": 0, + "depth": 1, "origin_server_ts": 0, "prev_events": [], - "room_id": self.intially_unjoined_room_id, "sender": f"@creator:{self.OTHER_SERVER_NAME}", "state_key": "", "type": EventTypes.Create, } - self.add_hashes_and_signatures_from_other_server( + + self.add_hashes_and_signatures_from_other_server(create_event_source) + create_event = make_test_pdu_event( create_event_source, - self.hs.config.server.default_room_version, - ) - create_event = make_event_from_dict( - create_event_source, - self.hs.config.server.default_room_version, - {}, - None, + self.room_version, ) + # Extract the room_id for use below + initially_unjoined_room_id = create_event.room_id + join_event_source = { - "auth_events": [create_event.event_id], + "auth_events": [], "content": {"membership": "join"}, - "depth": 1, + "depth": 2, "origin_server_ts": 100, "prev_events": [create_event.event_id], "sender": self.bob, "state_key": self.bob, - "room_id": self.intially_unjoined_room_id, + "room_id": initially_unjoined_room_id, "type": EventTypes.Member, } add_hashes_and_signatures( - self.hs.config.server.default_room_version, + self.room_version, join_event_source, self.hs.hostname, self.hs.signing_key, ) - join_event = make_event_from_dict( + join_event = make_test_pdu_event( join_event_source, - self.hs.config.server.default_room_version, - {}, - None, + self.room_version, ) mock_make_membership_event = AsyncMock( return_value=( self.OTHER_SERVER_NAME, join_event, - self.hs.config.server.default_room_version, + self.room_version, ) ) mock_send_join = AsyncMock( @@ -197,7 +191,7 @@ class TestJoinsLimitedByPerRoomRateLimiter(FederatingHomeserverTestCase): self.handler.update_membership( requester=create_requester(self.bob), target=UserID.from_string(self.bob), - room_id=self.intially_unjoined_room_id, + room_id=initially_unjoined_room_id, action=Membership.JOIN, remote_room_hosts=[self.OTHER_SERVER_NAME], ) @@ -208,7 +202,7 @@ class TestJoinsLimitedByPerRoomRateLimiter(FederatingHomeserverTestCase): self.handler.update_membership( requester=create_requester(self.chris), target=UserID.from_string(self.chris), - room_id=self.intially_unjoined_room_id, + room_id=initially_unjoined_room_id, action=Membership.JOIN, remote_room_hosts=[self.OTHER_SERVER_NAME], ), @@ -561,9 +555,9 @@ class TestMSC4155InviteFiltering(FederatingHomeserverTestCase): f = self.get_failure( self.fed_handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ), SynapseError, ).value @@ -607,9 +601,9 @@ class TestMSC4155InviteFiltering(FederatingHomeserverTestCase): f = self.get_failure( self.fed_handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ), SynapseError, ).value @@ -722,9 +716,9 @@ class TestMSC4380InviteBlocking(FederatingHomeserverTestCase): f = self.get_failure( self.fed_handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ), SynapseError, ).value diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index 0f8de6e7b9..a3a0e62dcf 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -31,6 +31,7 @@ from synapse.api.constants import ( JoinRules, Membership, RestrictedJoinRuleTypes, + RoomEncryptionAlgorithms, RoomTypes, ) from synapse.api.errors import AuthError, NotFoundError, SynapseError @@ -184,9 +185,6 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): result_room_ids = [] result_children_ids = [] for result_room in result["rooms"]: - # Ensure federation results are not leaking over the client-server API. - self.assertNotIn("allowed_room_ids", result_room) - result_room_ids.append(result_room["room_id"]) result_children_ids.append( [ @@ -232,7 +230,9 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): } ) self.get_success( - fed_handler.on_invite_request(fed_hostname, event, RoomVersions.V6) + fed_handler.on_invite_request( + origin=fed_hostname, event=event, room_version=RoomVersions.V6 + ) ) def test_simple_space(self) -> None: @@ -489,6 +489,80 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): ) self._assert_hierarchy(result, expected) + def test_summary_fields_restricted(self) -> None: + """ + The room entries returned to clients include the `room_version`, + `encryption` and `allowed_room_ids` fields, added to the client + `/hierarchy` API in Matrix 1.15. + + See https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1roomsroomidhierarchy + """ + restricted_room = self._create_room_with_join_rule( + JoinRules.RESTRICTED, + room_version=RoomVersions.V8.identifier, + allow=[ + { + "type": RestrictedJoinRuleTypes.ROOM_MEMBERSHIP, + "room_id": self.space, + "via": [self.hs.hostname], + } + ], + ) + self.helper.send_state( + restricted_room, + event_type=EventTypes.RoomEncryption, + body={"algorithm": RoomEncryptionAlgorithms.DEFAULT}, + tok=self.token, + ) + + result = self.get_success( + self.handler.get_room_hierarchy(create_requester(self.user), self.space) + ) + room_entries = {entry["room_id"]: entry for entry in result["rooms"]} + entry = room_entries[restricted_room] + self.assertEqual(entry["join_rule"], JoinRules.RESTRICTED) + self.assertEqual(entry["room_version"], RoomVersions.V8.identifier) + self.assertEqual(entry["encryption"], RoomEncryptionAlgorithms.DEFAULT) + self.assertEqual(entry["allowed_room_ids"], [self.space]) + + # Rooms without restricted join rules should not have the field at all. + self.assertNotIn("allowed_room_ids", room_entries[self.room]) + + def test_summary_fields_knock_restricted(self) -> None: + """ + Same as `test_summary_fields_restricted`, for a `knock_restricted` room. + + Joining such a room without an invite applies the restricted join rule, so + it also returns `allowed_room_ids`. + """ + knock_restricted_room = self._create_room_with_join_rule( + JoinRules.KNOCK_RESTRICTED, + room_version=RoomVersions.V10.identifier, + allow=[ + { + "type": RestrictedJoinRuleTypes.ROOM_MEMBERSHIP, + "room_id": self.space, + "via": [self.hs.hostname], + } + ], + ) + self.helper.send_state( + knock_restricted_room, + event_type=EventTypes.RoomEncryption, + body={"algorithm": RoomEncryptionAlgorithms.DEFAULT}, + tok=self.token, + ) + + result = self.get_success( + self.handler.get_room_hierarchy(create_requester(self.user), self.space) + ) + room_entries = {entry["room_id"]: entry for entry in result["rooms"]} + entry = room_entries[knock_restricted_room] + self.assertEqual(entry["join_rule"], JoinRules.KNOCK_RESTRICTED) + self.assertEqual(entry["room_version"], RoomVersions.V10.identifier) + self.assertEqual(entry["encryption"], RoomEncryptionAlgorithms.DEFAULT) + self.assertEqual(entry["allowed_room_ids"], [self.space]) + def test_complex_space(self) -> None: """ Create a "complex" space to see how it handles things like loops and subspaces. @@ -960,6 +1034,13 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): ) self._assert_hierarchy(result, expected) + # `allowed_room_ids` returned over federation should be passed through + # to the client. + room_entries = {entry["room_id"]: entry for entry in result["rooms"]} + self.assertEqual( + room_entries[restricted_accessible_room]["allowed_room_ids"], [self.room] + ) + def test_fed_invited(self) -> None: """ A room which the user was invited to should be included in the response. diff --git a/tests/handlers/test_send_email.py b/tests/handlers/test_send_email.py index acb88343f2..b7935ebfd1 100644 --- a/tests/handlers/test_send_email.py +++ b/tests/handlers/test_send_email.py @@ -29,6 +29,7 @@ from twisted.internet import defer from twisted.internet._sslverify import ClientTLSOptions from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import ensureDeferred +from twisted.internet.endpoints import _WrappingFactory from twisted.internet.interfaces import IProtocolFactory from twisted.internet.ssl import ContextFactory from twisted.mail import interfaces, smtp @@ -134,6 +135,7 @@ class SendEmailHandlerTestCaseIPv4(HomeserverTestCase): server_protocol.callLater = self.reactor.callLater # type: ignore[assignment] client_protocol = client_factory.buildProtocol(None) + assert client_protocol is not None client_protocol.makeConnection(FakeTransport(server_protocol, self.reactor)) server_protocol.makeConnection( FakeTransport( @@ -187,8 +189,11 @@ class SendEmailHandlerTestCaseIPv4(HomeserverTestCase): self.assertEqual(host, self.reactor.lookups["localhost"]) self.assertEqual(port, 465) # We need to make sure that TLS is happenning - context_factory = client_factory._wrappedFactory._testingContextFactory - self.assertIsInstance(context_factory, ClientTLSOptions) + assert isinstance(client_factory, _WrappingFactory) + context_factory = getattr( + client_factory._wrappedFactory, "_testingContextFactory", None + ) + assert isinstance(context_factory, ClientTLSOptions) self.assertEqual(context_factory._hostname, "example.org") # tlsname # And since we use endpoints, they go through reactor.connectTCP # which works differently to connectSSL on the testing reactor @@ -201,6 +206,7 @@ class SendEmailHandlerTestCaseIPv4(HomeserverTestCase): server_protocol.callLater = self.reactor.callLater # type: ignore[assignment] client_protocol = client_factory.buildProtocol(None) + assert client_protocol is not None client_protocol.makeConnection(FakeTransport(server_protocol, self.reactor)) server_protocol.makeConnection( FakeTransport( diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 2293988036..f91468db1e 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -19,7 +19,6 @@ # import logging from typing import AbstractSet -from unittest.mock import patch import attr from parameterized import parameterized, parameterized_class @@ -3018,21 +3017,6 @@ class ComputeInterestedRoomsShardTestCase( self.store = self.hs.get_datastores().main self.event_sources = hs.get_event_sources() - def _create_room(self, room_id: str, user_id: str, tok: str) -> None: - """ - Create a room with a specific room_id. We use this so that that we have a - consistent room_id across test runs that hashes to the same value and will be - sharded to a known worker in the tests. - """ - - # We control the room ID generation by patching out the - # `_generate_room_id` method - with patch( - "synapse.handlers.room.RoomCreationHandler._generate_room_id" - ) as mock: - mock.side_effect = lambda: room_id - self.helper.create_room_as(user_id, tok=tok) - def test_sharded_event_persisters(self) -> None: """ This test should catch bugs that would come from flawed stream position @@ -3070,19 +3054,15 @@ class ComputeInterestedRoomsShardTestCase( {"worker_name": "worker3"}, ) - # Specially crafted room IDs that get persisted on different workers. - # - # Sharded to worker1 - room_id1 = "!fooo:test" - # Sharded to worker2 - room_id2 = "!bar:test" - # Sharded to worker3 - room_id3 = "!quux:test" + # 3 different rooms, each on a different worker. Match them up based on the + # trailing number. + room_results = self._generate_rooms_on_worker( + user2_id, user2_tok, ["worker1", "worker2", "worker3"] + ) + room_id1 = room_results["worker1"] + room_id2 = room_results["worker2"] + room_id3 = room_results["worker3"] - # Create rooms on the different workers. - self._create_room(room_id1, user2_id, user2_tok) - self._create_room(room_id2, user2_id, user2_tok) - self._create_room(room_id3, user2_id, user2_tok) join_response1 = self.helper.join(room_id1, user1_id, tok=user1_tok) join_response2 = self.helper.join(room_id2, user1_id, tok=user1_tok) # Leave room2 diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index a74bd68df1..1002a0374c 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -33,7 +33,9 @@ from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.handlers.sync import ( + LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE, LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE, + LazyLoadedMembersCache, SyncConfig, SyncRequestKey, SyncResult, @@ -48,6 +50,7 @@ from synapse.types import ( MultiWriterStreamToken, RoomStreamToken, StreamKeyType, + StreamToken, UserID, create_requester, ) @@ -2074,7 +2077,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): self, ) -> None: """Test that with `include_profile_updates_in_sync` enabled the incremental - sync response includes a 'null' for users who are no longer sharing rooms. + sync response includes a 'null' for users who are no longer sharing rooms, due + to the other user leaving the last room. """ requester = create_requester(self.user) initial_result = self.get_success( @@ -2119,6 +2123,59 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): incremental_result.profile_updates["@other_user:test"], ) + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_null_profile_we_no_longer_sharing_rooms( + self, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes a 'null' for users who are no longer sharing rooms, due + us leaving the last shared room. + """ + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.helper.leave( + room=self.joined_room, + user=self.user, + tok=self.tok, + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertIsNone( + incremental_result.profile_updates["@other_user:test"], + ) + @parameterized.expand( [ True, @@ -2893,7 +2950,7 @@ class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): # Calculating the incrementals state will return the second state, and not the # first. - state = self.get_success( + state, _ = self.get_success( self.sync_handler._compute_state_delta_for_incremental_sync( room_id=joined_room, sync_config=generate_sync_config(user, use_state_after=True), @@ -2925,7 +2982,7 @@ class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): since_token = self.hs.get_event_sources().get_current_token() end_stream_token = self.hs.get_event_sources().get_current_token() - state = self.get_success( + state, _ = self.get_success( self.sync_handler._compute_state_delta_for_incremental_sync( room_id=joined_room, sync_config=generate_sync_config(user, use_state_after=True), @@ -2941,3 +2998,76 @@ class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): ) self.assertEqual(state, {}) + + +class LazyLoadedMembersCacheTestCase(tests.unittest.TestCase): + """Direct tests of the per-device cache of lazy-loaded members sent.""" + + @staticmethod + def _token(stream: int) -> StreamToken: + """A token whose room position is `stream`. Only the room position + matters to the cache.""" + return StreamToken.START.copy_and_replace( + StreamKeyType.ROOM, RoomStreamToken(stream=stream) + ) + + def test_forgets_only_entries_after_since(self) -> None: + """A request from `since` forgets the entries recorded after it and + keeps the rest.""" + cache = LazyLoadedMembersCache() + cache.mark_sent("@a:test", "$a", self._token(5)) + cache.mark_sent("@b:test", "$b", self._token(7)) + + # The client synced past both responses. + cache.start_request(self._token(7)) + self.assertTrue(cache.was_sent("@a:test", "$a")) + self.assertTrue(cache.was_sent("@b:test", "$b")) + + # The client lost the response that ended at 7. + cache.start_request(self._token(6)) + self.assertTrue(cache.was_sent("@a:test", "$a")) + self.assertFalse(cache.was_sent("@b:test", "$b")) + + # A later request from an even older token forgets the rest. + cache.start_request(self._token(4)) + self.assertFalse(cache.was_sent("@a:test", "$a")) + + def test_initial_sync_forgets_everything(self) -> None: + """An initial sync has no `since`, and the client starts from nothing.""" + cache = LazyLoadedMembersCache() + cache.mark_sent("@a:test", "$a", self._token(5)) + cache.start_request(None) + self.assertFalse(cache.was_sent("@a:test", "$a")) + + def test_different_event_is_not_sent(self) -> None: + """An entry only covers the membership event it was recorded with, so + a changed membership counts as not sent.""" + cache = LazyLoadedMembersCache() + cache.mark_sent("@a:test", "$a1", self._token(5)) + self.assertFalse(cache.was_sent("@a:test", "$a2")) + + def test_out_of_order_marks_are_still_forgotten(self) -> None: + """A response that started earlier can record its entries after a + later one. Its entries must still be forgotten by a request from + before either response.""" + cache = LazyLoadedMembersCache() + cache.mark_sent("@b:test", "$b", self._token(7)) + cache.mark_sent("@a:test", "$a", self._token(5)) + + cache.start_request(self._token(6)) + self.assertTrue(cache.was_sent("@a:test", "$a")) + self.assertFalse(cache.was_sent("@b:test", "$b")) + + def test_evicts_least_recently_used(self) -> None: + """The cache holds `LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE` entries and + evicts the least recently used one. A hit counts as a use.""" + cache = LazyLoadedMembersCache() + for i in range(LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE): + cache.mark_sent(f"@u{i}:test", f"$u{i}", self._token(1)) + # Touch the oldest entry so that it is no longer the eviction candidate. + self.assertTrue(cache.was_sent("@u0:test", "$u0")) + + cache.mark_sent("@new:test", "$new", self._token(1)) + self.assertTrue(cache.was_sent("@u0:test", "$u0")) + self.assertFalse(cache.was_sent("@u1:test", "$u1")) + self.assertTrue(cache.was_sent("@new:test", "$new")) diff --git a/tests/http/__init__.py b/tests/http/__init__.py index b19a484004..f179fb6874 100644 --- a/tests/http/__init__.py +++ b/tests/http/__init__.py @@ -45,7 +45,7 @@ def get_test_https_policy() -> BrowserLikePolicyForHTTPS: IPolicyForHTTPS """ ca_file = get_test_ca_cert_file() - with open(ca_file) as stream: + with open(ca_file, "rb") as stream: content = stream.read() cert = Certificate.loadPEM(content) trust_root = trustRootFromCertificates([cert]) diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 0b2eabaf98..f634d49b02 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -26,23 +26,19 @@ from unittest.mock import AsyncMock, call, patch import treq from netaddr import IPSet from service_identity import VerificationError -from zope.interface import implementer from twisted.internet import defer -from twisted.internet._sslverify import ClientTLSOptions, OpenSSLCertificateOptions from twisted.internet.defer import Deferred from twisted.internet.endpoints import _WrappingProtocol -from twisted.internet.interfaces import ( - IOpenSSLClientConnectionCreator, - IProtocolFactory, -) +from twisted.internet.interfaces import IProtocolFactory from twisted.internet.protocol import Factory, Protocol from twisted.protocols.tls import TLSMemoryBIOProtocol +from twisted.python.failure import Failure from twisted.web._newclient import ResponseNeverReceived from twisted.web.client import Agent from twisted.web.http import HTTPChannel, Request from twisted.web.http_headers import Headers -from twisted.web.iweb import IPolicyForHTTPS, IResponse +from twisted.web.iweb import IResponse from synapse.config.homeserver import HomeServerConfig from synapse.config.server import parse_proxy_config @@ -679,7 +675,9 @@ class MatrixFederationAgentTests(unittest.TestCase): self.assertEqual(port, 443) # fonx the connection - client_factory.clientConnectionFailed(None, Exception("nope")) + client_factory.clientConnectionFailed( + self.reactor.connectors[0], Failure(Exception("nope")) + ) # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the # .well-known request fails. @@ -765,7 +763,9 @@ class MatrixFederationAgentTests(unittest.TestCase): self.assertEqual(port, 443) # fonx the connection - client_factory.clientConnectionFailed(None, Exception("nope")) + client_factory.clientConnectionFailed( + self.reactor.connectors[-1], Failure(Exception("nope")) + ) # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the # .well-known request fails. @@ -1305,7 +1305,9 @@ class MatrixFederationAgentTests(unittest.TestCase): self.assertEqual(port, 443) # fonx the connection - client_factory.clientConnectionFailed(None, Exception("nope")) + client_factory.clientConnectionFailed( + self.reactor.connectors[-1], Failure(Exception("nope")) + ) # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the # .well-known request fails. @@ -1544,7 +1546,9 @@ class MatrixFederationAgentTests(unittest.TestCase): # fonx the connection attempt, this will be treated as a temporary # failure. - client_factory.clientConnectionFailed(None, Exception("nope")) + client_factory.clientConnectionFailed( + self.reactor.connectors[-1], Failure(Exception("nope")) + ) # There's a few sleeps involved, so we have to pump the reactor a # bit. @@ -1568,7 +1572,9 @@ class MatrixFederationAgentTests(unittest.TestCase): clients = self.reactor.tcpClients (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0) - client_factory.clientConnectionFailed(None, Exception("nope")) + client_factory.clientConnectionFailed( + self.reactor.connectors[-1], Failure(Exception("nope")) + ) self.reactor.pump((0.4,)) r = self.successResultOf(fetch_d) @@ -1627,7 +1633,9 @@ class MatrixFederationAgentTests(unittest.TestCase): self.assertEqual(port, 8443) # Fonx the connection - client_factory.clientConnectionFailed(None, Exception("nope")) + client_factory.clientConnectionFailed( + self.reactor.connectors[-1], Failure(Exception("nope")) + ) # There's a 300ms delay in HostnameEndpoint self.reactor.pump((0.4,)) @@ -1687,7 +1695,9 @@ class MatrixFederationAgentTests(unittest.TestCase): self.assertEqual(port, 8443) # Fonx the connection - client_factory.clientConnectionFailed(None, Exception("nope")) + client_factory.clientConnectionFailed( + self.reactor.connectors[-1], Failure(Exception("nope")) + ) # There's a 300ms delay in HostnameEndpoint self.reactor.pump((0.4,)) @@ -1745,7 +1755,9 @@ class MatrixFederationAgentTests(unittest.TestCase): self.assertEqual(port, 8443) # Fonx the connection - client_factory.clientConnectionFailed(None, Exception("nope")) + client_factory.clientConnectionFailed( + self.reactor.connectors[-1], Failure(Exception("nope")) + ) # There's a 300ms delay in HostnameEndpoint self.reactor.pump((0.4,)) @@ -1830,26 +1842,12 @@ def _get_test_protocol_factory() -> IProtocolFactory: Returns: interfaces.IProtocolFactory """ - server_factory = Factory.forProtocol(HTTPChannel) - - # Request.finish expects the factory to have a 'log' method. - server_factory.log = _log_request - - return server_factory + return _HTTPFactory() -def _log_request(request: str) -> None: - """Implements Factory.log, which is expected by Request.finish""" - logger.info("Completed request %s", request) +class _HTTPFactory(Factory): + protocol = HTTPChannel - -@implementer(IPolicyForHTTPS) -class TrustingTLSPolicyForHTTPS: - """An IPolicyForHTTPS which checks that the certificate belongs to the - right server, but doesn't check the certificate chain.""" - - def creatorForNetloc( - self, hostname: bytes, port: int - ) -> IOpenSSLClientConnectionCreator: - certificateOptions = OpenSSLCertificateOptions() - return ClientTLSOptions(hostname, certificateOptions.getContext()) + def log(self, request: Request) -> None: + """Request.finish expects the factory to have a 'log' method.""" + logger.info("Completed request %s", request) diff --git a/tests/http/test_client.py b/tests/http/test_client.py index 5c8c1220e4..394029b965 100644 --- a/tests/http/test_client.py +++ b/tests/http/test_client.py @@ -24,9 +24,13 @@ from unittest.mock import Mock from netaddr import IPSet +from OpenSSL import SSL from twisted.internet.defer import Deferred from twisted.internet.error import DNSLookupError +from twisted.internet.interfaces import IOpenSSLClientConnectionCreator +from twisted.internet.protocol import Factory, Protocol from twisted.internet.testing import AccumulatingProtocol +from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.failure import Failure from twisted.web.client import Agent, ResponseDone from twisted.web.iweb import UNKNOWN_LENGTH @@ -36,6 +40,7 @@ from synapse.http.client import ( BlocklistingAgentWrapper, BlocklistingReactorWrapper, BodyExceededMaxSize, + InsecureInterceptableContextFactory, MultipartResponse, _DiscardBodyWithMaxSizeProtocol, _MultipartParserProtocol, @@ -47,6 +52,21 @@ from tests.server import FakeTransport, get_clock from tests.unittest import TestCase +class InsecureInterceptableContextFactoryTests(TestCase): + def test_client_connection_creator(self) -> None: + """The insecure test policy provides connections without verification.""" + policy = InsecureInterceptableContextFactory() + creator = policy.creatorForNetloc(b"example.com", 443) + self.assertTrue(IOpenSSLClientConnectionCreator.providedBy(creator)) + + factory = TLSMemoryBIOFactory(creator, True, Factory.forProtocol(Protocol)) + protocol = factory.buildProtocol(None) + assert protocol is not None + connection = creator.clientConnectionForTLS(protocol) + self.assertIs(connection.get_context(), policy.getContext()) + self.assertEqual(connection.get_context().get_verify_mode(), SSL.VERIFY_NONE) + + class ReadMultipartResponseTests(TestCase): multipart_response_data1 = b"\r\n\r\n--6067d4698f8d40a0a794ea7d7379d53a\r\nContent-Type: application/json\r\n\r\n{}\r\n--6067d4698f8d40a0a794ea7d7379d53a\r\nContent-Type: text/plain\r\nContent-Disposition: inline; filename=test_upload\r\n\r\nfile_" multipart_response_data2 = ( @@ -367,6 +387,7 @@ class BlocklistingAgentTest(TestCase): # Make the connection and pump data through it. client = client_factory.buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -411,6 +432,7 @@ class BlocklistingAgentTest(TestCase): # Make the connection and pump data through it. client = client_factory.buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) diff --git a/tests/http/test_matrixfederationclient.py b/tests/http/test_matrixfederationclient.py index 4792bdc9bc..d4494da8d7 100644 --- a/tests/http/test_matrixfederationclient.py +++ b/tests/http/test_matrixfederationclient.py @@ -28,6 +28,7 @@ from twisted.internet import defer from twisted.internet.defer import Deferred, TimeoutError from twisted.internet.error import ConnectingCancelledError, DNSLookupError from twisted.internet.testing import MemoryReactor, StringTransport +from twisted.python.failure import Failure from twisted.web.client import Agent, ResponseNeverReceived from twisted.web.http import HTTPChannel from twisted.web.http_headers import Headers @@ -112,6 +113,7 @@ class FederationClientTests(HomeserverTestCase): # complete the connection and wire it up to a fake transport protocol = factory.buildProtocol(None) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -169,7 +171,7 @@ class FederationClientTests(HomeserverTestCase): self.assertEqual(host, "1.2.3.4") self.assertEqual(port, 8008) e = Exception("go away") - factory.clientConnectionFailed(None, e) + factory.clientConnectionFailed(self.reactor.connectors[0], Failure(e)) self.pump(0.5) f = self.failureResultOf(d) @@ -231,6 +233,7 @@ class FederationClientTests(HomeserverTestCase): conn = Mock() client = clients[0][2].buildProtocol(None) + assert client is not None client.makeConnection(conn) # Deferred is still without a result @@ -327,6 +330,7 @@ class FederationClientTests(HomeserverTestCase): conn = Mock() clients = self.reactor.tcpClients client = clients[0][2].buildProtocol(None) + assert client is not None client.makeConnection(conn) # Deferred does not have a result @@ -369,6 +373,7 @@ class FederationClientTests(HomeserverTestCase): # complete the connection and wire it up to a fake transport protocol = factory.buildProtocol(None) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -394,6 +399,7 @@ class FederationClientTests(HomeserverTestCase): self.assertEqual(host, "1.2.3.4") self.assertEqual(port, 8008) protocol = factory.buildProtocol(None) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -450,6 +456,7 @@ class FederationClientTests(HomeserverTestCase): conn = Mock() clients = self.reactor.tcpClients client = clients[0][2].buildProtocol(None) + assert client is not None client.makeConnection(conn) # Deferred does not have a result @@ -487,6 +494,7 @@ class FederationClientTests(HomeserverTestCase): conn = Mock() clients = self.reactor.tcpClients client = clients[0][2].buildProtocol(None) + assert client is not None client.makeConnection(conn) # Deferred does not have a result @@ -526,6 +534,7 @@ class FederationClientTests(HomeserverTestCase): # complete the connection and wire it up to a fake transport client = factory.buildProtocol(None) + assert client is not None conn = StringTransport() client.makeConnection(conn) @@ -581,6 +590,7 @@ class FederationClientTests(HomeserverTestCase): # complete the connection and wire it up to a fake transport client = factory.buildProtocol(None) + assert client is not None conn = StringTransport() client.makeConnection(conn) @@ -617,6 +627,7 @@ class FederationClientTests(HomeserverTestCase): clients = self.reactor.tcpClients self.assertEqual(len(clients), 1) client = clients[0][2].buildProtocol(None) + assert client is not None server = HTTPChannel() client.makeConnection(FakeTransport(server, self.reactor)) @@ -642,6 +653,7 @@ class FederationClientTests(HomeserverTestCase): # complete the connection and wire it up to a fake transport client = factory.buildProtocol(None) + assert client is not None conn = StringTransport() client.makeConnection(conn) @@ -690,6 +702,7 @@ class FederationClientTests(HomeserverTestCase): # complete the connection and wire it up to a fake transport protocol = factory.buildProtocol(None) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -736,6 +749,7 @@ class FederationClientTests(HomeserverTestCase): # complete the connection and wire it up to a fake transport protocol = factory.buildProtocol(None) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py index c65115b3e5..e72ac47e7b 100644 --- a/tests/http/test_proxyagent.py +++ b/tests/http/test_proxyagent.py @@ -36,7 +36,7 @@ from twisted.internet.endpoints import ( from twisted.internet.interfaces import IProtocol, IProtocolFactory from twisted.internet.protocol import Factory, Protocol from twisted.protocols.tls import TLSMemoryBIOProtocol -from twisted.web.http import HTTPChannel +from twisted.web.http import HTTPChannel, Request from synapse.config.server import ProxyConfig, parse_proxy_config from synapse.http.client import BlocklistingReactorWrapper @@ -995,14 +995,12 @@ def _get_test_protocol_factory() -> IProtocolFactory: Returns: interfaces.IProtocolFactory """ - server_factory = Factory.forProtocol(HTTPChannel) - - # Request.finish expects the factory to have a 'log' method. - server_factory.log = _log_request - - return server_factory + return _HTTPFactory() -def _log_request(request: str) -> None: - """Implements Factory.log, which is expected by Request.finish""" - logger.info("Completed request %s", request) +class _HTTPFactory(Factory): + protocol = HTTPChannel + + def log(self, request: Request) -> None: + """Request.finish expects the factory to have a 'log' method.""" + logger.info("Completed request %s", request) diff --git a/tests/http/test_server.py b/tests/http/test_server.py new file mode 100644 index 0000000000..24ce91bd1e --- /dev/null +++ b/tests/http/test_server.py @@ -0,0 +1,118 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import cast +from unittest.mock import Mock + +from twisted.web.server import Request + +from synapse.http.server import _ByteProducer +from synapse.logging.context import ( + LoggingContext, +) +from synapse.logging.opentracing import start_active_span + +try: + import jaeger_client + import opentracing + + from synapse.logging.scopecontextmanager import LogContextScopeManager +except ImportError: + jaeger_client = None # type: ignore + +from synapse.util.iterutils import chunk_seq + +from tests.unittest import TestCase + + +class ByteProducerTestCase(TestCase): + if jaeger_client is None: + skip = "Requires jaeger_client" # type: ignore[unreachable] + + def test_paused_response(self) -> None: + """Test that a paused response is correctly resumed and finished without + logging errors. + + This is a regression test where writing a large response that gets + paused by the transport produces the error "Closing scope ... which is + not the currently-active one" + """ + + ## 1. First, set up the Jaeger tracer. + config = jaeger_client.config.Config( + config={}, service_name="test", scope_manager=LogContextScopeManager() + ) + tracer = config.create_tracer( + sampler=jaeger_client.ConstSampler(True), + reporter=jaeger_client.reporter.NullReporter(), + ) + previous_tracer = opentracing.tracer + opentracing.set_global_tracer(tracer) + self.addCleanup(opentracing.set_global_tracer, previous_tracer) + + ## 2. Now create a mock Request. + request = Mock(spec=Request) + + written_buffer = b"" # The data written to the request + + def write(data: bytes) -> None: + nonlocal written_buffer + written_buffer += data + + # Pause after the first write, as the transport does when its send + # buffer fills up. + if request.write.call_count == 1: + request.registerProducer.call_args.args[0].pauseProducing() + + request.write.side_effect = write + + ## 3. Now we start writing the bytes to the request via the + ## _ByteProducer. This should not log any errors or warnings. + with ( + self.assertNoLogs("synapse.logging.context", "WARNING"), + self.assertNoLogs("synapse.logging.scopecontextmanager", "ERROR"), + ): + # The data that we write to the request via the _ByteProducer. This + # is arbitrary and large to ensure multiple chunks are written. It + # needs to be big enough that _ByteProducer will not try and batch + # up the chunks internally (if it does then the assertion that we + # have multiple writes below will fail). + buffer_to_write = b"x" * 6000 + + # The initial write happens within the request log context and span + with ( + LoggingContext(name="request", server_name="test_server"), + start_active_span("servlet"), + ): + # Break the buffer into chunks for writing. We use an arbitrary + # chunk size that ensures we have a few distinct writes. + iterable = chunk_seq(buffer_to_write, 2000) + + # Start writing the data. The _ByteProducer will start writing + # immediately on construction. + producer = _ByteProducer(cast(Request, request), iterable) + + # The request paused after the first write, and so it should not + # have finished yet. + request.finish.assert_not_called() + self.assertLess(len(written_buffer), len(buffer_to_write)) + self.assertTrue(producer._paused) + + # Mimic the request resuming the producer. This happens from the + # reactor and so outside the request log context. + producer.resumeProducing() + + # All data should now be written and the request should be finished. + request.finish.assert_called_once() + self.assertEqual(written_buffer, buffer_to_write) diff --git a/tests/http/test_simple_client.py b/tests/http/test_simple_client.py index 25c0e1081b..4db330dd73 100644 --- a/tests/http/test_simple_client.py +++ b/tests/http/test_simple_client.py @@ -25,6 +25,7 @@ from netaddr import IPSet from twisted.internet import defer from twisted.internet.error import DNSLookupError from twisted.internet.testing import MemoryReactor +from twisted.python.failure import Failure from synapse.http import RequestTimedOutError from synapse.http.client import SimpleHttpClient @@ -65,7 +66,7 @@ class SimpleHttpClientTests(HomeserverTestCase): self.assertEqual(host, "1.2.3.4") self.assertEqual(port, 8008) e = Exception("go away") - factory.clientConnectionFailed(None, e) + factory.clientConnectionFailed(self.reactor.connectors[0], Failure(e)) self.pump(0.5) f = self.failureResultOf(d) @@ -119,6 +120,7 @@ class SimpleHttpClientTests(HomeserverTestCase): conn = Mock() client = clients[0][2].buildProtocol(None) + assert client is not None client.makeConnection(conn) # Deferred is still without a result diff --git a/tests/http/test_site.py b/tests/http/test_site.py index c93aadfee1..532b7d0fc6 100644 --- a/tests/http/test_site.py +++ b/tests/http/test_site.py @@ -50,6 +50,7 @@ class SynapseRequestTestCase(HomeserverTestCase): # complete the connection and wire it up to a fake transport client_address = IPv6Address("TCP", "::1", 2345) protocol = factory.buildProtocol(client_address) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -70,6 +71,7 @@ class SynapseRequestTestCase(HomeserverTestCase): # now send an oversized request protocol = factory.buildProtocol(client_address) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -117,6 +119,7 @@ class SynapseRequestTestCase(HomeserverTestCase): # complete the connection and wire it up to a fake transport client_address = IPv6Address("TCP", "::1", 2345) protocol = factory.buildProtocol(client_address) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -137,6 +140,7 @@ class SynapseRequestTestCase(HomeserverTestCase): # now send request with content-type header protocol = factory.buildProtocol(client_address) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -168,6 +172,7 @@ class SynapseRequestTestCase(HomeserverTestCase): # complete the connection and wire it up to a fake transport client_address = IPv6Address("TCP", "::1", 2345) protocol = factory.buildProtocol(client_address) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -204,6 +209,7 @@ class SynapseRequestTestCase(HomeserverTestCase): # complete the connection and wire it up to a fake transport client_address = IPv6Address("TCP", "::1", 2345) protocol = factory.buildProtocol(client_address) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) @@ -237,6 +243,7 @@ class SynapseRequestTestCase(HomeserverTestCase): # complete the connection and wire it up to a fake transport client_address = IPv6Address("TCP", "::1", 2345) protocol = factory.buildProtocol(client_address) + assert protocol is not None transport = StringTransport() protocol.makeConnection(transport) diff --git a/tests/logging/test_remote_handler.py b/tests/logging/test_remote_handler.py index 534a1fc4ee..68bdc28573 100644 --- a/tests/logging/test_remote_handler.py +++ b/tests/logging/test_remote_handler.py @@ -39,6 +39,7 @@ def connect_logging_client( # written. factory = reactor.tcpClients.pop(client_id)[2] client = factory.buildProtocol(None) + assert isinstance(client, Protocol) server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, reactor)) client.makeConnection(FakeTransport(server, reactor, autoflush=False)) diff --git a/tests/media/test_html_preview.py b/tests/media/test_html_preview.py index ddcbbee897..d5f6785460 100644 --- a/tests/media/test_html_preview.py +++ b/tests/media/test_html_preview.py @@ -18,9 +18,9 @@ # [This file includes modifications made by New Vector Limited] # # +from unittest.mock import patch from synapse.media.preview_html import ( - _get_html_media_encodings, decode_body, parse_html_to_open_graph, summarize_paragraphs, @@ -29,14 +29,14 @@ from synapse.media.preview_html import ( from tests import unittest try: - import lxml + import bs4 except ImportError: - lxml = None # type: ignore[assignment] + bs4 = None # type: ignore[assignment] class SummarizeTestCase(unittest.TestCase): - if not lxml: - skip = "url preview feature requires lxml" + if not bs4: + skip = "url preview feature requires beautifulsoup4" def test_long_summarize(self) -> None: example_paras = [ @@ -153,8 +153,8 @@ class SummarizeTestCase(unittest.TestCase): class OpenGraphFromHtmlTestCase(unittest.TestCase): - if not lxml: - skip = "url preview feature requires lxml" + if not bs4: + skip = "url preview feature requires beautifulsoup4" def test_simple(self) -> None: html = b""" @@ -166,9 +166,9 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): """ - tree = decode_body(html, "http://example.com/test.html") - assert tree is not None - og = parse_html_to_open_graph(tree) + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."}) @@ -183,9 +183,9 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): """ - tree = decode_body(html, "http://example.com/test.html") - assert tree is not None - og = parse_html_to_open_graph(tree) + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."}) @@ -203,9 +203,9 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): """ - tree = decode_body(html, "http://example.com/test.html") - assert tree is not None - og = parse_html_to_open_graph(tree) + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) self.assertEqual( og, @@ -226,9 +226,9 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): """ - tree = decode_body(html, "http://example.com/test.html") - assert tree is not None - og = parse_html_to_open_graph(tree) + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."}) @@ -241,9 +241,9 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): """ - tree = decode_body(html, "http://example.com/test.html") - assert tree is not None - og = parse_html_to_open_graph(tree) + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) self.assertEqual(og, {"og:title": None, "og:description": "Some text."}) @@ -273,9 +273,9 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): """ - tree = decode_body(html, "http://example.com/test.html") - assert tree is not None - og = parse_html_to_open_graph(tree) + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) self.assertEqual(og, {"og:title": "Title", "og:description": "Some text."}) @@ -310,23 +310,25 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): """ - tree = decode_body(html, "http://example.com/test.html") - assert tree is not None - og = parse_html_to_open_graph(tree) + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) self.assertEqual(og, {"og:title": None, "og:description": "Some text."}) def test_empty(self) -> None: """Test a body with no data in it.""" html = b"" - tree = decode_body(html, "http://example.com/test.html") - self.assertIsNone(tree) + soup = decode_body(html, "http://example.com/test.html") + self.assertIsNone(soup) - def test_no_tree(self) -> None: + def test_no_soup(self) -> None: """A valid body with no tree in it.""" - html = b"\x00" - tree = decode_body(html, "http://example.com/test.html") - self.assertIsNone(tree) + with patch( + "bs4.BeautifulSoup", + side_effect=bs4.ParserRejectedMarkup("Invalid markup"), + ): + self.assertIsNone(decode_body(b"", "https://example.com/")) def test_xml(self) -> None: """Test decoding XML and ensure it works properly.""" @@ -339,24 +341,9 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): FooSome text. """.strip() - tree = decode_body(html, "http://example.com/test.html") - assert tree is not None - og = parse_html_to_open_graph(tree) - self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."}) - - def test_invalid_encoding(self) -> None: - """An invalid character encoding should be ignored and treated as UTF-8, if possible.""" - html = b""" - - Foo - - Some text. - - - """ - tree = decode_body(html, "http://example.com/test.html", "invalid-encoding") - assert tree is not None - og = parse_html_to_open_graph(tree) + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."}) def test_invalid_encoding2(self) -> None: @@ -370,10 +357,10 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): """ - tree = decode_body(html, "http://example.com/test.html") - assert tree is not None - og = parse_html_to_open_graph(tree) - self.assertEqual(og, {"og:title": "ÿÿ Foo", "og:description": "Some text."}) + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) + self.assertEqual(og, {"og:title": "˙˙ Foo", "og:description": "Some text."}) def test_windows_1252(self) -> None: """A body which uses cp1252, but doesn't declare that.""" @@ -385,10 +372,98 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): """ + soup = decode_body(html, "http://example.com/test.html") + assert soup is not None + og = parse_html_to_open_graph(soup) + self.assertIn("og:title", og) + og.pop("og:title") + self.assertEqual(og, {"og:description": "Some text."}) + + def test_image(self) -> None: + """Test that ensures an image can be pulled from the HTML.""" + + # Tags is a list of two-element tuples: the HTML tag and the expected image which + # is chosen. + # + # They're in a particular order such that we can prove "higher" priority tags + # are parsed out first, e.g. OpenGraph, then meta tags, then images of a certain + # height/width, favicons, etc. + tags = [ + ( + b"""""", + "meta-prop", + ), + ( + b"""""", + "meta-IMAGE", + ), + ( + b"""""", + "meta-image", + ), + (b"""""", "img"), + ( + b"""""", + "img", + ), + ( + b"""""", + "img", + ), + ( + b"""""", + "img", + ), + ( + b"""""", + "img", + ), + # Put this image again since if it is the *only* image it will be used. + ( + b"""""", + "img-no-width-no-height", + ), + ( + b"""""", + "favicon", + ), + ] + + while tags: + html = b"" + b"".join(t[0] for t in tags) + b"" + tree = decode_body(html, "http://example.com/test.html") + assert tree is not None + og = parse_html_to_open_graph(tree) + self.assertEqual( + og, + { + "og:title": None, + "og:description": None, + "og:image": f"https://example.com/{tags[0][1]}.png", + }, + ) + + # Remove the highest remaining priority item. + tags.pop(0) + + def test_image_bad_height_width(self) -> None: + """A bad height/width should be ignored.""" + + html = b""" + + + """ tree = decode_body(html, "http://example.com/test.html") assert tree is not None og = parse_html_to_open_graph(tree) - self.assertEqual(og, {"og:title": "ó", "og:description": "Some text."}) + self.assertEqual( + og, + { + "og:title": None, + "og:description": None, + "og:image": "http://example.com/no-height-width.png", + }, + ) def test_twitter_tag(self) -> None: """Twitter card tags should be used if nothing else is available.""" @@ -397,6 +472,7 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): + """ tree = decode_body(html, "http://example.com/test.html") @@ -408,6 +484,7 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): "og:title": None, "og:description": "Description", "og:site_name": "@matrixdotorg", + "og:image": "https://example.com/test.png", }, ) @@ -419,6 +496,8 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): + + """ tree = decode_body(html, "http://example.com/test.html") @@ -430,6 +509,7 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): "og:title": None, "og:description": "Real Description", "og:site_name": "matrix.org", + "og:image": "https://example.com/good.png", }, ) @@ -473,115 +553,36 @@ class OpenGraphFromHtmlTestCase(unittest.TestCase): }, ) - -class MediaEncodingTestCase(unittest.TestCase): - def test_meta_charset(self) -> None: - """A character encoding is found via the meta tag.""" - encodings = _get_html_media_encodings( - b""" - - - - - """, - "text/html", + def test_ignored_tags(self) -> None: + """Test ignored elements.""" + html = b""" +
Foo bar
+
A real description
+ """ + tree = decode_body(html, "http://example.com/test.html") + assert tree is not None + og = parse_html_to_open_graph(tree) + self.assertEqual( + og, + { + "og:title": None, + "og:description": "A real description", + }, ) - self.assertEqual(list(encodings), ["ascii", "utf-8", "cp1252"]) - # A less well-formed version. - encodings = _get_html_media_encodings( - b""" - - < meta charset = ascii> - - - """, - "text/html", + def test_aria_tags(self) -> None: + """Test ignored elements.""" + html = b""" +
Foo bar
+
A real description
+ """ + tree = decode_body(html, "http://example.com/test.html") + assert tree is not None + og = parse_html_to_open_graph(tree) + self.assertEqual( + og, + { + "og:title": None, + "og:description": "A real description", + }, ) - self.assertEqual(list(encodings), ["ascii", "utf-8", "cp1252"]) - - def test_meta_charset_underscores(self) -> None: - """A character encoding contains underscore.""" - encodings = _get_html_media_encodings( - b""" - - - - - """, - "text/html", - ) - self.assertEqual(list(encodings), ["shift_jis", "utf-8", "cp1252"]) - - def test_xml_encoding(self) -> None: - """A character encoding is found via the meta tag.""" - encodings = _get_html_media_encodings( - b""" - - - - """, - "text/html", - ) - self.assertEqual(list(encodings), ["ascii", "utf-8", "cp1252"]) - - def test_meta_xml_encoding(self) -> None: - """Meta tags take precedence over XML encoding.""" - encodings = _get_html_media_encodings( - b""" - - - - - - """, - "text/html", - ) - self.assertEqual(list(encodings), ["utf-16", "ascii", "utf-8", "cp1252"]) - - def test_content_type(self) -> None: - """A character encoding is found via the Content-Type header.""" - # Test a few variations of the header. - headers = ( - 'text/html; charset="ascii";', - "text/html;charset=ascii;", - 'text/html; charset="ascii"', - "text/html; charset=ascii", - 'text/html; charset="ascii;', - 'text/html; charset=ascii";', - ) - for header in headers: - encodings = _get_html_media_encodings(b"", header) - self.assertEqual(list(encodings), ["ascii", "utf-8", "cp1252"]) - - def test_fallback(self) -> None: - """A character encoding cannot be found in the body or header.""" - encodings = _get_html_media_encodings(b"", "text/html") - self.assertEqual(list(encodings), ["utf-8", "cp1252"]) - - def test_duplicates(self) -> None: - """Ensure each encoding is only attempted once.""" - encodings = _get_html_media_encodings( - b""" - - - - - - """, - 'text/html; charset="UTF_8"', - ) - self.assertEqual(list(encodings), ["utf-8", "cp1252"]) - - def test_unknown_invalid(self) -> None: - """A character encoding should be ignored if it is unknown or invalid.""" - encodings = _get_html_media_encodings( - b""" - - - - - """, - 'text/html; charset="invalid"', - ) - self.assertEqual(list(encodings), ["utf-8", "cp1252"]) diff --git a/tests/media/test_oembed.py b/tests/media/test_oembed.py index dc13c03df3..74fd77f330 100644 --- a/tests/media/test_oembed.py +++ b/tests/media/test_oembed.py @@ -34,14 +34,14 @@ from synapse.util.clock import Clock from tests.unittest import HomeserverTestCase try: - import lxml + import bs4 except ImportError: - lxml = None # type: ignore[assignment] + bs4 = None # type: ignore[assignment] class OEmbedTests(HomeserverTestCase): - if not lxml: - skip = "url preview feature requires lxml" + if not bs4: + skip = "url preview feature requires beautifulsoup4" def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.oembed = OEmbedProvider(hs) diff --git a/tests/media/test_url_previewer.py b/tests/media/test_url_previewer.py index 3d706c7e90..da6ea60ab1 100644 --- a/tests/media/test_url_previewer.py +++ b/tests/media/test_url_previewer.py @@ -29,14 +29,14 @@ from tests import unittest from tests.unittest import override_config try: - import lxml + import bs4 except ImportError: - lxml = None # type: ignore[assignment] + bs4 = None # type: ignore[assignment] class URLPreviewTests(unittest.HomeserverTestCase): - if not lxml: - skip = "url preview feature requires lxml" + if not bs4: + skip = "url preview feature requires beautifulsoup4" def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: config = self.default_config() diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index b4b14c87b9..79b3d27d11 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -282,11 +282,11 @@ class ModuleApiTestCase(BaseModuleApiTestCase): # Create and send a state event content = { "events_default": 0, - "users": {user_id: 100}, "state_default": 50, "users_default": 0, "events": {"test.event.type": 25}, } + event_dict = { "room_id": room_id, "type": "m.room.power_levels", @@ -743,6 +743,7 @@ class ModuleApiTestCase(BaseModuleApiTestCase): # Now do the happy path. user_id = self.register_user("user", "password") + access_token = self.login(user_id, "password") room_id, room_alias = self.get_success( self.module_api.create_room( @@ -750,6 +751,18 @@ class ModuleApiTestCase(BaseModuleApiTestCase): ) ) + # Check room creator. + channel = self.make_request( + "GET", + f"/_matrix/client/v3/rooms/{room_id}/state/m.room.create", + access_token=access_token, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual( + channel.json_body["room_version"], + self.hs.config.server.default_room_version.identifier, + ) + # Check room alias. self.assertEqual(room_alias, f"#foo-bar:{self.module_api.server_name}") @@ -758,6 +771,17 @@ class ModuleApiTestCase(BaseModuleApiTestCase): self.module_api.create_room(user_id=user_id, config={}, ratelimit=False) ) + channel = self.make_request( + "GET", + f"/_matrix/client/v3/rooms/{room_id}/state/m.room.create", + access_token=access_token, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual( + channel.json_body["room_version"], + self.hs.config.server.default_room_version.identifier, + ) + # Check room alias. self.assertIsNone(room_alias) @@ -825,7 +849,7 @@ class ModuleApiTestCase(BaseModuleApiTestCase): self.assertEqual(create_event.user_id, user_id) # type: ignore[attr-defined] # The event supports looking up keys via `__getitem__` although deprecated - self.assertEqual(create_event["room_id"], room_id) # type: ignore[index] + self.assertEqual(create_event["type"], EventTypes.Create) # type: ignore[index] class ModuleApiWorkerTestCase(BaseModuleApiTestCase, BaseMultiWorkerStreamTestCase): diff --git a/tests/push/test_bulk_push_rule_evaluator.py b/tests/push/test_bulk_push_rule_evaluator.py index 137bbe24b2..55c38c010b 100644 --- a/tests/push/test_bulk_push_rule_evaluator.py +++ b/tests/push/test_bulk_push_rule_evaluator.py @@ -650,3 +650,31 @@ class TestBulkPushRuleEvaluator(HomeserverTestCase): type="m.room.message", ) ) + + def _get_rule_ids(self) -> set[str]: + """Returns the IDs of all push rules that apply to Alice.""" + filtered_push_rules = self.get_success( + self.hs.get_datastores().main.get_push_rules_for_user(self.alice) + ) + return {rule.rule_id for rule, _ in filtered_push_rules.rules()} + + def test_msc4075_rules_omitted_when_disabled(self) -> None: + """The MSC4075 base rules should be filtered out unless the feature is enabled.""" + rule_ids = self._get_rule_ids() + self.assertNotIn( + "global/override/.org.matrix.msc4075.rule.rtc.invite_for_me", rule_ids + ) + self.assertNotIn( + "global/override/.org.matrix.msc4075.rule.rtc.invite_for_room", rule_ids + ) + + @override_config({"experimental_features": {"msc4075_enabled": True}}) + def test_msc4075_rules_present_when_enabled(self) -> None: + """The MSC4075 base rules should be included when the feature is enabled.""" + rule_ids = self._get_rule_ids() + self.assertIn( + "global/override/.org.matrix.msc4075.rule.rtc.invite_for_me", rule_ids + ) + self.assertIn( + "global/override/.org.matrix.msc4075.rule.rtc.invite_for_room", rule_ids + ) diff --git a/tests/push/test_http.py b/tests/push/test_http.py index 47521a773f..bd2ada9155 100644 --- a/tests/push/test_http.py +++ b/tests/push/test_http.py @@ -31,7 +31,6 @@ from synapse.rest import admin from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.rest.client import login, push_rule, pusher, receipts, room, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient from synapse.types import JsonDict from synapse.util.clock import Clock @@ -1233,24 +1232,6 @@ class MSC3881VersionsTestCase(HomeserverTestCase): versions.register_servlets, ] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = hs.get_proxied_http_client() - _ = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - def tearDown(self) -> None: # MemoryReactor doesn't trigger the shutdown phases, and we want the # Tokio thread pool to be stopped diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 791a6beab5..310049bf89 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -39,6 +39,7 @@ from synapse.replication.tcp.protocol import ( from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.server import HomeServer from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.server import FakeTransport @@ -169,17 +170,12 @@ class BaseStreamTestCase(unittest.HomeserverTestCase): # Set up client side protocol client_address = IPv4Address("TCP", "127.0.0.1", 1234) - client_protocol = client_factory.buildProtocol(("127.0.0.1", 1234)) + client_protocol = client_factory.buildProtocol(client_address) + assert client_protocol is not None # Set up the server side protocol server_address = IPv4Address("TCP", host, port) - # The type ignore is here because mypy doesn't think the host/port tuple is of - # the correct type, even though it is the exact example given for - # `twisted.internet.interfaces.IAddress`. - # Mypy was happy with the type before we overrode `buildProtocol` in - # `SynapseSite`, probably because there was enough inheritance indirection before - # withe the argument not having a type associated with it. - channel = self.site.buildProtocol((host, port)) # type: ignore[arg-type] + channel = self.site.buildProtocol(server_address) # hook into the channel's request factory so that we can keep a record # of the requests @@ -434,11 +430,12 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase): # Set up client side protocol client_address = IPv4Address("TCP", "127.0.0.1", 1234) - client_protocol = client_factory.buildProtocol(("127.0.0.1", 1234)) + client_protocol = client_factory.buildProtocol(client_address) + assert client_protocol is not None # Set up the server side protocol server_address = IPv4Address("TCP", host, port) - channel = self._hs_to_site[hs].buildProtocol((host, port)) # type: ignore[arg-type] + channel = self._hs_to_site[hs].buildProtocol(server_address) # Connect client to server and vice versa. client_to_server_transport = FakeTransport( @@ -469,6 +466,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase): client_address = IPv4Address("TCP", "127.0.0.1", 6379) client_protocol = client_factory.buildProtocol(client_address) + assert client_protocol is not None server_address = IPv4Address("TCP", host, port) server_protocol = self._redis_server.buildProtocol(server_address) @@ -483,6 +481,78 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase): ) server_protocol.makeConnection(server_to_client_transport) + def _generate_rooms_on_worker( + self, + user_id: str, + user_tok: str, + list_of_worker_names: list[str] | None = None, + try_at_most_count: int | None = None, + ) -> dict[str, str]: + """ + Given a list of worker names, generate rooms until there is at least one on each + of the named workers. + + Args: + user_id: The user_id of the user making the room. + user_tok: The token of the user making the room. + list_of_worker_names: A list of worker names that need to have rooms. By + default, each `events` worker provided in the homeserver config will be used. + try_at_most_count: A given number of iterations to try and produce rooms. If + not provided, use the number of workers multiplied by 3 for the count. + Returns: + A mapping of `worker_name`->`room_id` + """ + # Save a shorter reference to the `RoutableShardedWorkerHandlingConfig` that + # contains the information needed to not only identify the full list of workers + # (in case the default for `list_of_workernames` is used) and provides the + # routing hash function that decides which worker a given room id should go to. + events_writers_config = self.hs.config.worker.events_shard_config + + if list_of_worker_names is None: + _set_of_worker_names = set(events_writers_config.instances) + else: + _set_of_worker_names = set(list_of_worker_names) + + assert len(_set_of_worker_names) > 0 + # Save a copy of this to use now, we can use the original to assert expectations + # before returning. + set_of_workernames = set(_set_of_worker_names) + + results_mapping = {} + + # Maintain a count, in case of a runaway process. The number 3 has no magical + # significance other than at the time of writing it allowed all tests that used + # this function to pass. The count is on a one based index(counts down to 1 and + # doesn't go past), so this will allow for (3 * num_of_workers) attempts before + # giving up if `try_at_most_count` does not override. + count = try_at_most_count or len(set_of_workernames) * 3 + + while set_of_workernames: + # Rooms created at the same millisecond will have the same room_id for + # MSC4291 rooms. Bump the reactor by that much so a different room_id will + # be tried on the next iteration. + self.reactor.advance(Duration(milliseconds=1).as_secs()) + _room_id = self.helper.create_room_as(user_id, tok=user_tok) + + _worker_responsible = events_writers_config.get_instance(_room_id) + + if _worker_responsible in set_of_workernames: + results_mapping[_worker_responsible] = _room_id + # Remember to remove the worker now that it is found + set_of_workernames.remove(_worker_responsible) + + count -= 1 + if count == 1: + raise AssertionError( + "Count exhausted attempting to generate rooms. Aborting and failing test" + ) + + # Since this *IS* part of a test, lets make sure all worker names requested are + # accounted for + assert results_mapping.keys() == _set_of_worker_names + + return results_mapping + class TestReplicationDataHandler(ReplicationDataHandler): """Drop-in for ReplicationDataHandler which just collects RDATA rows""" diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 3896e0ce8a..929b1d028e 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -338,7 +338,6 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): "origin_server_ts": 1, "content": {"membership": Membership.JOIN}, "auth_events": [ - state_map[(EventTypes.Create, "")].event_id, state_map[(EventTypes.JoinRules, "")].event_id, ], "prev_events": list(prev_event_ids), diff --git a/tests/replication/test_multi_media_repo.py b/tests/replication/test_multi_media_repo.py index 000dbb5594..b2c755137b 100644 --- a/tests/replication/test_multi_media_repo.py +++ b/tests/replication/test_multi_media_repo.py @@ -104,9 +104,7 @@ class MediaRepoShardTestCase(BaseMultiWorkerStreamTestCase): (host, port, client_factory, _timeout, _bindAddress) = clients.pop() # build the test server - server_factory = Factory.forProtocol(HTTPChannel) - # Request.finish expects the factory to have a 'log' method. - server_factory.log = _log_request + server_factory = _HTTPFactory() server_tls_protocol = wrap_server_factory_for_tls( server_factory, self.reactor, sanlist=[b"DNS:example.com"] @@ -120,6 +118,7 @@ class MediaRepoShardTestCase(BaseMultiWorkerStreamTestCase): # Normally this would be done by the TCP socket code in Twisted, but we are # stubbing that out here. client_protocol = client_factory.buildProtocol(None) + assert client_protocol is not None client_protocol.makeConnection( FakeTransport(server_tls_protocol, self.reactor, client_protocol) ) @@ -130,7 +129,8 @@ class MediaRepoShardTestCase(BaseMultiWorkerStreamTestCase): ) # fish the test server back out of the server-side TLS protocol. - http_server: HTTPChannel = server_tls_protocol.wrappedProtocol + http_server = server_tls_protocol.wrappedProtocol + assert isinstance(http_server, HTTPChannel) # give the reactor a pump to get the TLS juices flowing. self.reactor.pump((0.1,)) @@ -366,9 +366,7 @@ class AuthenticatedMediaRepoShardTestCase(BaseMultiWorkerStreamTestCase): (host, port, client_factory, _timeout, _bindAddress) = clients.pop() # build the test server - server_factory = Factory.forProtocol(HTTPChannel) - # Request.finish expects the factory to have a 'log' method. - server_factory.log = _log_request + server_factory = _HTTPFactory() server_tls_protocol = wrap_server_factory_for_tls( server_factory, self.reactor, sanlist=[b"DNS:example.com"] @@ -382,6 +380,7 @@ class AuthenticatedMediaRepoShardTestCase(BaseMultiWorkerStreamTestCase): # Normally this would be done by the TCP socket code in Twisted, but we are # stubbing that out here. client_protocol = client_factory.buildProtocol(None) + assert client_protocol is not None client_protocol.makeConnection( FakeTransport(server_tls_protocol, self.reactor, client_protocol) ) @@ -392,7 +391,8 @@ class AuthenticatedMediaRepoShardTestCase(BaseMultiWorkerStreamTestCase): ) # fish the test server back out of the server-side TLS protocol. - http_server: HTTPChannel = server_tls_protocol.wrappedProtocol + http_server = server_tls_protocol.wrappedProtocol + assert isinstance(http_server, HTTPChannel) # give the reactor a pump to get the TLS juices flowing. self.reactor.pump((0.1,)) @@ -585,6 +585,9 @@ class AuthenticatedMediaRepoShardTestCase(BaseMultiWorkerStreamTestCase): return sum(len(files) for _, _, files in os.walk(path)) -def _log_request(request: Request) -> None: - """Implements Factory.log, which is expected by Request.finish""" - logger.info("Completed request %s", request) +class _HTTPFactory(Factory): + protocol = HTTPChannel + + def log(self, request: Request) -> None: + """Request.finish expects the factory to have a 'log' method.""" + logger.info("Completed request %s", request) diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py index f37394c1f5..1b9b9aa630 100644 --- a/tests/replication/test_sharded_event_persister.py +++ b/tests/replication/test_sharded_event_persister.py @@ -18,8 +18,6 @@ # [This file includes modifications made by New Vector Limited] # # -import logging -from unittest.mock import patch from twisted.internet.testing import MemoryReactor @@ -32,8 +30,6 @@ from synapse.util.clock import Clock from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import make_request -logger = logging.getLogger(__name__) - class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): """Checks event persisting sharding works""" @@ -63,17 +59,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): } return conf - def _create_room(self, room_id: str, user_id: str, tok: str) -> None: - """Create a room with given room_id""" - - # We control the room ID generation by patching out the - # `_generate_room_id` method - with patch( - "synapse.handlers.room.RoomCreationHandler._generate_room_id" - ) as mock: - mock.side_effect = lambda: room_id - self.helper.create_room_as(user_id, tok=tok) - def test_basic(self) -> None: """Simple test to ensure that multiple rooms can be created and joined, and that different rooms get handled by different instances. @@ -145,10 +130,16 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): ) sync_hs_site = self._hs_to_site[sync_hs] - # Specially selected room IDs that get persisted on different workers. - room_id1 = "!foo:test" - room_id2 = "!baz:test" + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + store = self.hs.get_datastores().main + + room_generation_results = self._generate_rooms_on_worker(user_id, access_token) + room_id1 = room_generation_results["worker1"] + room_id2 = room_generation_results["worker2"] + + # Verify that the expected worker is responsible for the specific room self.assertEqual( self.hs.config.worker.events_shard_config.get_instance(room_id1), "worker1" ) @@ -156,15 +147,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): self.hs.config.worker.events_shard_config.get_instance(room_id2), "worker2" ) - user_id = self.register_user("user", "pass") - access_token = self.login("user", "pass") - - store = self.hs.get_datastores().main - - # Create two room on the different workers. - self._create_room(room_id1, user_id, access_token) - self._create_room(room_id2, user_id, access_token) - # The other user joins self.helper.join( room=room_id1, user=self.other_user_id, tok=self.other_access_token diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 4deb3c29f4..b776929e21 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -3318,15 +3318,25 @@ class MakeRoomAdminTestCase(unittest.HomeserverTestCase): self.creator, tok=self.creator_tok, is_public=True ) - # The creator drops admin rights in the room. + # In the case that this is a msc4289 compliant room, we can not use the room + # creator as the "puppeted" user. Join another user to use as a sentinel + # instead. The room's creator will leave the room before the command runs. + self.helper.join(room_id, self.second_user_id, tok=self.second_tok) + + # The creator gives a placeholder of rights in the room. The power level can be + # 0, as long as it is present it will be considered. Having it be missing will + # produce a different error: "No local admin user in room" which is not what + # this test is supposed to check pl = self.helper.get_state( room_id, EventTypes.PowerLevels, tok=self.creator_tok ) - pl["users"][self.creator] = 0 + pl["users"][self.second_user_id] = 0 self.helper.send_state( room_id, EventTypes.PowerLevels, body=pl, tok=self.creator_tok ) + self.helper.leave(room_id, self.creator, tok=self.creator_tok) + channel = self.make_request( "POST", f"/_synapse/admin/v1/rooms/{room_id}/make_room_admin", diff --git a/tests/rest/client/sliding_sync/test_extension_profiles.py b/tests/rest/client/sliding_sync/test_extension_profiles.py index 5c2d17547c..2e76d6923c 100644 --- a/tests/rest/client/sliding_sync/test_extension_profiles.py +++ b/tests/rest/client/sliding_sync/test_extension_profiles.py @@ -525,7 +525,7 @@ class SlidingSyncProfilesTestCase(SlidingSyncBase): ) -> None: """ Test that profile extension response returns a null for the user in - incremental sync. + incremental sync, if the user left the last shared room. """ # Make an initial Sliding Sync request with the profiles extension enabled profiles_config: dict = { @@ -552,6 +552,46 @@ class SlidingSyncProfilesTestCase(SlidingSyncBase): ], ) + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_null_profile_returned_we_left_all_rooms( + self, + request_fields: bool, + ) -> None: + """ + Test that profile extension response returns a null for the user in + incremental sync, if we left the last shared room with a user. + """ + # Make an initial Sliding Sync request with the profiles extension enabled + profiles_config: dict = { + "enabled": True, + } + if request_fields: + profiles_config["fields"] = ["field"] + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": profiles_config, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + self.helper.leave(self.joined_room, self.user, tok=self.tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # We should see a null profile + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + ) + @override_config({"include_profile_updates_in_sync": True}) def test_profile_returned_if_user_left_then_rejoined(self) -> None: """ diff --git a/tests/rest/client/sliding_sync/test_rooms_invites.py b/tests/rest/client/sliding_sync/test_rooms_invites.py index 85628833f5..09a6674f53 100644 --- a/tests/rest/client/sliding_sync/test_rooms_invites.py +++ b/tests/rest/client/sliding_sync/test_rooms_invites.py @@ -21,7 +21,7 @@ import synapse.rest.admin from synapse.api.constants import EventTypes, HistoryVisibility from synapse.rest.client import login, room, sync from synapse.server import HomeServer -from synapse.types import UserID +from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -66,6 +66,48 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): self.storage_controllers = hs.get_storage_controllers() super().prepare(reactor, clock, hs) + self.room_version = hs.config.server.default_room_version + + def assertHasSubset( + self, + container: list[JsonDict], + contains_these: list[JsonDict], + ) -> None: + """ + Fail the test if the count of elements in both `container` and `contains_these` + do not match, and if the individual elements in `container` is not a super set + of the individual elements of `contains_these`. + + Useful for comparison of lists of PDU dictionaries that may or may not have + stripped state, so the minimum required data is asserted to be present. + + (Note: maybe revert after https://github.com/element-hq/synapse/pull/19723 if + this is still necessary) + """ + assert contains_these, "`contains_these` was empty" + assert container, "`container` was empty" + + self.assertEqual( + len(container), + len(contains_these), + f"container must have the same number of elements as contains_these:\n\n{container=}\n\n{contains_these=}", + ) + + for dict_to_search_for in contains_these: + for container_entry in container: + # The <= operator is a subset comparison operator when used on + # 'set-like' containers + if dict_to_search_for.items() <= container_entry.items(): + # The searched for item was found, move on. Break will skip the + # else statement below + break + else: + # Searching the 'container' for this subset yielded nothing, that is + # an error. + raise AssertionError( + "The searched for dict was not present in the container:\n\n" + f"Searched for {dict_to_search_for}\n\nContainer: {container}" + ) def test_rooms_invite_shared_history_initial_sync(self) -> None: """ @@ -73,7 +115,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): initial sync. This is an `invite` room so we should only have `stripped_state` (no `timeline`) - but we also shouldn't see any timeline events because the history visiblity is + but we also shouldn't see any timeline events because the history visibility is `shared` and we haven't joined the room yet. """ user1_id = self.register_user("user1", "pass") @@ -85,7 +127,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) # Ensure we're testing with a room with `shared` history visibility which means - # history visible until you actually join the room. + # history won't be visible until you actually join the room. history_visibility_response = self.helper.get_state( room_id1, EventTypes.RoomHistoryVisibility, tok=user2_tok ) @@ -138,12 +180,15 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1], ) # We should have some `stripped_state` so the potential joiner can identify the - # room (we don't care about the order). - self.assertCountEqual( + # room (we don't care about the order). The exception is the creation event + # which is a full PDU format from room v12 and newer + self.assertHasSubset( response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"room_version": "11"}, + "content": { + "room_version": self.room_version.identifier, + }, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -167,7 +212,6 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): "type": "m.room.member", }, ], - response_body["rooms"][room_id1]["invite_state"], ) def test_rooms_invite_shared_history_incremental_sync(self) -> None: @@ -248,12 +292,15 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1], ) # We should have some `stripped_state` so the potential joiner can identify the - # room (we don't care about the order). - self.assertCountEqual( + # room (we don't care about the order). The exception is the creation event + # which is a full PDU format from room v12 and newer + self.assertHasSubset( response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"room_version": "11"}, + "content": { + "room_version": self.room_version.identifier, + }, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -277,7 +324,6 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): "type": "m.room.member", }, ], - response_body["rooms"][room_id1]["invite_state"], ) def test_rooms_invite_world_readable_history_initial_sync(self) -> None: @@ -369,12 +415,15 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1], ) # We should have some `stripped_state` so the potential joiner can identify the - # room (we don't care about the order). - self.assertCountEqual( + # room (we don't care about the order). The exception is the creation event + # which is a full PDU format from room v12 and newer + self.assertHasSubset( response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"room_version": "11"}, + "content": { + "room_version": self.room_version.identifier, + }, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -398,7 +447,6 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): "type": "m.room.member", }, ], - response_body["rooms"][room_id1]["invite_state"], ) def test_rooms_invite_world_readable_history_incremental_sync(self) -> None: @@ -495,12 +543,15 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1], ) # We should have some `stripped_state` so the potential joiner can identify the - # room (we don't care about the order). - self.assertCountEqual( + # room (we don't care about the order). The exception is the creation event + # which is a full PDU format from room v12 and newer + self.assertHasSubset( response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"room_version": "11"}, + "content": { + "room_version": self.room_version.identifier, + }, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -524,5 +575,4 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): "type": "m.room.member", }, ], - response_body["rooms"][room_id1]["invite_state"], ) diff --git a/tests/rest/client/sliding_sync/test_rooms_required_state.py b/tests/rest/client/sliding_sync/test_rooms_required_state.py index 901f22a35d..5aff19c7fd 100644 --- a/tests/rest/client/sliding_sync/test_rooms_required_state.py +++ b/tests/rest/client/sliding_sync/test_rooms_required_state.py @@ -1289,7 +1289,7 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): self.helper.send_state( room_id1, event_type=EventTypes.PowerLevels, - body={"users": {user1_id: 50, user2_id: 100}}, + body={"users": {user1_id: 50}}, tok=user2_tok, ) self.helper.send_state( diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py index 4eb918d5c5..38196ca2fc 100644 --- a/tests/rest/client/test_auth.py +++ b/tests/rest/client/test_auth.py @@ -344,6 +344,16 @@ class UIAuthTests(unittest.HomeserverTestCase): }, ) + def test_ui_auth_null_auth(self) -> None: + """ + A null `auth` value is treated as if it was omitted. + """ + channel = self.delete_device( + self.user_tok, self.device_id, HTTPStatus.UNAUTHORIZED, {"auth": None} + ) + self.assertIn("session", channel.json_body) + self.assertIn({"stages": ["m.login.password"]}, channel.json_body["flows"]) + @override_config({"password_config": {"enabled": "only_for_reauth"}}) def test_ui_auth_with_passwords_for_reauth_only(self) -> None: """ diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index 3af7d13858..93e933d991 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -20,16 +20,23 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor -from synapse.api.errors import Codes +from synapse.api.errors import Codes, NotFoundError from synapse.rest import admin -from synapse.rest.client import delayed_events, login, room, sync, versions +from synapse.rest.client import ( + account, + delayed_events, + login, + room, + sync, + versions, +) from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient -from synapse.types import JsonDict +from synapse.types import JsonDict, create_requester from synapse.util.clock import Clock from synapse.util.duration import Duration from tests import unittest +from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import FakeChannel from tests.unittest import HomeserverTestCase @@ -41,24 +48,6 @@ _EVENT_TYPE = "com.example.test" class DelayedEventsUnstableSupportTestCase(HomeserverTestCase): servlets = [versions.register_servlets] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = hs.get_proxied_http_client() - _ = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - def tearDown(self) -> None: # MemoryReactor doesn't trigger the shutdown phases, and we want the # Tokio thread pool to be stopped @@ -86,6 +75,7 @@ class DelayedEventsTestCase(HomeserverTestCase): servlets = [ admin.register_servlets, + account.register_servlets, delayed_events.register_servlets, login.register_servlets, room.register_servlets, @@ -784,6 +774,130 @@ class DelayedEventsTestCase(HomeserverTestCase): self._find_sent_delayed_event(self.user1_access_token, delay_id, False) self._find_sent_delayed_event(self.user2_access_token, delay_id, False) + @parameterized.expand((("client_api", False), ("admin_api", True))) + def test_delayed_events_are_cancelled_on_deactivation( + self, _name: str, by_admin: bool + ) -> None: + store = self.hs.get_datastores().main + handler = self.hs.get_delayed_events_handler() + + # user1 schedules a delayed message event and a delayed state event. + state_key = "to_cancel_on_deactivation" + user1_delay_ids = [] + # TODO: Schedule these through the dedicated delayed events endpoint + # instead of the delay query parameter on `/send` and `/state`, once + # https://github.com/element-hq/synapse/pull/19354 has landed. + for path in ( + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 900), + _get_path_for_delayed_state(self.room_id, _EVENT_TYPE, state_key, 900), + ): + channel = self.make_request( + "POST" if "/send/" in path else "PUT", + path, + {"key": "value"}, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + user1_delay_ids.append(channel.json_body["delay_id"]) + + # user2 schedules a delayed event to be sent later than user1's. + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 2000), + {"key": "value"}, + self.user2_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + user2_delay_id = channel.json_body["delay_id"] + + self.assertEqual(2, len(self._get_delayed_events())) + + # The next send is scheduled for user1's delayed events. + assert handler._next_delayed_event_call is not None + self.assertLess( + handler._next_delayed_event_call.getTime() - self.reactor.seconds(), 1 + ) + + if by_admin: + self.register_user("admin", "pass", admin=True) + channel = self.make_request( + "POST", + f"/_synapse/admin/v1/deactivate/{self.user1_user_id}", + {}, + self.login("admin", "pass"), + ) + else: + channel = self.make_request( + "POST", + "account/deactivate", + { + "auth": { + "type": "m.login.password", + "user": self.user1_user_id, + "password": "pass", + }, + }, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # user1's delayed events are gone... + self.assertListEqual( + [], self.get_success(store.get_all_delayed_events_for_user("user1")) + ) + for delay_id in user1_delay_ids: + self.get_failure( + store.get_delayed_event_for_user(delay_id, "user1"), NotFoundError + ) + + # ...and the next send has been rescheduled for user2's delayed event. + assert handler._next_delayed_event_call is not None + self.assertGreater( + handler._next_delayed_event_call.getTime() - self.reactor.seconds(), 1 + ) + + # Nothing gets sent when user1's delayed events would have timed out. + self.reactor.advance(1) + for delay_id in user1_delay_ids: + self._find_sent_delayed_event(self.user2_access_token, delay_id, False) + self.helper.get_state( + self.room_id, + _EVENT_TYPE, + self.user2_access_token, + state_key=state_key, + expect_code=HTTPStatus.NOT_FOUND, + ) + + # user2's delayed event is unaffected, and still gets sent. + self._find_sent_delayed_event(self.user2_access_token, user2_delay_id, False) + self.reactor.advance(1) + for delay_id in user1_delay_ids: + self._find_sent_delayed_event(self.user2_access_token, delay_id, False) + self._find_sent_delayed_event(self.user2_access_token, user2_delay_id, True) + + def test_deactivation_cancels_the_timer_when_no_delayed_events_remain( + self, + ) -> None: + handler = self.hs.get_delayed_events_handler() + + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 900), + {}, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertIsNotNone(handler._next_delayed_event_call) + + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + self.user1_user_id, + erase_data=False, + requester=create_requester(self.user1_user_id), + ) + ) + self.assertIsNone(handler._next_delayed_event_call) + def _get_delayed_events(self) -> list[JsonDict]: channel = self.make_request( "GET", @@ -858,6 +972,56 @@ class DelayedEventsTestCase(HomeserverTestCase): self.fail("Did not find event with matching delay_id") +class DelayedEventsWorkerTestCase(BaseMultiWorkerStreamTestCase): + """Tests delayed events when some of the work happens off the main process.""" + + servlets = [ + admin.register_servlets, + delayed_events.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["max_event_delay_duration"] = "24h" + return config + + def test_delayed_events_are_cancelled_on_deactivation_from_worker(self) -> None: + worker_hs = self.make_worker_hs("synapse.app.generic_worker") + store = self.hs.get_datastores().main + handler = self.hs.get_delayed_events_handler() + + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + room_id = self.helper.create_room_as(user_id, tok=access_token) + + channel = self.make_request( + "POST", + _get_path_for_delayed_send(room_id, _EVENT_TYPE, 900), + {}, + access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertIsNotNone(handler._next_delayed_event_call) + + # Deactivate the account on the worker. The main process, which is the + # one to send delayed events, must get to know about it. + self.get_success( + worker_hs.get_deactivate_account_handler().deactivate_account( + user_id, + erase_data=False, + requester=create_requester(user_id), + ) + ) + + self.assertListEqual( + [], self.get_success(store.get_all_delayed_events_for_user("user")) + ) + self.assertIsNone(handler._next_delayed_event_call) + + def _get_path_for_delayed_state( room_id: str, event_type: str, state_key: str, delay_ms: int ) -> str: diff --git a/tests/rest/client/test_keys.py b/tests/rest/client/test_keys.py index 4f3d6a365f..c957354cfe 100644 --- a/tests/rest/client/test_keys.py +++ b/tests/rest/client/test_keys.py @@ -342,6 +342,43 @@ class KeyQueryTestCase(unittest.HomeserverTestCase): ) self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + def test_device_signing_null_or_non_object_auth(self) -> None: + """A null `auth` asks for UIA like an omitted one, and a non-object `auth` + is rejected, rather than either causing a 500.""" + password = "wonderland" + device_id = "ABCDEFGHI" + alice_id = self.register_user("alice", password) + alice_token = self.login("alice", password, device_id=device_id) + + channel = self.make_request( + "POST", + "/_matrix/client/v3/keys/device_signing/upload", + self.make_device_keys(alice_id, device_id), + alice_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + + keys = self.make_device_keys(alice_id, device_id) + keys["auth"] = None + channel = self.make_request( + "POST", + "/_matrix/client/v3/keys/device_signing/upload", + keys, + alice_token, + ) + self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.result) + self.assertIn("session", channel.json_body) + + keys["auth"] = "m.login.password" + channel = self.make_request( + "POST", + "/_matrix/client/v3/keys/device_signing/upload", + keys, + alice_token, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.BAD_JSON) + class SigningKeyUploadServletTestCase(unittest.HomeserverTestCase): servlets = [ diff --git a/tests/rest/client/test_login_token_request.py b/tests/rest/client/test_login_token_request.py index d6b2cf054e..28a0fca9ce 100644 --- a/tests/rest/client/test_login_token_request.py +++ b/tests/rest/client/test_login_token_request.py @@ -19,12 +19,14 @@ # # +from http import HTTPStatus + from twisted.internet.testing import MemoryReactor +from synapse.api.errors import Codes from synapse.rest import admin from synapse.rest.client import login, login_token_request, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient from synapse.util.clock import Clock from tests import unittest @@ -48,19 +50,6 @@ class LoginTokenRequestServletTestCase(unittest.HomeserverTestCase): self.hs.config.registration.auto_join_rooms = [] self.hs.config.captcha.enable_registration_captcha = False - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = self.hs.get_proxied_http_client() - _ = HttpClient( - reactor=self.hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - return self.hs def tearDown(self) -> None: @@ -126,6 +115,31 @@ class LoginTokenRequestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 200, channel.result) self.assertEqual(channel.json_body["user_id"], user_id) + @override_config({"login_via_existing_session": {"enabled": True}}) + def test_uia_null_auth(self) -> None: + """A null `auth` value is treated as if it was omitted.""" + self.register_user(self.user, self.password) + token = self.login(self.user, self.password) + + channel = self.make_request( + "POST", GET_TOKEN_ENDPOINT, {"auth": None}, access_token=token + ) + self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, msg=channel.result) + self.assertIn("session", channel.json_body) + self.assertIn({"stages": ["m.login.password"]}, channel.json_body["flows"]) + + @override_config({"login_via_existing_session": {"enabled": True}}) + def test_uia_non_object_auth(self) -> None: + """A non-object `auth` value is rejected with a 400.""" + self.register_user(self.user, self.password) + token = self.login(self.user, self.password) + + channel = self.make_request( + "POST", GET_TOKEN_ENDPOINT, {"auth": "m.login.password"}, access_token=token + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, msg=channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.BAD_JSON) + @override_config( {"login_via_existing_session": {"enabled": True, "require_ui_auth": False}} ) diff --git a/tests/rest/client/test_matrixrtc.py b/tests/rest/client/test_matrixrtc.py index 63c7632a88..545a5a8e8c 100644 --- a/tests/rest/client/test_matrixrtc.py +++ b/tests/rest/client/test_matrixrtc.py @@ -27,7 +27,6 @@ from synapse.config.matrixrtc import TransportConfigModel from synapse.rest import admin from synapse.rest.client import login, matrixrtc, register, room, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient from synapse.util.clock import Clock from tests import unittest @@ -157,24 +156,6 @@ class MatrixRtcVersionsTestCase(HomeserverTestCase): servlets = [versions.register_servlets] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = hs.get_proxied_http_client() - _ = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - def tearDown(self) -> None: # MemoryReactor doesn't trigger the shutdown phases, and we want the # Tokio thread pool to be stopped diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index b20417a6ed..014ad18e30 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -83,9 +83,9 @@ from tests.test_utils import SMALL_PNG from tests.unittest import override_config try: - import lxml + import bs4 except ImportError: - lxml = None # type: ignore[assignment] + bs4 = None # type: ignore[assignment] class MediaDomainBlockingTests(unittest.HomeserverTestCase): @@ -194,8 +194,8 @@ class MediaDomainBlockingTests(unittest.HomeserverTestCase): class URLPreviewTests(unittest.HomeserverTestCase): - if not lxml: - skip = "url preview feature requires lxml" + if not bs4: + skip = "url preview feature requires beauitfulsoup4" servlets = [media.register_servlets] hijack_auth = True @@ -314,6 +314,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -380,6 +381,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -410,6 +412,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -446,6 +449,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -487,6 +491,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -501,7 +506,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() self.assertEqual(channel.code, 200) - self.assertEqual(channel.json_body["og:title"], "\u0434\u043a\u0430") + self.assertIn("og:title", channel.json_body) def test_overlong_title(self) -> None: self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")] @@ -522,6 +527,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -555,6 +561,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -664,6 +671,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) @@ -780,6 +788,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Extract Synapse's tcp client client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None # Build a fake remote server to reply with server = AccumulatingProtocol() @@ -833,6 +842,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Respond with the HTML. client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -848,6 +858,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Respond with the photo. client = self.reactor.tcpClients[1][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -882,6 +893,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -924,6 +936,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -964,6 +977,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1026,6 +1040,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1063,6 +1078,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1079,6 +1095,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Ensure a second request is made to the photo URL. client = self.reactor.tcpClients[1][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1123,6 +1140,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1172,6 +1190,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1246,6 +1265,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1269,6 +1289,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Ensure a second request is made to the oEmbed URL. client = self.reactor.tcpClients[1][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1287,6 +1308,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Ensure a third request is made to the photo URL. client = self.reactor.tcpClients[2][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1338,6 +1360,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1379,6 +1402,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1546,6 +1570,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 30f58f1cc3..b3a719d851 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -1037,3 +1037,64 @@ class OwnProfileUnrestrictedTestCase(unittest.HomeserverTestCase): access_token=self.requester_tok, ) self.assertEqual(channel.code, 200, channel.result) + + +class ProfileRatelimitTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + login.register_servlets, + profile.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.owner = self.register_user("owner", "pass") + self.owner_tok = self.login("owner", "pass") + self.other = self.register_user("other", "pass", displayname="Bob") + self.other_tok = self.login("other", "pass") + + @unittest.override_config({"rc_profile": {"per_second": 0.1, "burst_count": 3}}) + def test_ratelimit_authenticated(self) -> None: + """Profile lookups from an authenticated user are rate limited per user, + with the limit shared across the profile endpoints. + """ + channel = self.make_request( + "GET", f"/profile/{self.other}", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", f"/profile/{self.other}/displayname", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", f"/profile/{self.other}/avatar_url", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", f"/profile/{self.other}", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 429, channel.result) + + # Another user is not affected by the first user's limit. + channel = self.make_request( + "GET", f"/profile/{self.owner}", access_token=self.other_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + @unittest.override_config({"rc_profile": {"per_second": 0.1, "burst_count": 3}}) + def test_ratelimit_unauthenticated(self) -> None: + """Unauthenticated profile lookups are rate limited per IP address.""" + for _ in range(3): + channel = self.make_request("GET", f"/profile/{self.other}") + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request("GET", f"/profile/{self.other}") + self.assertEqual(channel.code, 429, channel.result) + + # An authenticated user is not affected by the per-IP limit. + channel = self.make_request( + "GET", f"/profile/{self.other}", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 200, channel.result) diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index 67f78ee24f..2937bb030c 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -22,6 +22,7 @@ import datetime import importlib.resources as importlib_resources import os +from http import HTTPStatus from typing import Any, cast from unittest.mock import AsyncMock @@ -220,6 +221,22 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 200, msg=channel.result) self.assertLessEqual(det_data.items(), channel.json_body.items()) + def test_POST_null_auth(self) -> None: + """A null `auth` value is treated as if it was omitted.""" + request_data = {"username": "kermit", "password": "monkey", "auth": None} + channel = self.make_request(b"POST", self.url, request_data) + + self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, msg=channel.result) + self.assertIn("session", channel.json_body) + + def test_POST_non_object_auth(self) -> None: + """A non-object `auth` value is rejected with a 400.""" + request_data = {"username": "kermit", "password": "monkey", "auth": ["session"]} + channel = self.make_request(b"POST", self.url, request_data) + + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, msg=channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.BAD_JSON) + @override_config({"enable_registration": False}) def test_POST_disabled_registration(self) -> None: request_data = {"username": "kermit", "password": "monkey"} diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index d0df8809c3..c5a01a9a7b 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -45,7 +45,6 @@ from synapse.api.constants import ( RoomTypes, ) from synapse.api.errors import Codes, HttpResponseException -from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService from synapse.events import EventBase, make_event_from_dict from synapse.events.snapshot import EventContext @@ -4804,6 +4803,9 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): self.creator = self.register_user("creator", "test") self.creator_tok = self.login("creator", "test") + self.good_admin = self.register_user("good_admin", "test") + self.good_admin_tok = self.login("good_admin", "test") + self.bad_user_id = self.register_user("bad", "test") self.bad_tok = self.login("bad", "test") @@ -4816,6 +4818,8 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): self.hs.config.experimental.msc4293_enabled = True + self.room_version = self.hs.config.server.default_room_version + def _check_redactions( self, original_events: list[EventBase], @@ -4923,16 +4927,13 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -4947,7 +4948,6 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): self.assertEqual(r[("m.room.member", bad_user)].membership, "join") auth_ids = [ - r[("m.room.create", "")].event_id, r[("m.room.power_levels", "")].event_id, r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -4966,7 +4966,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5017,7 +5017,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5042,16 +5042,13 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5066,7 +5063,6 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): self.assertEqual(r[("m.room.member", bad_user)].membership, "join") auth_ids = [ - r[("m.room.create", "")].event_id, r[("m.room.power_levels", "")].event_id, r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5085,7 +5081,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5126,16 +5122,13 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): # user should be able to join again channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V10, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5153,7 +5146,6 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): self._storage_controllers.state.get_current_state(self.room_id) ) auth_ids = [ - new_state[("m.room.create", "")].event_id, new_state[("m.room.power_levels", "")].event_id, new_state[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5174,7 +5166,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5196,13 +5188,16 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): def test_redaction_flag_ignored_for_user_if_banner_lacks_redaction_power( self, ) -> None: - # change power levels so creator can ban but not redact + # change power levels so room admin can ban but not redact. Do not use the + # room's creator, as the power level they hold is not consistently definable + # after msc4289(creator has infinite power level). self.helper.send_state( self.room_id, "m.room.power_levels", - {"events_default": 0, "redact": 100, "users": {self.creator: 75}}, + {"events_default": 0, "redact": 100, "users": {self.good_admin: 75}}, tok=self.creator_tok, ) + self.helper.join(self.room_id, self.good_admin, tok=self.good_admin_tok) self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) # bad user sends some messages @@ -5224,11 +5219,11 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): } self.helper.change_membership( self.room_id, - self.creator, + self.good_admin, self.bad_user_id, "ban", content, - self.creator_tok, + self.good_admin_tok, ) filter = json.dumps({"types": [EventTypes.Message]}) @@ -5288,16 +5283,13 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5312,7 +5304,6 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): self.assertEqual(r[("m.room.member", bad_user)].membership, "join") auth_ids = [ - r[("m.room.create", "")].event_id, r[("m.room.power_levels", "")].event_id, r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5331,7 +5322,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5382,7 +5373,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5404,16 +5395,13 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5428,7 +5416,6 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): self.assertEqual(r[("m.room.member", bad_user)].membership, "join") auth_ids = [ - r[("m.room.create", "")].event_id, r[("m.room.power_levels", "")].event_id, r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5447,7 +5434,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5483,16 +5470,13 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): # user re-joins after kick channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5510,7 +5494,6 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): self._storage_controllers.state.get_current_state(self.room_id) ) auth_ids = [ - new_state[("m.room.create", "")].event_id, new_state[("m.room.power_levels", "")].event_id, new_state[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5531,7 +5514,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5553,13 +5536,16 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): def test_redaction_flag_ignored_for_user_if_kicker_lacks_redaction_power( self, ) -> None: - # change power levels so creator can kick but not redact + # change power levels so room admin can kick but not redact. Do not use the + # room's creator, as the power level they hold is not consistently definable + # after msc4289(creator has infinite power level). self.helper.send_state( self.room_id, "m.room.power_levels", - {"events_default": 0, "redact": 100, "users": {self.creator: 75}}, + {"events_default": 0, "redact": 100, "users": {self.good_admin: 75}}, tok=self.creator_tok, ) + self.helper.join(self.room_id, self.good_admin, tok=self.good_admin_tok) self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) # bad user sends some messages @@ -5581,11 +5567,11 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): } self.helper.change_membership( self.room_id, - self.creator, + self.good_admin, self.bad_user_id, "kick", content, - self.creator_tok, + self.good_admin_tok, ) filter = json.dumps({"types": [EventTypes.Message]}) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 2da5348492..20fdacdc01 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -45,7 +45,7 @@ from tests.federation.transport.test_knocking import ( KnockingStrippedStateEventHelperMixin, ) from tests.rest.client.test_rooms import make_request_with_cancellation_test -from tests.server import TimedOutException +from tests.server import FakeChannel, TimedOutException from tests.test_utils.event_injection import ( inject_event, persist_message_and_state_event_in_one_batch, @@ -399,6 +399,9 @@ class SyncKnockTestCase(KnockingStrippedStateEventHelperMixin): self.check_knock_room_state_against_room_state( room_state_events, self.expected_room_state ) + # Ensure the events have been stripped + for event in room_state_events: + self.assertNotIn("signatures", event) class SyncCreateEventInPrejoinStateTestCase(unittest.HomeserverTestCase): @@ -502,7 +505,7 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): self.room_id, EventTypes.PowerLevels, { - "users": {self.user_id: 100, self.user2: 100}, + "users": {self.user2: 100}, "users_default": 0, "events": { "m.room.name": 50, @@ -1681,3 +1684,334 @@ class SyncStateAfterArchivedRoomTestCase(unittest.HomeserverTestCase): ], [bob_member_event_id_at_leave], ) + + +class SyncLazyLoadedMembersCacheTestCase(unittest.HomeserverTestCase): + """Tests for the per-device cache of lazy-loaded members already sent. + + A membership left out of a response as already sent is lost for good if + the client never processed the response that carried it. The cache must + only trust a response once the client has synced from its `next_batch`. + """ + + servlets = [ + synapse.rest.admin.register_servlets, + room.register_servlets, + login.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.alice = self.register_user("alice", "password") + self.alice_device = "alice_device" + self.alice_tok = self.login("alice", "password", device_id=self.alice_device) + self.bob = self.register_user("bob", "password") + self.bob_tok = self.login("bob", "password") + + self.get_success( + hs.get_datastores().main.set_features_for_user( + self.alice, {ExperimentalFeature.MSC4222: True} + ) + ) + + # Named room, so that hero calculation doesn't inject current + # membership state into the response and confuse the assertions. + self.room_id = self.helper.create_room_as( + self.alice, tok=self.alice_tok, extra_content={"name": "test room"} + ) + self.helper.join(self.room_id, self.bob, tok=self.bob_tok) + + # Push bob's join out of the initial sync's timeline, so that his + # membership has not been sent to the client when a test starts. + for i in range(10): + self.helper.send(self.room_id, body=f"pad {i}", tok=self.alice_tok) + + def _sync_request( + self, + use_state_after: bool, + since: str | None = None, + *, + timeline_limit: int = 10, + timeline_not_types: list[str] | None = None, + timeout_ms: int = 0, + await_result: bool = True, + ) -> FakeChannel: + """Make a lazy-loading `/sync` request as alice.""" + timeline_filter: JsonDict = {"limit": timeline_limit} + if timeline_not_types is not None: + timeline_filter["not_types"] = timeline_not_types + sync_filter: JsonDict = { + "room": { + "timeline": timeline_filter, + "state": {"lazy_load_members": True}, + } + } + url = f"/sync?filter={json.dumps(sync_filter)}&timeout={timeout_ms}" + if use_state_after: + url += "&org.matrix.msc4222.use_state_after=true" + if since is not None: + url += f"&since={since}" + return self.make_request( + "GET", url, access_token=self.alice_tok, await_result=await_result + ) + + def _sync( + self, + use_state_after: bool, + since: str | None = None, + *, + timeline_limit: int = 10, + timeline_not_types: list[str] | None = None, + ) -> JsonDict: + """Sync as alice and return the response body.""" + channel = self._sync_request( + use_state_after, + since, + timeline_limit=timeline_limit, + timeline_not_types=timeline_not_types, + ) + self.assertEqual(channel.code, 200, channel.result) + return channel.json_body + + def _room(self, response: JsonDict) -> JsonDict: + """Extract the room dict for the test room from a sync response.""" + rooms = response["rooms"].get("join", {}) + self.assertIn(self.room_id, rooms, f"room missing from sync: {response}") + return rooms[self.room_id] + + def _timeline_ids(self, response: JsonDict) -> list[str]: + return [e["event_id"] for e in self._room(response)["timeline"]["events"]] + + def _state_events( + self, response: JsonDict, use_state_after: bool + ) -> list[JsonDict]: + """Extract the list of state events from the test room in a sync response.""" + key = "org.matrix.msc4222.state_after" if use_state_after else "state" + return self._room(response)[key]["events"] + + def _state_ids(self, response: JsonDict, use_state_after: bool) -> list[str]: + """Extract the list of state event IDs from the test room in a sync + response.""" + return [e["event_id"] for e in self._state_events(response, use_state_after)] + + def _members_in_state(self, response: JsonDict, use_state_after: bool) -> list[str]: + """Extract the list of member state keys from the test room in a sync + response.""" + return [ + e["state_key"] + for e in self._state_events(response, use_state_after) + if e["type"] == EventTypes.Member + ] + + @parameterized.expand([("state", False), ("state_after", True)]) + def test_member_sent_once_when_acknowledged( + self, _: str, use_state_after: bool + ) -> None: + """A lazy-loaded member is sent the first time it is needed and left + out once the client has synced past the response that carried it.""" + t0 = self._sync(use_state_after)["next_batch"] + + self.helper.send(self.room_id, body="one", tok=self.bob_tok) + response = self._sync(use_state_after, t0) + self.assertEqual(self._members_in_state(response, use_state_after), [self.bob]) + t1 = response["next_batch"] + + self.helper.send(self.room_id, body="two", tok=self.bob_tok) + response = self._sync(use_state_after, t1) + self.assertEqual(self._members_in_state(response, use_state_after), []) + + @parameterized.expand([("state", False), ("state_after", True)]) + def test_member_not_resent_on_resync_from_token_that_carried_it( + self, _: str, use_state_after: bool + ) -> None: + """A client that resumes from an older token is only sent again the + members carried by the responses after that token. + + A member sent in the response that ended at the resumed-from token was + processed by the client before it saved that token, so it must not be + sent again. matrix-js-sdk saves its sync store every few minutes and + resumes from the saved token on reload, so this is the common case. + """ + t0 = self._sync(use_state_after)["next_batch"] + + # bob's membership is carried by the response that ends at `t1`. + self.helper.send(self.room_id, body="one", tok=self.bob_tok) + response = self._sync(use_state_after, t0) + self.assertEqual(self._members_in_state(response, use_state_after), [self.bob]) + t1 = response["next_batch"] + + # The client processes further responses, then reloads from `t1`. + self.helper.send(self.room_id, body="two", tok=self.bob_tok) + self._sync(use_state_after, t1) + self.helper.send(self.room_id, body="three", tok=self.bob_tok) + response = self._sync(use_state_after, t1) + self.assertEqual( + self._members_in_state(response, use_state_after), + [], + f"bob's membership re-sent on resume from the token that carried " + f"it: {response}", + ) + + @parameterized.expand([("state", False), ("state_after", True)]) + def test_member_resent_on_retry_from_same_token( + self, _: str, use_state_after: bool + ) -> None: + """A client that gives up on a slow request and retries it must get the + lazy-loaded members again. It never processed the first response.""" + t0 = self._sync(use_state_after)["next_batch"] + + self.helper.send(self.room_id, body="hello", tok=self.bob_tok) + for attempt in range(2): + response = self._sync(use_state_after, t0) + self.assertEqual( + self._members_in_state(response, use_state_after), + [self.bob], + f"bob's membership missing on attempt {attempt}: {response}", + ) + + def test_membership_change_resent_on_resync_from_older_token(self) -> None: + """A membership change must be in `state_after` every time the sync + window covers it, even if it has been sent before. + + Clients do not persist every `since` token. matrix-js-sdk saves its + sync store every few minutes, so after a reload it resumes from a token + older than the one the change was first delivered on. A `state_after` + client never applies state from the timeline, so if the change is left + out of `state_after` it keeps the stale membership until it clears its + cache. A kicked user stays shown as joined. + """ + t0 = self._sync(use_state_after=True)["next_batch"] + + self.helper.send(self.room_id, body="hello", tok=self.bob_tok) + t1 = self._sync(use_state_after=True, since=t0)["next_batch"] + + kick = self.helper.change_membership( + self.room_id, self.alice, self.bob, "leave", tok=self.alice_tok + ) + + # The live client sees the kick. + response = self._sync(use_state_after=True, since=t1) + self.assertIn(kick["event_id"], self._state_ids(response, use_state_after=True)) + + # The client reloads from the older persisted token. + response = self._sync(use_state_after=True, since=t0) + self.assertIn(kick["event_id"], self._timeline_ids(response)) + self.assertIn( + kick["event_id"], + self._state_ids(response, use_state_after=True), + f"membership change missing from state_after on re-sync from an " + f"older token: {response}", + ) + + def test_gappy_membership_change_resent_on_resync_from_older_token( + self, + ) -> None: + """The same for `state`. A membership change that falls in the gap of + a limited timeline is only conveyed by `state`, so it must be there + every time the sync window covers it.""" + t0 = self._sync(use_state_after=False, timeline_limit=1)["next_batch"] + + self.helper.send(self.room_id, body="hello", tok=self.bob_tok) + t1 = self._sync(use_state_after=False, since=t0, timeline_limit=1)["next_batch"] + + kick = self.helper.change_membership( + self.room_id, self.alice, self.bob, "leave", tok=self.alice_tok + ) + # A later message, so that the kick falls into the gap of the + # one-event timeline. + self.helper.send(self.room_id, body="after", tok=self.alice_tok) + + # The live client sees the kick in `state`. + response = self._sync(use_state_after=False, since=t1, timeline_limit=1) + self.assertTrue(self._room(response)["timeline"]["limited"]) + self.assertNotIn(kick["event_id"], self._timeline_ids(response)) + self.assertIn( + kick["event_id"], self._state_ids(response, use_state_after=False) + ) + + # The client reloads from the older persisted token. + response = self._sync(use_state_after=False, since=t0, timeline_limit=1) + self.assertNotIn(kick["event_id"], self._timeline_ids(response)) + self.assertIn( + kick["event_id"], + self._state_ids(response, use_state_after=False), + f"membership change missing from state on re-sync from an older " + f"token: {response}", + ) + + @parameterized.expand([("state", False), ("state_after", True)]) + def test_member_not_resent_after_empty_long_poll_wakeup( + self, _: str, use_state_after: bool + ) -> None: + """A long-poll that wakes up with nothing to send keeps waiting. The + response it eventually returns must still leave out the members the + client already has. + + The notifier calls back into the sync handler on every wakeup within + one request. An empty result is not a response the client lost. + """ + not_types = ["m.spurious"] + t0 = self._sync(use_state_after, timeline_not_types=not_types)["next_batch"] + + self.helper.send(self.room_id, body="one", tok=self.bob_tok) + response = self._sync(use_state_after, t0, timeline_not_types=not_types) + self.assertEqual(self._members_in_state(response, use_state_after), [self.bob]) + t1 = response["next_batch"] + + channel = self._sync_request( + use_state_after, + t1, + timeline_not_types=not_types, + timeout_ms=30000, + await_result=False, + ) + + # Wakes the long-poll, but the filter drops the event, so the result is + # empty and the notifier waits again. + self.helper.send_event( + self.room_id, type="m.spurious", content={}, tok=self.bob_tok + ) + self.assertFalse(channel.is_finished()) + + self.helper.send(self.room_id, body="two", tok=self.bob_tok) + channel.await_result() + self.assertEqual(channel.code, 200, channel.result) + response = channel.json_body + self.assertEqual(len(self._timeline_ids(response)), 1) + self.assertEqual(self._members_in_state(response, use_state_after), []) + + @parameterized.expand([("state", False), ("state_after", True)]) + def test_membership_change_sent_when_cache_claims_it_was( + self, _: str, use_state_after: bool + ) -> None: + """A membership change since `since` is sent even when the cache says + the client already has it. + + The cache only decides whether to leave out the memberships fetched for + timeline senders. It can be wrong: a retry of a request can run while + the original is still being built, and the original then marks members + the retry has to send. A change must never depend on it. + """ + t0 = self._sync(use_state_after, timeline_limit=1)["next_batch"] + + kick = self.helper.change_membership( + self.room_id, self.alice, self.bob, "leave", tok=self.alice_tok + ) + # A later message, so that the kick falls into the gap of the + # one-event timeline and only `state` can carry it. + self.helper.send(self.room_id, body="after the kick", tok=self.alice_tok) + + # Claim the kick was already sent in the response that ended at `t0`. + # Syncing from `t0` does not forget this, as the client has acknowledged + # that response. + t0_token = self.get_success( + StreamToken.from_string(self.hs.get_datastores().main, t0) + ) + cache = self.hs.get_sync_handler().get_lazy_loaded_members_cache( + (self.alice, self.alice_device) + ) + cache.mark_sent(self.bob, kick["event_id"], t0_token) + + response = self._sync(use_state_after, t0, timeline_limit=1) + self.assertNotIn(kick["event_id"], self._timeline_ids(response)) + self.assertIn(kick["event_id"], self._state_ids(response, use_state_after)) diff --git a/tests/rest/client/test_sync_sticky_events.py b/tests/rest/client/test_sync_sticky_events.py index 2e52a0ad16..095d5c1022 100644 --- a/tests/rest/client/test_sync_sticky_events.py +++ b/tests/rest/client/test_sync_sticky_events.py @@ -13,6 +13,7 @@ import json import sqlite3 from dataclasses import dataclass +from http import HTTPStatus from unittest.mock import patch from urllib.parse import quote @@ -351,6 +352,125 @@ class SyncStickyEventsTestCase(unittest.HomeserverTestCase): f"Sticky event from ignored user {sticky_event_id} should not be in sticky section", ) + def test_newly_joined_room_gets_all_sticky_events(self) -> None: + """ + Test that joining a room delivers that room's whole sticky event backlog, + even though those sticky events are behind the client's sync position. + + > When the user joins a room, the server MUST include all unexpired sticky + > events for that room in their subsequent sync response. + > — https://github.com/matrix-org/matrix-spec-proposals/blob/d42301b17859d046953563dc7f046edd51789a12/proposals/4354-sticky-events.md#sync-api-changes + """ + # A filter that excludes message events, so that the sticky events come down + # the sticky section rather than the timeline section. + filter_json = json.dumps( + {"room": {"timeline": {"types": ["io.element.example"]}}} + ) + + sticky_event_id = self.helper.send_sticky_event( + self.room_id, + EventTypes.Message, + duration=Duration(minutes=1), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=self.token, + )["event_id"] + + # user2 syncs before joining the room, which moves their sticky events stream + # position past the sticky event above. + user2_id = self.register_user("user2", "pass") + user2_token = self.login(user2_id, "pass") + channel = self.make_request( + "GET", + f"/sync?filter={quote(filter_json)}", + access_token=user2_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + since = channel.json_body["next_batch"] + + # user2 joins the room + self.helper.join(self.room_id, user2_id, tok=user2_token) + + # The next (incremental) sync gives user2 the historical sticky event. + channel = self.make_request( + "GET", + f"/sync?filter={quote(filter_json)}&since={since}", + access_token=user2_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + sticky_events = channel.json_body["rooms"]["join"][self.room_id][ + "msc4354_sticky" + ]["events"] + self.assertEqual( + {sticky_event["event_id"] for sticky_event in sticky_events}, + {sticky_event_id}, + ) + + def test_displayname_change_is_not_newly_joined_room(self) -> None: + """ + Test that a displayname change (`m.room.member` from `join` to `join`) + does not cause the room to be treated as newly joined. + That is to say, we do NOT send down all sticky events again in this case. + """ + # A filter that excludes message events, so that the sticky events come down + # the sticky section rather than the timeline section. + # We include `m.room.member` so that we witness the membership change + # and receive a sync entry for the room. + filter_json = json.dumps({"room": {"timeline": {"types": [EventTypes.Member]}}}) + + sticky_event_id = self.helper.send_sticky_event( + self.room_id, + EventTypes.Message, + duration=Duration(minutes=1), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=self.token, + )["event_id"] + + # user2 joins the room and syncs, getting the sticky event and a `since` + # token past it. + user2_id = self.register_user("user2", "pass") + user2_token = self.login(user2_id, "pass") + self.helper.join(self.room_id, user2_id, tok=user2_token) + channel = self.make_request( + "GET", + f"/sync?filter={quote(filter_json)}", + access_token=user2_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + sticky_events = channel.json_body["rooms"]["join"][self.room_id][ + "msc4354_sticky" + ]["events"] + self.assertEqual( + [sticky_event["event_id"] for sticky_event in sticky_events], + [sticky_event_id], + ) + since = channel.json_body["next_batch"] + + # user2 changes their display name, sending `m.room.member` + self.helper.send_state( + self.room_id, + EventTypes.Member, + { + "membership": "join", + "displayname": "some_per_room_name", + }, + tok=user2_token, + state_key=user2_id, + ) + + # The next (incremental) sync must not transmit the sticky event again. + channel = self.make_request( + "GET", + f"/sync?filter={quote(filter_json)}&since={since}", + access_token=user2_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + room_entry = channel.json_body["rooms"]["join"][self.room_id] + self.assertNotIn( + "msc4354_sticky", + room_entry, + f"Display name change should not resend the room's sticky events: {room_entry}", + ) + def test_history_visibility_bypass_for_sticky_events(self) -> None: """ Test that joined users can see sticky events even when history visibility diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py index 5eaa6f9fb2..0b0fb6c4f7 100644 --- a/tests/rest/client/test_third_party_rules.py +++ b/tests/rest/client/test_third_party_rules.py @@ -576,7 +576,6 @@ class ThirdPartyRulesTestCase(unittest.FederatingHomeserverTestCase): "kick": 50, "redact": 50, "state_default": 50, - "users": {self.user_id: 100}, "users_default": 0, }, tok=self.tok, diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index 6cb85c94c4..cc4073cafc 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -134,7 +134,9 @@ class UpgradeRoomTest(unittest.HomeserverTestCase): "m.room.power_levels", tok=self.creator_token, ) - power_levels["users"][self.other] = 100 + # In the Matrix Spec v1.16, m.room.tombstone events are MUST explicitly be + # higher than `state_default` per MSC4289. + power_levels["users"][self.other] = 150 self.helper.send_state( self.room_id, "m.room.power_levels", @@ -160,7 +162,9 @@ class UpgradeRoomTest(unittest.HomeserverTestCase): "m.room.power_levels", tok=self.creator_token, ) - power_levels["users_default"] = 100 + # In the Matrix Spec v1.16, m.room.tombstone events are MUST explicitly be + # higher than `state_default` per MSC4289. + power_levels["users_default"] = 150 self.helper.send_state( self.room_id, "m.room.power_levels", @@ -220,8 +224,8 @@ class UpgradeRoomTest(unittest.HomeserverTestCase): tok=self.creator_token, ) - # Set creator's power level to the string "100" instead of the integer `100`. - power_levels["users"][self.creator] = "100" + # Set other's power level to the string "100" instead of the integer `100`. + power_levels["users"][self.other] = "100" # Synapse refuses to accept new stringy power level events. Bypass this by # neutering the validation. @@ -252,7 +256,7 @@ class UpgradeRoomTest(unittest.HomeserverTestCase): ) # We should now have an integer power level. - self.assertEqual(new_power_levels["users"][self.creator], 100, new_power_levels) + self.assertEqual(new_power_levels["users"][self.other], 100, new_power_levels) def test_events_field_missing(self) -> None: """Regression test for https://github.com/matrix-org/synapse/issues/16715.""" diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index 1ed6bb145b..6ecff9de6f 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -17,7 +17,6 @@ from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient from synapse.types import JsonDict from synapse.util.clock import Clock @@ -37,24 +36,6 @@ class VersionsTestCase(unittest.HomeserverTestCase): versions.register_servlets, ] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = hs.get_proxied_http_client() - _ = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - def tearDown(self) -> None: # MemoryReactor doesn't trigger the shutdown phases, and we want the # Tokio thread pool to be stopped diff --git a/tests/rest/media/test_url_preview.py b/tests/rest/media/test_url_preview.py index 32e78fc12a..3c7472eced 100644 --- a/tests/rest/media/test_url_preview.py +++ b/tests/rest/media/test_url_preview.py @@ -19,6 +19,7 @@ # # import base64 +import hashlib import json import os import re @@ -35,7 +36,7 @@ from twisted.web.resource import Resource from synapse.config.oembed import OEmbedEndpointConfig from synapse.media.url_previewer import IMAGE_CACHE_EXPIRY_MS from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from synapse.util.stringutils import parse_and_validate_mxc_uri @@ -45,14 +46,14 @@ from tests.test_utils import SMALL_PNG from tests.unittest import override_config try: - import lxml + import bs4 except ImportError: - lxml = None # type: ignore[assignment] + bs4 = None # type: ignore[assignment] class URLPreviewTests(unittest.HomeserverTestCase): - if not lxml: - skip = "url preview feature requires lxml" + if not bs4: + skip = "url preview feature requires beautifulsoup4" hijack_auth = True user_id = "@test:user" @@ -180,6 +181,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -246,6 +248,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -276,6 +279,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -312,6 +316,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -353,6 +358,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -367,7 +373,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() self.assertEqual(channel.code, 200) - self.assertEqual(channel.json_body["og:title"], "\u0434\u043a\u0430") + self.assertIn("og:title", channel.json_body) def test_overlong_title(self) -> None: self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")] @@ -388,6 +394,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -421,6 +428,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -528,6 +536,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) @@ -644,6 +653,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Extract Synapse's tcp client client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None # Build a fake remote server to reply with server = AccumulatingProtocol() @@ -697,6 +707,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Respond with the HTML. client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -712,6 +723,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Respond with the photo. client = self.reactor.tcpClients[1][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -746,6 +758,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -767,6 +780,145 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.assertEqual(channel.code, 200) self.assertNotIn("og:image", channel.json_body) + def _quarantine_hash(self, content: bytes) -> None: + """Mark the sha256 of `content` as belonging to quarantined media.""" + store = self.hs.get_datastores().main + + # `get_is_hash_quarantined` returns False until the sha256 indexes have + # been built, which would make the tests below pass for the wrong reason. + self.wait_for_background_updates() + + self.get_success( + store.store_local_media( + media_id="quarantinedmedia", + media_type="image/png", + time_now_ms=self.clock.time_msec(), + upload_name="quarantined.png", + media_length=len(content), + user_id=UserID.from_string(self.user_id), + sha256=hashlib.sha256(content).hexdigest(), + ) + ) + self.get_success( + store.quarantine_media_by_id("test", "quarantinedmedia", "@admin:test") + ) + + def _count_url_cache_files(self) -> int: + """The number of files stored in the url preview cache.""" + return sum( + len(files) + for _, _, files in os.walk(os.path.join(self.media_store_path, "url_cache")) + ) + + def test_image_quarantined_by_hash(self) -> None: + """An image which matches a quarantined hash should be left out of the + preview entirely, rather than returned as an unfetchable mxc:// URI.""" + self._quarantine_hash(SMALL_PNG) + + self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")] + self.lookups["cdn.matrix.org"] = [(IPv4Address, "10.1.2.4")] + + result = ( + b"""""" + ) + + channel = self.make_request( + "GET", + "/_matrix/media/v3/preview_url?url=http://matrix.org", + shorthand=False, + await_result=False, + ) + self.pump() + + # Respond with the HTML. + client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None + server = AccumulatingProtocol() + server.makeConnection(FakeTransport(client, self.reactor)) + client.makeConnection(FakeTransport(server, self.reactor)) + client.dataReceived( + ( + b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n" + b'Content-Type: text/html; charset="utf8"\r\n\r\n' + ) + % (len(result),) + + result + ) + self.pump() + + # Respond with the photo. + client = self.reactor.tcpClients[1][2].buildProtocol(None) + assert client is not None + server = AccumulatingProtocol() + server.makeConnection(FakeTransport(client, self.reactor)) + client.makeConnection(FakeTransport(server, self.reactor)) + client.dataReceived( + ( + b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n" + b"Content-Type: image/png\r\n\r\n" + ) + % (len(SMALL_PNG),) + + SMALL_PNG + ) + self.pump() + + # The rest of the preview is returned, but without any image. + self.assertEqual(channel.code, 200) + self.assertNotIn("og:image", channel.json_body) + self.assertNotIn("og:image:type", channel.json_body) + self.assertNotIn("og:image:width", channel.json_body) + self.assertNotIn("og:image:height", channel.json_body) + self.assertNotIn("matrix:image:size", channel.json_body) + + # No reference to the image should have been kept: the only url cache + # media is the previewed page itself. + media = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_onecol( + "local_media_repository", + {}, + "url_cache", + desc="test_url_cache_media", + ) + ) + self.assertEqual( + [url for url in media if url is not None], ["http://matrix.org"] + ) + self.assertEqual(self._count_url_cache_files(), 1) + + def test_direct_image_quarantined_by_hash(self) -> None: + """Previewing an image which itself matches a quarantined hash should + 404, in the same way as fetching the quarantined media would.""" + self._quarantine_hash(SMALL_PNG) + + self.lookups["cdn.matrix.org"] = [(IPv4Address, "10.1.2.4")] + + channel = self.make_request( + "GET", + "/_matrix/media/v3/preview_url?url=http://cdn.matrix.org/foo.png", + shorthand=False, + await_result=False, + ) + self.pump() + + client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None + server = AccumulatingProtocol() + server.makeConnection(FakeTransport(client, self.reactor)) + client.makeConnection(FakeTransport(server, self.reactor)) + client.dataReceived( + ( + b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n" + b"Content-Type: image/png\r\n\r\n" + ) + % (len(SMALL_PNG),) + + SMALL_PNG + ) + self.pump() + + self.assertEqual(channel.code, 404) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + self.assertEqual(self._count_url_cache_files(), 0) + @unittest.override_config( {"url_preview_url_blacklist": [{"netloc": "cdn.matrix.org"}]} ) @@ -788,6 +940,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -828,6 +981,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -890,6 +1044,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -927,6 +1082,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -943,6 +1099,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Ensure a second request is made to the photo URL. client = self.reactor.tcpClients[1][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -987,6 +1144,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1036,6 +1194,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1110,6 +1269,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1133,6 +1293,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Ensure a second request is made to the oEmbed URL. client = self.reactor.tcpClients[1][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1151,6 +1312,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Ensure a third request is made to the photo URL. client = self.reactor.tcpClients[2][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1202,6 +1364,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1243,6 +1406,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) @@ -1412,6 +1576,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, self.reactor)) client.makeConnection(FakeTransport(server, self.reactor)) diff --git a/tests/server.py b/tests/server.py index 5297e0b1ac..d0c9b42fdf 100644 --- a/tests/server.py +++ b/tests/server.py @@ -24,6 +24,7 @@ import json import logging import os import os.path +import select import sqlite3 import time import uuid @@ -69,6 +70,7 @@ from twisted.internet.interfaces import ( IPushProducer, IReactorPluggableNameResolver, IReactorTime, + IReadDescriptor, IResolverSimple, ITCPTransport, ITransport, @@ -715,7 +717,7 @@ class ThreadedMemoryReactorClock(MemoryReactorClock): self, host: str, port: int, - factory: ClientFactory, + factory: "ClientFactory[Any]", timeout: float = 30, bindAddress: tuple[str, int] | None = None, ) -> IConnector: @@ -757,6 +759,29 @@ class ThreadedMemoryReactorClock(MemoryReactorClock): # main thread. super().advance(0) + # Now poll anything registered with `addReader`. A real reactor does + # this in its poll loop, but `MemoryReactor` only stores the readers, so + # results from Rust futures (see `TwistedDispatch`) would never reach + # their deferreds. Firing those deferreds can in turn queue more + # callbacks hence the recursive `advance(0)`. + readable = self._poll_readers() + if readable: + for reader in readable: + reader.doRead() + self.advance(0) + + def _poll_readers(self) -> list[IReadDescriptor]: + """The readers registered with `addReader` that have data waiting.""" + readers = {reader.fileno(): reader for reader in self.getReaders()} + if not readers: + return [] + + # Now poll the readers to see if any have data waiting. + poller = select.poll() + for fileno in readers: + poller.register(fileno, select.POLLIN) + return [readers[fileno] for fileno, _event in poller.poll(0)] + def cleanup_test_reactor_system_event_triggers( reactor: ThreadedMemoryReactorClock, @@ -864,9 +889,15 @@ def make_fake_db_pool( pool.runWithConnection = runWithConnection # type: ignore[method-assign] pool.runInteraction = runInteraction # type: ignore[assignment] - # Replace the thread pool with a threadless 'thread' pool + + # First, stop the original thread pool. + pool.threadpool.stop() + # Then, replace it with a threadless 'thread' pool pool.threadpool = ThreadPool(reactor) + + # Start it up. pool.running = True + return pool @@ -1146,6 +1177,7 @@ def connect_client( """ factory = reactor.tcpClients.pop(client_id)[2] client = factory.buildProtocol(None) + assert client is not None server = AccumulatingProtocol() server.makeConnection(FakeTransport(client, reactor)) client.makeConnection(FakeTransport(server, reactor)) diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index a9924e28b1..36208bb82e 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -23,18 +23,21 @@ from typing import ( Collection, Iterable, Mapping, + Sequence, TypeVar, ) import attr +from parameterized import parameterized from twisted.internet import defer from synapse.api.constants import EventTypes, JoinRules, Membership -from synapse.api.room_versions import RoomVersions +from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.event_auth import auth_types_for_event from synapse.events import EventBase from synapse.state.v2 import ( + ConflictCache, _get_auth_chain_difference, _get_power_level_for_sender, lexicographical_topological_sort, @@ -181,6 +184,59 @@ INITIAL_EVENTS = [ INITIAL_EDGES = ["START", "IMZ", "IMC", "IMB", "IJR", "IPOWER", "IMA", "CREATE"] +ZARA_KEY = (EventTypes.Member, ZARA) +TOPIC_KEY = (EventTypes.Topic, "") + + +def _member(node_id: str, sender: str, state_key: str, content: dict) -> FakeEvent: + return FakeEvent( + id=node_id, + sender=sender, + type=EventTypes.Member, + state_key=state_key, + content=content, + ) + + +# Events for the conflict cache tests. All branch off START. +# +# PA is a power levels event that gives Bob PL 50. T1 and T2 are topic changes +# by Bob. Bob has no power under IPOWER, so they only pass auth if PA is in the +# state. +# +# ZJ1 and ZJ2 are Zara re-joining on two branches, and INV1 and INV2 are +# invites she sends to Evelyn on each branch. The invites pull the joins into +# the auth chain difference (see `test_conflict_cache_key_repartitioned`). +CACHE_TEST_CASE_EVENTS = [ + FakeEvent( + id="PA", + sender=ALICE, + type=EventTypes.PowerLevels, + state_key="", + content={"users": {ALICE: 100, BOB: 50}}, + ), + FakeEvent(id="T1", sender=BOB, type=EventTypes.Topic, state_key="", content={}), + FakeEvent(id="T2", sender=BOB, type=EventTypes.Topic, state_key="", content={}), + _member("ZJ1", ZARA, ZARA, MEMBERSHIP_CONTENT_JOIN), + _member("ZJ2", ZARA, ZARA, MEMBERSHIP_CONTENT_JOIN), + _member("INV1", ZARA, EVELYN, {"membership": Membership.INVITE}), + _member("INV2", ZARA, EVELYN, {"membership": Membership.INVITE}), +] + +CACHE_TEST_CASE_EDGES = [ + ["PA", "START"], + ["T1", "START"], + ["T2", "START"], + ["INV1", "ZJ1", "START"], + ["INV2", "ZJ2", "START"], +] + +# Room versions that use v2 and v2.1 state resolution respectively. Both have +# the same auth rules. The difference is that v2.1 starts the iterative auth +# checks from empty state rather than from the unconflicted state. +V2_ROOM = RoomVersions.V11 +V21_ROOM = RoomVersions.HydraV11 + class StateTestCase(unittest.TestCase): def test_ban_vs_pl(self) -> None: @@ -453,21 +509,137 @@ class StateTestCase(unittest.TestCase): self.do_check(events, edges, expected_state_ids) - def do_check( + # Helpers for the conflict cache tests. These use a plain dict as the + # cache, so `len(conflict_cache)` after a call tells us whether it was a + # hit or a miss. + + def _build_cache_scenario(self) -> None: + self.event_map, self.state_at_event = self.build_event_graph( + CACHE_TEST_CASE_EVENTS, CACHE_TEST_CASE_EDGES + ) + + def _state(self, *node_ids: str) -> StateMap[str]: + """The state at START with the given events applied on top.""" + state = dict(self.state_at_event["START"]) + for node_id in node_ids: + event = self.event_map[EventID(node_id, "example.com").to_string()] + state[(event.type, event.state_key)] = event.event_id + return state + + def _resolve_with_cache( + self, + room_version: RoomVersion, + state_sets: Sequence[StateMap[str]], + conflict_cache: ConflictCache, + ) -> StateMap[str]: + return self.successResultOf( + defer.ensureDeferred( + resolve_events_with_store( + FakeClock(), + ROOM_ID, + room_version, + state_sets, + event_map=None, + state_res_store=TestStateResolutionStore(self.event_map), + conflict_cache=conflict_cache, + ) + ) + ) + + @parameterized.expand((V2_ROOM, V21_ROOM)) + def test_conflict_cache_shared_across_unconflicted_state( + self, room_version: RoomVersion + ) -> None: + """Test that two resolutions with the same conflicted set but different + unconflicted state share a cache entry, and that each result still + includes its own unconflicted state. + + The unconflicted state differs on Zara's membership. The topic events + aren't authed against that, so under v2 it isn't part of the cache key. + """ + self._build_cache_scenario() + conflict_cache: dict[bytes, StateMap[str]] = {} + + with_zara = [self._state("PA", "T1"), self._state("PA", "T2")] + without_zara = [ + {key: value for key, value in state.items() if key != ZARA_KEY} + for state in with_zara + ] + + first = self._resolve_with_cache(room_version, with_zara, conflict_cache) + second = self._resolve_with_cache(room_version, without_zara, conflict_cache) + self.assertEqual(len(conflict_cache), 1, "expected a cache hit") + + self.assertIn(ZARA_KEY, first) + self.assertNotIn(ZARA_KEY, second) + + # Everything else agrees. + self.assertEqual({k: v for k, v in first.items() if k != ZARA_KEY}, second) + + def test_conflict_cache_keys_on_base_state(self) -> None: + """Test that under v2 the cache key includes the unconflicted state the + auth checks depend on. Changing the power levels is a cache miss and + gives a different result.""" + self._build_cache_scenario() + + powerless = [self._state("T1"), self._state("T2")] + powered = [self._state("PA", "T1"), self._state("PA", "T2")] + + conflict_cache: dict[bytes, StateMap[str]] = {} + under_ipower = self._resolve_with_cache(V2_ROOM, powerless, conflict_cache) + under_pa = self._resolve_with_cache(V2_ROOM, powered, conflict_cache) + self.assertEqual(len(conflict_cache), 2, "expected a cache miss") + + # Bob's topics fail auth under IPOWER and pass under PA. + self.assertNotIn(TOPIC_KEY, under_ipower) + self.assertIn(TOPIC_KEY, under_pa) + + def test_conflict_cache_key_repartitioned(self) -> None: + """Test that a cached result is correct when the same conflicted set is + split differently between conflicted and unconflicted keys. + + In the first call Zara's membership is unconflicted, but ZJ1 and ZJ2 are + in the auth chain difference (via the invites) and so are in the + conflicted set. In the second call Zara's membership is itself + conflicted. Both calls have the same cache key, so the cached result + must include the resolved Zara membership, even though the first call + overrides it with its unconflicted state. + """ + self._build_cache_scenario() + + agreed = [self._state("INV1"), self._state("INV2")] + conflicting = [self._state("ZJ1", "INV1"), self._state("ZJ2", "INV2")] + + conflict_cache: dict[bytes, StateMap[str]] = {} + agreed_result = self._resolve_with_cache(V21_ROOM, agreed, conflict_cache) + warm = self._resolve_with_cache(V21_ROOM, conflicting, conflict_cache) + self.assertEqual(len(conflict_cache), 1, "expected a cache hit") + + # The first call's unconflicted state takes precedence over the cached + # resolution. + self.assertEqual(agreed_result[ZARA_KEY], self._state()[ZARA_KEY]) + + # The second call gets the winner from the cached resolution, the same + # one it computes from cold. + conflict_cache.clear() + cold = self._resolve_with_cache(V21_ROOM, conflicting, conflict_cache) + self.assertNotEqual(warm[ZARA_KEY], agreed_result[ZARA_KEY]) + self.assertEqual(warm, cold) + + def build_event_graph( self, events: list[FakeEvent], edges: list[list[str]], - expected_state_ids: list[str], - ) -> None: - """Take a list of events and edges and calculate the state of the - graph at END, and asserts it matches `expected_state_ids` + ) -> tuple[dict[str, EventBase], dict[str, StateMap[str]]]: + """Build the graph of `INITIAL_EVENTS` plus `events`. Args: events edges: A list of chains of event edges, e.g. `[[A, B, C]]` are edges A->B and B->C. - expected_state_ids: The expected state at END, (excluding - the keys that haven't changed since START). + + Returns: + The events by event ID, and the state after each node ID. """ # We want to sort the events into topological order for processing. graph: dict[str, set[str]] = {} @@ -539,6 +711,26 @@ class StateTestCase(unittest.TestCase): state_at_event[node_id] = state_after event_map[event_id] = event + return event_map, state_at_event + + def do_check( + self, + events: list[FakeEvent], + edges: list[list[str]], + expected_state_ids: list[str], + ) -> None: + """Take a list of events and edges and calculate the state of the + graph at END, and asserts it matches `expected_state_ids` + + Args: + events + edges: A list of chains of event edges, e.g. + `[[A, B, C]]` are edges A->B and B->C. + expected_state_ids: The expected state at END, (excluding + the keys that haven't changed since START). + """ + event_map, state_at_event = self.build_event_graph(events, edges) + expected_state = {} for node_id in expected_state_ids: # expected_state_ids are node IDs rather than event IDs, diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 577229c119..e67d36cf63 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -105,7 +105,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): # To fix isinstance(...) checks. fake_engine.__class__ = engine.__class__ # type: ignore[assignment] - db = DatabasePool(Mock(), Mock(config=db_config), fake_engine) + db = DatabasePool(Mock(hostname="test"), Mock(config=db_config), fake_engine) db._db_pool = conn_pool self.datastore = SQLBaseStore(db, None, hs) # type: ignore[arg-type] diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py index 054e4a4d0b..3437b6dd77 100644 --- a/tests/storage/test_cleanup_extrems.py +++ b/tests/storage/test_cleanup_extrems.py @@ -283,52 +283,76 @@ class CleanupExtremDummyEventsTestCase(HomeserverTestCase): self.user = UserID.from_string(self.register_user("user1", "password")) self.token1 = self.login("user1", "password") self.requester = create_requester(self.user) - self.room_id, _, _ = self.get_success( - self.room_creator.create_room(self.requester, {"visibility": "public"}) - ) self.event_creator = homeserver.get_event_creation_handler() homeserver.config.consent.user_consent_version = self.CONSENT_VERSION def test_send_dummy_event(self) -> None: - self._create_extremity_rich_graph() + room_id, _, _ = self.get_success( + self.room_creator.create_room(self.requester, {"visibility": "public"}) + ) + self._create_extremity_rich_graph(room_id, self.user) # Pump the reactor repeatedly so that the background updates have a # chance to run. self.pump(20) latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) + self.store.get_latest_event_ids_in_room(room_id) ) self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids)) @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=0) def test_send_dummy_events_when_insufficient_power(self) -> None: - self._create_extremity_rich_graph() - # Criple power levels + # The Plan: + # create a new room with main user(creator). + # join with second user(user2). + # creator changes power levels of user2 so cannot send any events. + # create extremities graph. + # creators leaves - this should remove the user with potentially infinite power + # level. Side effect: this event will capture 10 extremities. + # move time(pump()), try and resolve with dummy events. Should fail. + # creator rejoins. Side effect: this will take tie up 10 more extremities. + # move time(pump()), try and resolve with dummy events again. Should succeed. + + room_id, _, _ = self.get_success( + self.room_creator.create_room(self.requester, {"visibility": "public"}) + ) + + user2 = self.register_user("user2", "password") + token2 = self.login("user2", "password") + self.helper.join(room_id, user2, tok=token2) + + # Cripple power levels for this new user self.helper.send_state( - self.room_id, + room_id, EventTypes.PowerLevels, - body={"users": {str(self.user): -1}}, + body={"users": {user2: -1}}, tok=self.token1, ) + + self._create_extremity_rich_graph(room_id, self.user) + + # The room creator leaves now. They are the only member of the room with a power + # level that can not be tweaked + self.helper.leave(room_id, self.user.to_string(), tok=self.token1) + # Pump the reactor repeatedly so that the background updates have a # chance to run. self.pump(10 * 60) latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) + self.store.get_latest_event_ids_in_room(room_id) ) # Check that the room has not been pruned self.assertTrue(len(latest_event_ids) > 10) - # New user with regular levels - user2 = self.register_user("user2", "password") - token2 = self.login("user2", "password") - self.helper.join(self.room_id, user2, tok=token2) + # Rejoin the room creator + self.helper.join(room_id, self.user.to_string(), tok=self.token1) + # Running the background updates again should clean up the extremities self.pump(10 * 60) latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) + self.store.get_latest_event_ids_in_room(room_id) ) self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids)) @@ -373,18 +397,20 @@ class CleanupExtremDummyEventsTestCase(HomeserverTestCase): 0, ) - def _create_extremity_rich_graph(self) -> None: + def _create_extremity_rich_graph( + self, room_id: str, user_for_sending: UserID + ) -> None: """Helper method to create bushy graph on demand""" - event_id_start = self.create_and_send_event(self.room_id, self.user) + event_id_start = self.create_and_send_event(room_id, user_for_sending) for _ in range(self.EXTREMITIES_COUNT): self.create_and_send_event( - self.room_id, self.user, prev_event_ids=[event_id_start] + room_id, user_for_sending, prev_event_ids=[event_id_start] ) latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) + self.store.get_latest_event_ids_in_room(room_id) ) self.assertEqual(len(latest_event_ids), 50) diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index bfe7049fb2..158426f83a 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -472,6 +472,82 @@ class EventPushActionsStoreTestCase(HomeserverTestCase): self.get_success(self.store._rotate_notifs()) _assert_badge(1) + def test_count_aggregation_badge_recount_is_scoped_per_room(self) -> None: + """ + Regression test: a room whose summary row is out of date must be recounted + from `event_push_actions`, even when another room has an up-to-date summary + for the same thread ID. + + The set of threads a valid summary was found for used to be keyed on the + thread ID alone, so a single room with an up-to-date `main` summary excluded + `main` from the recount in *every* room, dropping those rooms' counts. + """ + user_id, token, other_id, other_token, room_id = self._create_users_and_room() + + stale_room_id = self.helper.create_room_as(user_id, tok=token) + self.helper.join(stale_room_id, other_id, tok=other_token) + + def _send(room: str) -> str: + return self.helper.send_event( + room, + type="m.room.message", + content={"msgtype": "m.text", "body": "msg"}, + tok=other_token, + )["event_id"] + + def _read(room: str, event_id: str) -> None: + self.get_success( + self.store.insert_receipt( + room, + "m.read", + user_id=user_id, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + + def _badge(room: str) -> int: + counts = self.get_success( + self.store.db_pool.runInteraction( + "get-aggregate-unread-counts", + self.store._get_unread_counts_by_room_for_user_txn, + user_id, + ) + ) + return counts.get(room, 0) + + # `room_id` keeps an up-to-date summary throughout, so its `main` thread is + # always one we found a valid summary for. + first = _send(room_id) + _send(room_id) + + stale_first = _send(stale_room_id) + stale_second = _send(stale_room_id) + _send(stale_room_id) + + # Read one event in each room and rotate, so that both summary rows record + # the receipt they were calculated against. + _read(room_id, first) + _read(stale_room_id, stale_first) + self.get_success(self.store._rotate_notifs()) + + self.assertEqual(_badge(room_id), 1) + self.assertEqual(_badge(stale_room_id), 2) + + # A second receipt, which rotation has not processed yet: `stale_room_id`'s + # summary row no longer matches it, so its count has to be recovered from + # `event_push_actions`. + _read(stale_room_id, stale_second) + + self.assertEqual(_badge(room_id), 1) + self.assertEqual(_badge(stale_room_id), 1) + # A new event, not yet rotated, while the summary row is still stale. + _send(stale_room_id) + + self.assertEqual(_badge(room_id), 1) + self.assertEqual(_badge(stale_room_id), 2) + def test_count_aggregation_threads(self) -> None: """ This is essentially the same test as test_count_aggregation, but adds diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py index a42247b498..8eb2befe9c 100644 --- a/tests/storage/test_id_generators.py +++ b/tests/storage/test_id_generators.py @@ -19,8 +19,11 @@ # # +import sqlite3 from unittest import mock +from immutabledict import immutabledict + from twisted.internet.defer import CancelledError, Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor @@ -31,16 +34,23 @@ from synapse.storage.database import ( LoggingDatabaseConnection, LoggingTransaction, ) +from synapse.storage.engines import Sqlite3Engine from synapse.storage.types import Cursor -from synapse.storage.util.id_generators import MultiWriterIdGenerator +from synapse.storage.util.id_generators import ( + MultiWriterIdGenerator, + advance_multiwriter_sharded_token_after_partial_read, + make_multiwriter_sharded_token_bounds_sql, + stream_current_position_gauge, +) from synapse.storage.util.sequence import ( LocalSequenceGenerator, PostgresSequenceGenerator, SequenceGenerator, ) +from synapse.types import MultiWriterStreamToken from synapse.util.clock import Clock -from tests.unittest import HomeserverTestCase +from tests.unittest import HomeserverTestCase, TestCase from tests.utils import USE_POSTGRES_FOR_TESTS @@ -109,6 +119,14 @@ class MultiWriterIdGeneratorBase(HomeserverTestCase): ) return self.instances[instance_name] + def _get_reported_metric_position(self) -> int: + """The position the metric reports for the test stream.""" + + return self.get_prometheus_metric_current_value( + stream_current_position_gauge, + stream_name="test_stream", + ) + def _replicate(self, instance_name: str) -> None: """Similate a replication event for the given instance.""" @@ -229,6 +247,24 @@ class MultiWriterIdGeneratorTestCase(MultiWriterIdGeneratorBase): self.assertEqual(id_gen.get_positions(), {"master": 8}) self.assertEqual(id_gen.get_current_token_for_writer("master"), 8) + def test_current_position_metric(self) -> None: + """The reported position follows `get_current_token`.""" + + self._insert_rows("master", 7) + + id_gen = self._create_id_generator() + + self.assertEqual(self._get_reported_metric_position(), 7) + + async def _get_next_async() -> None: + async with id_gen.get_next(): + pass + + self.get_success(_get_next_async()) + + self.assertEqual(id_gen.get_current_token(), 8) + self.assertEqual(self._get_reported_metric_position(), 8) + def test_cancelled_enter_does_not_wedge_position(self) -> None: """Reproduces presence getting stuck. @@ -886,3 +922,333 @@ class MultiTableMultiWriterIdGeneratorTestCase(MultiWriterIdGeneratorBase): self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 7) self.assertEqual(second_id_gen.get_current_token_for_writer("second"), 7) self.assertEqual(second_id_gen.get_persisted_upto_position(), 7) + + +class ShardedTokenHelpersPureTestCase(TestCase): + """ + Non-database tests for the helpers for reading multi-writer streams. + """ + + def test_bounds_sql_documented_example(self) -> None: + """ + Tests that the example in the docstring is what we actually generate. + """ + clause, values = make_multiwriter_sharded_token_bounds_sql( + Sqlite3Engine({}), + stream_id_column="se.stream_id", + instance_name_column="se.instance_name", + from_token_exclusive=MultiWriterStreamToken( + stream=5, instance_map=immutabledict({"worker1": 8}) + ), + to_token_inclusive=MultiWriterStreamToken( + stream=10, instance_map=immutabledict({"worker2": 14}) + ), + ) + self.assertEqualNormalisingWhitespace( + clause, + """ + ( + ? < se.stream_id + AND se.stream_id <= ? + AND NOT (se.instance_name IS ? AND se.stream_id <= ?) + AND ( + se.stream_id <= ? + OR (se.instance_name IS ? AND se.stream_id <= ?) + ) + ) + """, + ) + self.assertEqual(list(values), [5, 14, "worker1", 8, 10, "worker2", 14]) + + def test_token_after_partial_read_does_not_go_backwards(self) -> None: + """ + Tests that advancing the token after a partial read doesn't let it go + backwards. + + This is relevant because Synapse workers don't always advance their current + position at the same time. + """ + # As a scenario: the client has already read up to a baseline position of 60, + # but this reader worker has only caught up to 10. + # It can nonetheless see that worker2 has reached 70. + from_token = MultiWriterStreamToken(stream=60) + to_token = MultiWriterStreamToken( + stream=10, instance_map=immutabledict({"worker2": 70}) + ) + + resume_token = advance_multiwriter_sharded_token_after_partial_read( + from_token_exclusive=from_token, + to_token_inclusive=to_token, + last_read_stream_id=64, + ) + + # worker2 advances to what we read; everyone else stays where they were. + self.assertEqual( + resume_token, + MultiWriterStreamToken( + stream=60, instance_map=immutabledict({"worker2": 64}) + ), + ) + self.assertTrue( + from_token.is_before_or_eq(resume_token), + f"Expected {from_token} <= {resume_token}", + ) + + +class ShardedTokenHelpersDatabaseTestCase(TestCase): + """Tests for the helpers that read a range of a multi-writer stream. + + These don't need a homeserver: they only exercise the SQL that + `make_multiwriter_sharded_token_bounds_sql` builds, against a throwaway + SQLite table. + + NOTE: `ShardedTokenHelpersDatabaseNullTestCase` inherits all these tests + with a separate set of tables. + """ + + ROWS: list[tuple[int, str | None]] = [ + # (stream_id, instance_name) + (6, "worker1"), + (7, "worker1"), + (8, "worker2"), + (9, "worker1"), + (10, "worker3"), + (11, "worker1"), + (12, "worker2"), + (13, "worker3"), + (14, "worker2"), + ] + """ + (stream_id, instance_name) rows to insert into the example stream table. + """ + + SHARDED_TOKEN_RANGES = [ + ( + MultiWriterStreamToken(stream=5), + MultiWriterStreamToken(stream=14), + ), + ( + MultiWriterStreamToken(stream=5), + MultiWriterStreamToken( + stream=10, instance_map=immutabledict({"worker2": 14}) + ), + ), + ( + MultiWriterStreamToken( + stream=5, instance_map=immutabledict({"worker1": 9}) + ), + MultiWriterStreamToken( + stream=10, instance_map=immutabledict({"worker2": 14}) + ), + ), + ( + MultiWriterStreamToken( + stream=5, instance_map=immutabledict({"worker1": 7}) + ), + MultiWriterStreamToken( + stream=8, instance_map=immutabledict({"worker1": 11, "worker3": 13}) + ), + ), + ( + MultiWriterStreamToken( + stream=5, instance_map=immutabledict({"worker1": 9, "worker2": 8}) + ), + MultiWriterStreamToken( + stream=10, + instance_map=immutabledict( + {"worker1": 11, "worker2": 14, "worker3": 13} + ), + ), + ), + ] + """(from, to) token pairs to exercise the bounds against `ROWS`.""" + + def setUp(self) -> None: + # Just used for detecting SQL language support, + # doesn't actually connect... + self.database_engine = Sqlite3Engine({}) + self.conn = sqlite3.connect(":memory:") + self.addCleanup(self.conn.close) + self.conn.execute( + "CREATE TABLE streamtable (stream_id INTEGER PRIMARY KEY, instance_name TEXT)" + ) + self.conn.executemany("INSERT INTO streamtable VALUES (?, ?)", self.ROWS) + + def _select( + self, + from_token: MultiWriterStreamToken, + to_token: MultiWriterStreamToken, + limit: int | None = None, + ) -> list[tuple[int, str]]: + """ + Helper that selects the rows within the given bounds, in stream order, + using `make_multiwriter_sharded_token_bounds_sql`. + + Bounds: from_token < ... <= to_token + """ + clause, values = make_multiwriter_sharded_token_bounds_sql( + self.database_engine, + stream_id_column="streamtable.stream_id", + instance_name_column="streamtable.instance_name", + from_token_exclusive=from_token, + to_token_inclusive=to_token, + ) + + limit_clause = "" + if limit is not None: + limit_clause = f"LIMIT {limit}" + + return list( + self.conn.execute( + f""" + SELECT stream_id, instance_name + FROM streamtable + WHERE {clause} + ORDER BY stream_id ASC + {limit_clause} + """, + values, + ) + ) + + def _expected( + self, + from_token: MultiWriterStreamToken, + to_token: MultiWriterStreamToken, + ) -> list[tuple[int, str | None]]: + """ + Helper that returns the rows that ought to be within the given bounds. + It uses `MultiWriterStreamToken.is_stream_position_in_range` as the 'ground truth'. + + Bounds: from_token < ... <= to_token + """ + + return [ + (stream_id, instance_name) + for stream_id, instance_name in self.ROWS + if MultiWriterStreamToken.is_stream_position_in_range( + from_token, to_token, instance_name, stream_id + ) + ] + + def test_bounds_sql_selects_expected_rows(self) -> None: + """ + Tests that the `make_multiwriter_sharded_token_bounds_sql` selects the correct rows, + using `MultiWriterStreamToken.is_stream_position_in_range` as the 'ground truth'. + """ + for from_token, to_token in self.SHARDED_TOKEN_RANGES: + with self.subTest(from_token=str(from_token), to_token=str(to_token)): + self.assertEqual( + self._select(from_token, to_token), + self._expected(from_token, to_token), + ) + + def test_token_after_partial_read(self) -> None: + """ + Tests that when advancing the token after a partial read, the subsequent read returns all the remaining rows + but no more (especially not duplicates). + """ + for from_token, to_token in self.SHARDED_TOKEN_RANGES: + all_rows = self._expected(from_token, to_token) + for limit in range(1, len(all_rows) + 1): + with self.subTest( + from_token=str(from_token), to_token=str(to_token), limit=limit + ): + read = self._select(from_token, to_token, limit=limit) + resume_token = advance_multiwriter_sharded_token_after_partial_read( + from_token_exclusive=from_token, + to_token_inclusive=to_token, + last_read_stream_id=read[-1][0], + ) + + # The resumption point must be somewhere within the range we + # were asked to read. + self.assertTrue( + from_token.is_before_or_eq(resume_token), + f"Expected {from_token} <= {resume_token}", + ) + self.assertTrue( + resume_token.is_before_or_eq(to_token), + f"Expected {resume_token} <= {to_token}", + ) + + self.assertEqual( + read, + self._expected(from_token, resume_token), + f"The rows we read between {from_token} < ... <= {resume_token} seem wrong.", + ) + + self.assertEqual( + read + self._select(resume_token, to_token), + all_rows, + f"We did a limited read of {limit} rows within {from_token} < ... <= {to_token}, " + f"giving us an advanced token {resume_token}. " + f"We then read {resume_token} < ... <= {to_token} and expected to get " + "all the rows within range (in order, no duplicates), but didn't.", + ) + + +class ShardedTokenHelpersDatabaseNullTestCase(ShardedTokenHelpersDatabaseTestCase): + """ + Variant of `ShardedTokenHelpersDatabaseTestCase` where there are rows with + `instance_name` being `NULL`, corresponding to rows that existed before the stream was sharded. + """ + + ROWS = [ + # (stream_id, instance_name) + (4, None), + (5, None), + (6, None), + (7, None), + (8, "worker2"), + (9, "worker1"), + (10, "worker3"), + (11, "worker1"), + (12, "worker2"), + (13, "worker3"), + (14, "worker2"), + ] + """ + (stream_id, instance_name) rows to insert into the example stream table. + """ + + SHARDED_TOKEN_RANGES = [ + ( + MultiWriterStreamToken(stream=5), + MultiWriterStreamToken(stream=14), + ), + ( + MultiWriterStreamToken(stream=5), + MultiWriterStreamToken( + stream=10, instance_map=immutabledict({"worker2": 14}) + ), + ), + ( + MultiWriterStreamToken( + stream=5, instance_map=immutabledict({"worker1": 9}) + ), + MultiWriterStreamToken( + stream=10, instance_map=immutabledict({"worker2": 14}) + ), + ), + ( + MultiWriterStreamToken( + stream=5, instance_map=immutabledict({"worker1": 7}) + ), + MultiWriterStreamToken( + stream=8, instance_map=immutabledict({"worker1": 11, "worker3": 13}) + ), + ), + ( + MultiWriterStreamToken( + stream=5, instance_map=immutabledict({"worker1": 9, "worker2": 8}) + ), + MultiWriterStreamToken( + stream=10, + instance_map=immutabledict( + {"worker1": 11, "worker2": 14, "worker3": 13} + ), + ), + ), + ] + """(from, to) token pairs to exercise the bounds against `ROWS`.""" diff --git a/tests/storage/test_receipts.py b/tests/storage/test_receipts.py index 27875dcebb..3413978c18 100644 --- a/tests/storage/test_receipts.py +++ b/tests/storage/test_receipts.py @@ -21,11 +21,13 @@ from typing import Collection +from immutabledict import immutabledict + from twisted.internet.testing import MemoryReactor from synapse.api.constants import ReceiptTypes from synapse.server import HomeServer -from synapse.types import UserID, create_requester +from synapse.types import MultiWriterStreamToken, UserID, create_requester from synapse.util.clock import Clock from tests.test_utils.event_injection import create_event @@ -307,3 +309,67 @@ class ReceiptTestCase(HomeserverTestCase): [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE], room_id=self.room_id2 ) self.assertEqual(res, event2_1_id) + + +class MultiWriterReceiptPagingTestCase(HomeserverTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.store = homeserver.get_datastores().main + + def test_reached_token_does_not_cover_receipts_from_lagging_writer(self) -> None: + """ + Writers `rw1` and `rw2` alternate stream IDs 1001 to 1008. The paging + worker has heard from `rw2` up to 1008 but from `rw1` only up to 1000. + + With a limit of 4 the SQL fetches 1001 to 1004, then `rw1`'s 1001 and + 1003 are filtered out as beyond its known position. The returned token + must not claim `rw1` was handled past 1000, or those receipts are + skipped once `rw1` catches up. + """ + self.get_success( + self.store.db_pool.simple_insert_many( + table="receipts_linearized", + keys=( + "stream_id", + "instance_name", + "room_id", + "receipt_type", + "user_id", + "event_id", + "data", + ), + values=[ + ( + stream_id, + "rw1" if stream_id % 2 else "rw2", + f"!room_{stream_id}:test", + ReceiptTypes.READ, + "@user:test", + f"$event_{stream_id}", + "{}", + ) + for stream_id in range(1001, 1009) + ], + desc="insert_receipts", + ) + ) + + from_key = MultiWriterStreamToken(stream=1000) + to_key = MultiWriterStreamToken( + stream=1000, instance_map=immutabledict({"rw2": 1008}) + ) + + rooms_to_events, reached_token = self.get_success( + self.store.get_linearized_receipts_for_all_rooms( + to_key=to_key, from_key=from_key, limit=4 + ) + ) + + # `rw1`'s receipts are behind its position in `to_key`, so none of them + # are returned... + self.assertFalse({"!room_1001:test", "!room_1003:test"} & set(rooms_to_events)) + + # ...and therefore the returned token must not claim `rw1` has been + # handled beyond 1000. + self.assertLessEqual(reached_token.get_stream_pos_for_instance("rw1"), 1000) diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py index 6c3506f348..46019c2e7b 100644 --- a/tests/storage/test_state.py +++ b/tests/storage/test_state.py @@ -30,6 +30,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions from synapse.events import EventBase +from synapse.logging.context import LoggingContext from synapse.server import HomeServer from synapse.types import JsonDict, RoomID, StateMap, UserID from synapse.types.state import StateFilter @@ -644,6 +645,113 @@ class StateStoreTestCase(HomeserverTestCase): ) self.assertEqual(context.state_group_before_event, groups[0][0]) + def test_get_partial_filtered_current_state_ids_concrete(self) -> None: + """A filter with no wildcards is served from the per-key cache, and the + absence of a key is cached too.""" + room_id = self.room.to_string() + + create = self.inject_state_event( + self.room, self.u_alice, EventTypes.Create, "", {} + ) + name = self.inject_state_event( + self.room, self.u_alice, EventTypes.Name, "", {"name": "test room"} + ) + + state_filter = StateFilter.from_types( + [(EventTypes.Create, ""), (EventTypes.Name, ""), (EventTypes.Topic, "")] + ) + + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + + self.assertEqual( + dict(state), + { + (EventTypes.Create, ""): create.event_id, + (EventTypes.Name, ""): name.event_id, + }, + ) + + # The room has no topic, and that fact is cached, so asking again does + # not go back to the database. + sentinel = object() + cache = self.store._get_current_state_event_id.cache + self.assertIsNone( + cache.get_immediate((room_id, (EventTypes.Topic, "")), sentinel) + ) + self.assertEqual( + cache.get_immediate((room_id, (EventTypes.Name, "")), sentinel), + name.event_id, + ) + + def test_get_partial_filtered_current_state_ids_invalidation(self) -> None: + """Persisting a new state event invalidates the cached entries for the + room.""" + room_id = self.room.to_string() + + self.inject_state_event(self.room, self.u_alice, EventTypes.Create, "", {}) + + state_filter = StateFilter.from_types([(EventTypes.Name, "")]) + + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual(dict(state), {}) + + name = self.inject_state_event( + self.room, self.u_alice, EventTypes.Name, "", {"name": "test room"} + ) + + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual(dict(state), {(EventTypes.Name, ""): name.event_id}) + + name2 = self.inject_state_event( + self.room, self.u_alice, EventTypes.Name, "", {"name": "renamed"} + ) + + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual(dict(state), {(EventTypes.Name, ""): name2.event_id}) + + def test_get_partial_filtered_current_state_ids_uses_full_cache(self) -> None: + """Test that fetching a single state key from a room with a full cache + hits the full cache and does not go to the database.""" + + room_id = self.room.to_string() + + create = self.inject_state_event( + self.room, self.u_alice, EventTypes.Create, "", {} + ) + name = self.inject_state_event( + self.room, self.u_alice, EventTypes.Name, "", {"name": "test room"} + ) + + # prime the full cache + state_filter = StateFilter.all() + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual( + dict(state), + { + (EventTypes.Create, ""): create.event_id, + (EventTypes.Name, ""): name.event_id, + }, + ) + + # now fetch a single key and check that it hits the full cache + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: + state_filter = StateFilter.from_types([(EventTypes.Name, "")]) + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual(dict(state), {(EventTypes.Name, ""): name.event_id}) + self.assertEqual(ctx.get_resource_usage().db_txn_count, 0) + class CurrentStateDeltaStreamTestCase(HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: diff --git a/tests/storage/test_sticky_events.py b/tests/storage/test_sticky_events.py index 6335553f0a..c73da479f4 100644 --- a/tests/storage/test_sticky_events.py +++ b/tests/storage/test_sticky_events.py @@ -31,7 +31,7 @@ from synapse.util.clock import Clock from synapse.util.duration import Duration from tests import unittest -from tests.test_utils.event_injection import inject_event +from tests.test_utils.event_injection import create_event, inject_event from tests.utils import USE_POSTGRES_FOR_TESTS @@ -283,6 +283,76 @@ class StickyEventsTestCase(unittest.HomeserverTestCase): self.assertEqual(len(updates), 1) self.assertEqual(updates[0].event_id, event_non_outlier.event_id) + def test_redacted_before_persisted_not_tracked(self) -> None: + """ + Tests that a sticky event which was redacted before we persisted it + (i.e. we learned of its redaction first, as can happen over federation) + is not sent down to clients over sync and is not added to the `sticky_events` table. + """ + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + self.helper.join(self.room_id, user2_id, tok=user2_tok) + + persist_controller = self.hs.get_storage_controllers().persistence + assert persist_controller is not None + + # Create the sticky event, but do not persist it yet + sticky_event, sticky_event_context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + sender=user2_id, + type=EventTypes.Message, + content={"body": "sticky", "msgtype": "m.text"}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + # Create the redaction of the sticky event and persist it first, + # as if it had arrived over federation before the sticky event. + redaction_event, redaction_event_context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + sender=user2_id, + type=EventTypes.Redaction, + content={ + "reason": "nothing here but us trees", + "redacts": sticky_event.event_id, + }, + ) + ) + self.get_success( + persist_controller.persist_event(redaction_event, redaction_event_context) + ) + + # Now the sticky event arrives over federation and is persisted *after* the redaction + self.get_success( + persist_controller.persist_event(sticky_event, sticky_event_context) + ) + + # The event should have been persisted in its redacted form. + event = self.get_success(self.store.get_event(sticky_event.event_id)) + self.assertEqual(event.event_id, sticky_event.event_id) + self.assertEqual(event.internal_metadata.redacted_by, redaction_event.event_id) + self.assertEqual(event.content, {}) + self.assertIsNone(event.sticky_duration()) + + # Since it is redacted, it must not have been added to the sticky_events + # table... + sticky_events = self.get_success( + self.store.db_pool.simple_select_list( + table="sticky_events", keyvalues=None, retcols=("event_id",) + ) + ) + self.assertEqual(sticky_events, []) + + # ...nor shown to clients down sync. + self.assertEqual(self._get_visible_sticky_event_ids(), set()) + def test_soft_failed_events_are_tracked(self) -> None: """ Tests that sticky events marked as soft_failed ARE inserted @@ -485,7 +555,7 @@ class StickyEventsTestCase(unittest.HomeserverTestCase): self.helper.send_state( self.room_id, EventTypes.PowerLevels, - body={"users": {self.user_id: 100, user2_id: -1}, "events_default": 0}, + body={"users": {user2_id: -1}, "events_default": 0}, tok=self.token, ) @@ -517,7 +587,7 @@ class StickyEventsTestCase(unittest.HomeserverTestCase): self.helper.send_state( self.room_id, EventTypes.PowerLevels, - body={"users": {self.user_id: 100, user2_id: 0}, "events_default": 0}, + body={"users": {user2_id: 0}, "events_default": 0}, tok=self.token, ) diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 4ece16e9a0..c5e880d7b2 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -35,7 +35,6 @@ from synapse.api.constants import ( ) from synapse.api.filtering import Filter from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.rest import admin from synapse.rest.client import login, room @@ -50,6 +49,7 @@ from synapse.types import ( ) from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_pdu_event from tests.test_utils.event_injection import create_event from tests.unittest import FederatingHomeserverTestCase, HomeserverTestCase @@ -690,7 +690,6 @@ class GetCurrentStateDeltaMembershipChangesForUserTestCase(HomeserverTestCase): extra_content={ "power_level_content_override": { "users": { - user2_id: 100, # Allow user1 to send state in the room user1_id: 100, } @@ -861,7 +860,6 @@ class GetCurrentStateDeltaMembershipChangesForUserTestCase(HomeserverTestCase): extra_content={ "power_level_content_override": { "users": { - user2_id: 100, # Allow user1 to send state in the room user1_id: 100, } @@ -1350,6 +1348,7 @@ class GetCurrentStateDeltaMembershipChangesForUserFederationTestCase( self.store = self.hs.get_datastores().main self.event_sources = hs.get_event_sources() self.room_member_handler = hs.get_room_member_handler() + self.room_version = self.hs.config.server.default_room_version def test_remote_join(self) -> None: """ @@ -1361,98 +1360,96 @@ class GetCurrentStateDeltaMembershipChangesForUserFederationTestCase( before_join_token = self.event_sources.get_current_token() - intially_unjoined_room_id = f"!example:{self.OTHER_SERVER_NAME}" - # Remotely join a room on another homeserver. # # To do this we have to mock the responses from the remote homeserver. We also # patch out a bunch of event checks on our end. - create_event_source = { + creation_event_source = { "auth_events": [], "content": { - "creator": f"@creator:{self.OTHER_SERVER_NAME}", - "room_version": self.hs.config.server.default_room_version.identifier, + "room_version": self.room_version.identifier, }, - "depth": 0, + "depth": 1, "origin_server_ts": 0, "prev_events": [], - "room_id": intially_unjoined_room_id, "sender": f"@creator:{self.OTHER_SERVER_NAME}", "state_key": "", "type": EventTypes.Create, } + self.add_hashes_and_signatures_from_other_server( - create_event_source, - self.hs.config.server.default_room_version, + creation_event_source, + self.room_version, ) - create_event = make_event_from_dict( - create_event_source, - self.hs.config.server.default_room_version, - {}, - None, + # This handles FrozenEvent* creation for us, and will create the room_id if this + # is a msc4291 room + creation_event = make_test_pdu_event( + creation_event_source, + self.room_version, ) + + initially_unjoined_room_id = creation_event.room_id + creator_join_event_source = { - "auth_events": [create_event.event_id], + "auth_events": [], "content": { "membership": "join", }, - "depth": 1, + "depth": 2, "origin_server_ts": 1, "prev_events": [], - "room_id": intially_unjoined_room_id, + "room_id": initially_unjoined_room_id, "sender": f"@creator:{self.OTHER_SERVER_NAME}", "state_key": f"@creator:{self.OTHER_SERVER_NAME}", "type": EventTypes.Member, } + self.add_hashes_and_signatures_from_other_server( creator_join_event_source, - self.hs.config.server.default_room_version, + self.room_version, ) - creator_join_event = make_event_from_dict( + creator_join_event = make_test_pdu_event( creator_join_event_source, - self.hs.config.server.default_room_version, - {}, - None, + self.room_version, ) # Our local user is going to remote join the room join_event_source = { - "auth_events": [create_event.event_id], + "auth_events": [], "content": {"membership": "join"}, - "depth": 1, + "depth": 3, "origin_server_ts": 100, "prev_events": [creator_join_event.event_id], "sender": user1_id, "state_key": user1_id, - "room_id": intially_unjoined_room_id, + "room_id": initially_unjoined_room_id, "type": EventTypes.Member, } + add_hashes_and_signatures( - self.hs.config.server.default_room_version, + self.room_version, join_event_source, self.hs.hostname, self.hs.signing_key, ) - join_event = make_event_from_dict( + join_event = make_test_pdu_event( join_event_source, - self.hs.config.server.default_room_version, - {}, - None, + self.room_version, ) mock_make_membership_event = AsyncMock( return_value=( self.OTHER_SERVER_NAME, join_event, - self.hs.config.server.default_room_version, + self.room_version, ) ) mock_send_join = AsyncMock( return_value=SendJoinResult( join_event, self.OTHER_SERVER_NAME, - state=[create_event, creator_join_event], - auth_chain=[create_event, creator_join_event], + state=[creation_event, creator_join_event], + auth_chain=[creation_event, creator_join_event], partial_state=False, servers_in_room=frozenset(), state_dag=None, @@ -1483,7 +1480,7 @@ class GetCurrentStateDeltaMembershipChangesForUserFederationTestCase( self.room_member_handler.update_membership( requester=create_requester(user1_id), target=UserID.from_string(user1_id), - room_id=intially_unjoined_room_id, + room_id=initially_unjoined_room_id, action=Membership.JOIN, remote_room_hosts=[self.OTHER_SERVER_NAME], ) @@ -1520,7 +1517,7 @@ class GetCurrentStateDeltaMembershipChangesForUserFederationTestCase( membership_changes, [ CurrentStateDeltaMembership( - room_id=intially_unjoined_room_id, + room_id=initially_unjoined_room_id, event_id=join_event.event_id, event_pos=join_pos, membership="join", diff --git a/tests/synapse_rust/test_http_client.py b/tests/synapse_rust/test_http_client.py index 845fe2b503..330fbeeba6 100644 --- a/tests/synapse_rust/test_http_client.py +++ b/tests/synapse_rust/test_http_client.py @@ -86,25 +86,13 @@ class StubServer(HTTPServer): class HttpClientTestCase(HomeserverTestCase): - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self._http_client = hs.get_proxied_http_client() self._rust_http_client = HttpClient( - reactor=hs.get_reactor(), + runtime=hs.get_rust_runtime(), user_agent=self._http_client.user_agent.decode("utf8"), ) - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.server = StubServer() def tearDown(self) -> None: diff --git a/tests/synapse_rust/test_logcontext.py b/tests/synapse_rust/test_logcontext.py new file mode 100644 index 0000000000..5e1cefaff9 --- /dev/null +++ b/tests/synapse_rust/test_logcontext.py @@ -0,0 +1,293 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +"""Tests for logcontext attribution across the Python/Rust boundary. + +The current logcontext is stored on the Rust side +(`synapse.synapse_rust.logcontext`), visible +from both Python (reactor/threadpool threads) and Rust (tokio tasks). These +tests exercise the two guarantees that gives us: + +1. Log records emitted from Rust while a task is being polled (e.g. reqwest + connecting) are attributed to the logcontext that was current when Python + called into Rust — not the sentinel. +2. When Rust calls back into Python (`run_python_awaitable`, as the Rust + `/versions` handler does for its per-user feature DB lookup), the Python code + runs in that same logcontext, so its DB-transaction accounting lands on the + right request. +""" + +import logging +import time +from typing import Callable + +from twisted.internet.testing import MemoryReactor + +from synapse.logging.context import ( + LoggingContext, + LoggingContextFilter, + PreserveLoggingContext, + _Sentinel, + current_context, + run_in_background, +) +from synapse.rest import admin +from synapse.rest.client import login +from synapse.server import HomeServer +from synapse.synapse_rust import reset_logging_config +from synapse.synapse_rust.http_client import HttpClient +from synapse.util.clock import Clock + +from tests.unittest import HomeserverTestCase + +logger = logging.getLogger(__name__) + +# Log-target roots that Rust code emits under while running on tokio worker +# threads: the reqwest dependency stack, plus "synapse"/"synapse_rust" because +# the Rust crate is itself named `synapse` (see rust/Cargo.toml). Anything +# emitted under these while a task is being polled should be attributed to the +# caller's logcontext, never the sentinel. +# +# NB: bare "synapse" also matches every *Python* `synapse.*` record, so the +# attribution assertion below implicitly relies on nothing else logging during +# the pump (MemoryReactor with `advance(0)`, so no timed background work fires). +# If this test starts flaking on unrelated records, tighten this filter rather +# than weakening the assertion. +_RUST_LOGGER_ROOTS = frozenset( + {"reqwest", "hyper", "hyper_util", "h2", "rustls", "synapse_rust", "synapse"} +) + + +class RustLogContextTestCase(HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` + # below. Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` + # callbacks if it's already running and we rely on that to start the Tokio + # thread pool in Rust. + self._http_client = hs.get_proxied_http_client() + self._rust_http_client = HttpClient( + runtime=hs.get_rust_runtime(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the Tokio + # thread pool to be stopped. + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.user_id = self.register_user("user1", "pass") + + def _check_current_logcontext(self, expected: str) -> None: + context = current_context() + assert isinstance(context, (LoggingContext, _Sentinel)), context + self.assertEqual(str(context), expected, f"expected {expected}, saw {context}") + + def _run_in_logcontext_and_pump( + self, name: str, body: Callable[[dict[str, object]], None] + ) -> dict[str, object]: + """Run `body` inside a fresh `LoggingContext(name)`, then pump the + reactor (yielding to the Tokio pool) until it sets `result["done"]`. + + Returns the `result` dict that `body` populated. Asserts that the + caller's logcontext is intact afterwards and that we end back in the + sentinel. + """ + self._check_current_logcontext("sentinel") + result: dict[str, object] = {} + + with LoggingContext(name=name, server_name="test_server"): + body(result) + + with PreserveLoggingContext(): + # Generous upper bound (the work is a real HTTP round-trip or DB + # hop on a possibly-loaded CI box); the loop exits early via + # `result["done"]`, and we fail below if it never gets set. + for _ in range(50000): + if result.get("done"): + break + # Let the Tokio worker threads make progress... + time.sleep(0) + # ...and run anything they scheduled back on the reactor. + self.reactor.advance(0) + + # The caller's logcontext must be intact after firing off the work. + self._check_current_logcontext(name) + + # ...and we must not have leaked it into the reactor. + self._check_current_logcontext("sentinel") + + self.assertTrue( + result.get("done"), + "work never finished; the test probably didn't pump long enough", + ) + return result + + def test_rust_log_records_attributed_to_caller_logcontext(self) -> None: + """A log record emitted from Rust on a tokio thread (reqwest connecting) + is attributed to the caller's logcontext via `LoggingContextFilter`, not + the sentinel.""" + records: list[tuple[str, object]] = [] + + class CapturingHandler(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + records.append((record.name, getattr(record, "request", ""))) + + handler = CapturingHandler() + # The global filter is what copies `str(current_context())` onto the + # record as `record.request`; attach it so we observe what Synapse would. + handler.addFilter(LoggingContextFilter()) + + root = logging.getLogger() + root.addHandler(handler) + + # Turn up the Rust-side loggers so reqwest actually emits, and refresh + # pyo3-log's cached levels so it forwards them. + saved_levels = { + name: logging.getLogger(name).level for name in _RUST_LOGGER_ROOTS + } + for name in _RUST_LOGGER_ROOTS: + logging.getLogger(name).setLevel(logging.DEBUG) + reset_logging_config() + + try: + server = _StubServer() + self.addCleanup(server.shutdown) + + def body(result: dict[str, object]) -> None: + async def do() -> None: + try: + await self._rust_http_client.get( + url=server.endpoint, + response_limit=1 * 1024 * 1024, + ) + finally: + result["done"] = True + + run_in_background(do) + + self._run_in_logcontext_and_pump("http-caller", body) + finally: + root.removeHandler(handler) + for name, level in saved_levels.items(): + logging.getLogger(name).setLevel(level) + reset_logging_config() + + rust_records = [ + (name, req) + for (name, req) in records + if name.split(".", 1)[0] in _RUST_LOGGER_ROOTS + ] + self.assertTrue( + rust_records, + "expected at least one Rust-origin log record (e.g. reqwest connecting); " + f"captured loggers: {sorted({name for name, _ in records})}", + ) + for name, req in rust_records: + self.assertEqual( + req, + "http-caller", + f"Rust log record from {name!r} was attributed to {req!r}, " + "not the caller's logcontext", + ) + + def test_db_callback_runs_in_caller_logcontext(self) -> None: + """The Rust `/versions` handler's per-user feature lookup calls back into + Python via `run_python_awaitable`; the DB transaction it runs must be + accounted against the caller's logcontext. The failure mode is the + awaitable running in the sentinel instead, silently losing the + accounting.""" + versions_handler = self.hs.get_rust_handlers().versions + + def body(result: dict[str, object]) -> None: + async def do() -> None: + try: + context = current_context() + assert isinstance(context, LoggingContext) + before = context.get_resource_usage().db_txn_count + + # Passing a user id makes the Rust handler do a per-user + # feature DB lookup (msc3881/msc3575 default to off), which + # goes Rust -> run_python_awaitable -> runInteraction. + await versions_handler.get_versions(self.user_id) + + after = context.get_resource_usage().db_txn_count + result["db_txn_delta"] = after - before + finally: + result["done"] = True + + run_in_background(do) + + result = self._run_in_logcontext_and_pump("db-caller", body) + + db_txn_delta = result.get("db_txn_delta", 0) + assert isinstance(db_txn_delta, int) + self.assertGreaterEqual( + db_txn_delta, + 1, + "the Rust handler's DB work was not accounted against the caller's " + "logcontext — run_python_awaitable is not restoring it (ran in the " + "sentinel instead)", + ) + + +class _StubServer: + """A real HTTP server on a random port, served from a background thread.""" + + def __init__(self) -> None: + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"ok": true}') + + def log_message(self, format: str, *args: object) -> None: + pass + + self._server = HTTPServer(("127.0.0.1", 0), Handler) + self._thread = threading.Thread( + target=self._server.serve_forever, + name="StubServer", + kwargs={"poll_interval": 0.01}, + daemon=True, + ) + self._thread.start() + + @property + def endpoint(self) -> str: + return f"http://127.0.0.1:{self._server.server_port}/" + + def shutdown(self) -> None: + self._server.shutdown() + self._thread.join() diff --git a/tests/test_notifier.py b/tests/test_notifier.py index c65134e832..deb7a9adf4 100644 --- a/tests/test_notifier.py +++ b/tests/test_notifier.py @@ -11,10 +11,12 @@ # . import logging +from collections import Counter from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from synapse.notifier import wait_for_stream_token_timeout_counter from synapse.server import HomeServer from synapse.types import MultiWriterStreamToken, StreamKeyType, StreamToken from synapse.util.clock import Clock @@ -134,3 +136,60 @@ class NotifierTestCase(tests.unittest.HomeserverTestCase): # Make sure we gave up waiting and not caught-up (False) wait_result = self.get_success(wait_d) self.assertEqual(wait_result, False) + + def test_wait_for_stream_token_timeout_counts_each_lagging_stream(self) -> None: + """ + Test that a timeout while lagging on more than one stream is counted against + each of them. + """ + + lagging_stream_keys = [StreamKeyType.RECEIPT, StreamKeyType.DEVICE_LIST] + + # Create a new token with the stream IDs artificially advanced far into + # the future. + token = StreamToken.START + token = token.copy_and_advance( + StreamKeyType.RECEIPT, + MultiWriterStreamToken(stream=10000000000), # arbitrarily large + ) + token = token.copy_and_advance( + StreamKeyType.DEVICE_LIST, + MultiWriterStreamToken(stream=10000000000), # arbitrarily large + ) + + counts_before = self._get_timeout_counts_from_metric() + + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + + # Advance time to make the `wait_for_stream_token(...)` timeout. + self.reactor.advance(Duration(seconds=11).as_secs()) + + # Make sure we gave up waiting and not caught-up (False) + self.assertEqual(self.get_success(wait_d), False) + + self.assertEqual( + self._get_timeout_counts_from_metric() - counts_before, + Counter({stream_key.value: 1 for stream_key in lagging_stream_keys}), + ) + + def _get_timeout_counts_from_metric(self) -> "Counter[str]": + """The `wait_for_stream_token` timeout counts for this server, keyed by the + `stream_key` label. + + Useful because the Prometheus metrics are process-wide, and so shared + between tests. We use this to compare against a count taken before the + code under test ran. + """ + + # Fetch all samples from the `wait_for_stream_token_timeout_counter` + # metric. The only label (other than the automatic `server_name` label) + # is `stream_key`. + samples = self.get_prometheus_metric_current_values( + wait_for_stream_token_timeout_counter, + ) + + counts: Counter[str] = Counter() + for sample in samples: + counts[sample.labels["stream_key"]] = int(sample.value) + + return counts diff --git a/tests/test_state.py b/tests/test_state.py index b69a09dab5..bb0b7f8878 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +from collections import ChainMap from typing import ( Any, Collection, @@ -34,9 +35,15 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.state import StateHandler, StateResolutionHandler, _make_state_cache_entry +from synapse.state import ( + StateHandler, + StateResolutionHandler, + _make_state_cache_entry, + _state_map_size, +) from synapse.types import JsonDict, MutableStateMap, StateMap from synapse.types.state import StateFilter +from synapse.util import MutableOverlayMapping from synapse.util.macaroons import MacaroonGenerator from tests import unittest @@ -245,6 +252,7 @@ class StateTestCase(unittest.TestCase): ) reactor, clock = get_clock() hs.config = default_config(server_name="tesths", parse=True) + hs.hostname = "tesths" hs.get_datastores.return_value = Mock( main=self.dummy_store, state_deletion=dummy_deletion_store, @@ -863,6 +871,31 @@ class StateTestCase(unittest.TestCase): result = yield defer.ensureDeferred(self.state.compute_event_context(event)) return result + def test_state_map_size(self) -> None: + "Cache sizing counts every held entry, not the distinct keys" + + base: StateMap[str] = {("a", ""): "A", ("b", ""): "B", ("c", ""): "C"} + self.assertEqual(_state_map_size(base), 3) + + # Overriding and deleting keys leaves `len()` alone but holds entries. + overlay = MutableOverlayMapping(base) + overlay[("a", "")] = "A2" + overlay[("d", "")] = "D" + del overlay[("b", "")] + self.assertEqual(len(overlay), 3) + self.assertEqual(_state_map_size(overlay), 3 + 2 + 1) + + # Nested overlays are followed down. + outer = MutableOverlayMapping(overlay) + outer[("e", "")] = "E" + self.assertEqual(len(outer), 4) + self.assertEqual(_state_map_size(outer), 6 + 1) + + # A `ChainMap` holds every layer in full, however much they overlap. + chain = ChainMap({("a", ""): "A3", ("f", ""): "F"}, outer) + self.assertEqual(len(chain), 5) + self.assertEqual(_state_map_size(chain), 2 + 7) + def test_make_state_cache_entry(self) -> None: "Test that calculating a prev_group and delta is correct" diff --git a/tests/test_utils/event_builders.py b/tests/test_utils/event_builders.py index a5d686801d..f81468eb0b 100644 --- a/tests/test_utils/event_builders.py +++ b/tests/test_utils/event_builders.py @@ -112,6 +112,20 @@ def make_test_pdu_event( defaults as `make_test_event` before delegating. """ pdu = {**default_event_fields(room_version), **pdu} + + # For room versions where the create event's room_id is derived from its + # event ID (v11+ format), omit the default room_id on create events so each + # create event ends up with a distinct room_id. + # + # We can't do this in the `default_event_fields` as we don't know the event + # type at that point. + if ( + room_version.msc4291_room_ids_as_hashes + and pdu["type"] == "m.room.create" + and pdu["state_key"] == "" + ): + pdu.pop("room_id", None) + return event_from_pdu_json(pdu, room_version, received_time=received_time) diff --git a/tests/test_visibility.py b/tests/test_visibility.py index 9a5efbdd39..0654f84351 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -608,9 +608,11 @@ class FilterEventsOutOfBandEventsForClientTestCase( self.get_success( self.hs.get_federation_server().on_invite_request( - self.OTHER_SERVER_NAME, - invite_pdu, - "9", + origin=self.OTHER_SERVER_NAME, + expected_event_id=invite_event_id, + expected_room_id="!room:id", + event_json=invite_pdu, + room_version_id="9", ) ) diff --git a/tests/unittest.py b/tests/unittest.py index 4881a6650a..91adfd8318 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -19,6 +19,7 @@ # [This file includes modifications made by New Vector Limited] # # +import copy import functools import gc import hashlib @@ -26,11 +27,13 @@ import hmac import json import logging import os +import re import secrets import sys import time from collections.abc import Set from typing import ( + TYPE_CHECKING, Any, Awaitable, Callable, @@ -48,6 +51,7 @@ from unittest.mock import Mock, patch import canonicaljson import signedjson.key import unpaddedbase64 +from prometheus_client.samples import Sample from typing_extensions import Concatenate, Never, ParamSpec, override from twisted.internet import defer @@ -75,6 +79,7 @@ from synapse.logging.context import ( current_context, set_current_context, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.rest import RegisterServletsFunc from synapse.server import HomeServer from synapse.storage.keys import FetchKeyResult @@ -94,6 +99,9 @@ from tests.test_utils import event_injection, setup_awaitable_errors from tests.test_utils.logging_setup import setup_logging from tests.utils import checked_cast, default_config, setupdb +if TYPE_CHECKING: + from prometheus_client.registry import Collector + setupdb() setup_logging() @@ -163,8 +171,14 @@ def deepcopy_config(config: _TConfig) -> _TConfig: if attr_name.startswith("__") or attr_name == "root": continue attr = getattr(config, attr_name) + new_attr: Any if isinstance(attr, Config): new_attr = deepcopy_config(attr) + elif isinstance(attr, (list, dict, set)): + # Copy mutable containers so that tests which modify config values + # in place (e.g. appending to a list) don't leak those changes into + # the cached config object, and thus into every other test. + new_attr = copy.deepcopy(attr) else: new_attr = attr @@ -447,6 +461,32 @@ class TestCase(unittest.TestCase): self._fail_with_set_inequality(actual_items, expected_items, message, exact) + def assertEqualNormalisingWhitespace( + self, received: str, expected: str, msg: str | None = None + ) -> None: + """ + Fail the test if `received` and `expected` are not equal, + after having normalised all whitespace. + + By normalising whitespace, we mean that any run of whitespace + is replaced by a single ` `, on both sides. + The front and back of the strings are also trimmed. + + Whitespace characters considered are: `\n`, `\t`, ` `. + """ + PATTERN = r"[ \n\t]+" + REPLACEMENT = " " + normalised_received = re.sub(PATTERN, REPLACEMENT, received).strip() + normalised_expected = re.sub(PATTERN, REPLACEMENT, expected).strip() + + if normalised_received == normalised_expected: + return + + msg = "" if msg is None else msg + self.fail( + f"Expected strings to match, after normalising whitespace: {msg}\nReceived: {received}\nExpected: {expected}\nReceived (normalised): {normalised_received}\nExpected (normalised): {normalised_expected}" + ) + def DEBUG(target: TV) -> TV: """A decorator to set the .loglevel attribute to logging.DEBUG. @@ -1218,6 +1258,116 @@ class HomeserverTestCase(TestCase): event_injection.inject_member_event(self.hs, room, user, membership) ) + def get_prometheus_metric_current_values( + self, metric: "Collector", **labels: str + ) -> list[Sample]: + """Get all samples of a prometheus metric with the given labels. + + Note that the metrics outlives each individual test, so it may hold + values from previous tests. + + The `metric` must return exactly one unique series name among its + samples, i.e. it should be a Counter or a Gauge, rather than a + Histogram. + + Automatically includes SERVER_NAME_LABEL. + """ + + labels = dict(labels) + labels[SERVER_NAME_LABEL] = self.hs.hostname + + # Matching samples for the given labels. + found_samples = [] + + for collected in metric.collect(): + for sample in collected.samples: + # Old versions of `prometheus_client` also emit a `_created` + # timestamp sample alongside each sample, which is never what we + # want. + if sample.name.endswith("_created"): + continue + + # Check that all the labels match. If any label doesn't match, + # we skip this sample. + for label, value in labels.items(): + if sample.labels.get(label) != value: + break + else: + # We didn't break, so all the labels matched. Return this + # sample's value. + found_samples.append(sample) + + # Ensure that there is exactly one unique series name among the found + # samples. This is to guard against someone passing in eg a Histogram + # metric, which will return multiple series (eg `_total`, `_count`, + # `_buckets`, etc), which is likely not what the caller intended. + series_names = {sample.name for sample in found_samples} + if len(series_names) > 1: + raise AssertionError( + f"Expected exactly one unique sample name for metric {metric}, but found:\n\n" + + "\n".join(series_names) + + "\n\n" + + "This is a quirk of this test utility (`get_prometheus_metric_current_values(...)`), " + + "not your metric and means things like histogram metrics which have `_total`, `_count`, " + + "`buckets` sample names aren't compatible with this util at the moment." + ) + + return found_samples + + def get_prometheus_metric_current_value( + self, metric: "Collector", **labels: str + ) -> int: + """Get the value of a prometheus metric with the given labels. + + This function will raise an AssertionError if there is not exactly one + sample with the given labels. + + Note that the metrics outlives each individual test, so it may hold + values from previous tests. + + The `metric` must return exactly one unique series name among its + samples, i.e. it should be a Counter or a Gauge, rather than a + Histogram. + + Automatically includes SERVER_NAME_LABEL. + """ + + found_samples = self.get_prometheus_metric_current_values(metric, **labels) + + # The caller expects there to be exactly one sample with the given + # labels. If there are multiple (or zero) samples, we error. + if len(found_samples) == 1: + # We found exactly one sample with the given labels, so return its + # value. + return int(found_samples[0].value) + elif len(found_samples) > 1: + # We found multiple samples with the given labels, so we error. We + # helpfully include the differences in labels between the samples to + # help the caller figure out why they got multiple samples. + labels_differences_dicts = _get_dict_differences( + [sample.labels for sample in found_samples] + ) + differences_str = "\n".join(f" {diff}" for diff in labels_differences_dicts) + + raise AssertionError( + f"Multiple metrics found for '{metric}' with labels {labels}\n\n" + "`get_prometheus_metric_current_value(...)` expects you to be specific enough" + "with labels that only one metric matches. Either, the metrics changed and" + "that's wrong in and of itself or you need to update the test to be more" + "specific with the labels. The extra labels you can match with are:\n" + f"{differences_str}" + ) + else: + all_metrics = "\n".join( + f" {sample.labels}" + for collected in metric.collect() + for sample in collected.samples + ) + raise AssertionError( + f"No metric found for {metric} with labels {labels}\n" + f"All metrics:\n{all_metrics}" + ) + class FederatingHomeserverTestCase(HomeserverTestCase): """ @@ -1400,3 +1550,25 @@ def skip_unless(condition: bool, reason: str) -> Callable[[TV], TV]: return f return decorator + + +def _get_dict_differences(dicts: list[dict[str, str]]) -> list[dict[str, str]]: + """Return a list of dicts where each dict has the common key/values removed. + + Useful for printing comparisons of prometheus metrics with different labels. + """ + if not dicts: + return [] + + # Find the common key/values across all dicts + common_items = set(dicts[0].items()) + for d in dicts[1:]: + common_items.intersection_update(d.items()) + + # Remove the common items from each dict + differences = [] + for d in dicts: + diff = {k: v for k, v in d.items() if (k, v) not in common_items} + differences.append(diff) + + return differences diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py index 8964359a6e..8346cab4f0 100644 --- a/tests/util/test_expiring_cache.py +++ b/tests/util/test_expiring_cache.py @@ -87,6 +87,49 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get("key3"), [4, 5]) self.assertEqual(cache.get("key4"), [6, 7]) + def test_size_callback_eviction(self) -> None: + reactor, clock = get_clock() + cache: ExpiringCache[str, list[int]] = ExpiringCache( + cache_name="test", + server_name="testserver", + hs=self.hs, + clock=clock, + max_len=5, + # Size each value by its sum, so that `len()` and the reported + # size disagree. + size_callback=sum, + ) + + cache["key"] = [1] + cache["key2"] = [2] + cache["key3"] = [2] + + self.assertEqual(cache.get("key"), [1]) + self.assertEqual(cache.get("key2"), [2]) + self.assertEqual(cache.get("key3"), [2]) + self.assertEqual(len(cache), 5) + + # Three entries of `len()` 1 each, but a size of 6, so the oldest two + # go. + cache["key4"] = [3] + self.assertEqual(cache.get("key"), None) + self.assertEqual(cache.get("key2"), None) + self.assertEqual(cache.get("key3"), [2]) + self.assertEqual(cache.get("key4"), [3]) + self.assertEqual(len(cache), 5) + + def test_iterable_and_size_callback_are_exclusive(self) -> None: + reactor, clock = get_clock() + with self.assertRaises(ValueError): + ExpiringCache( + cache_name="test", + server_name="testserver", + hs=self.hs, + clock=clock, + iterable=True, + size_callback=len, + ) + def test_time_eviction(self) -> None: reactor, clock = get_clock() cache: ExpiringCache[str, int] = ExpiringCache( diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py index a4114cdfcc..230257105d 100644 --- a/tests/util/test_logcontext.py +++ b/tests/util/test_logcontext.py @@ -20,13 +20,19 @@ # import logging +import threading +from contextlib import contextmanager from typing import Callable, Generator, cast +from unittest.mock import patch from twisted.internet import defer, reactor as _reactor +import synapse.logging.context as context_module from synapse.logging.context import ( SENTINEL_CONTEXT, + ContextRequest, LoggingContext, + LoggingContextFilter, PreserveLoggingContext, _Sentinel, current_context, @@ -34,6 +40,7 @@ from synapse.logging.context import ( nested_logging_context, run_coroutine_in_background, run_in_background, + set_current_context, ) from synapse.types import ISynapseReactor from synapse.util.clock import Clock @@ -700,6 +707,226 @@ class LoggingContextTestCase(unittest.TestCase): self.assertEqual(nested_context.name, "foo-bar") +# A stand-in `(ru_utime, ru_stime)` value to pass to `start()`/`stop()`. +# +# The code under test only checks whether an rusage was supplied at all +# (`None` means the platform doesn't track per-thread CPU); the values +# themselves don't matter here. A constant also avoids depending on +# `RUSAGE_THREAD` support on the platform running the tests. +_TRUTHY_RUSAGE = (0.0, 0.0) + + +@contextmanager +def _capture_logcontext_errors() -> Generator[list[str], None, None]: + """Capture the messages passed to `logcontext_error` while inside the block. + + `logcontext_error` is looked up as a module global at call time, so patching + the module attribute intercepts every call site (`LoggingContext`, + `PreserveLoggingContext`, `run_in_background`, ...). + """ + messages: list[str] = [] + with patch.object(context_module, "logcontext_error", messages.append): + yield messages + + +class LogContextErrorMessageTestCase(unittest.TestCase): + """Tests asserting the exact messages passed to `logcontext_error`, and + the conditions that trigger each one. + """ + + def setUp(self) -> None: + # These tests call `__enter__`/`__exit__`/`start`/`stop` by hand. Let's + # make sure the current context ends up back at the sentinel regardless + # of what a test does. + self.assertIs(current_context(), SENTINEL_CONTEXT) + self.addCleanup(set_current_context, SENTINEL_CONTEXT) + + def test_enter_wrong_previous_context(self) -> None: + with _capture_logcontext_errors() as messages: + # `ctx` records the sentinel as its `previous_context` (we are at the + # sentinel now), but we enter it while a *different* context is active. + ctx = LoggingContext(name="ctx", server_name="s") + other = LoggingContext(name="other", server_name="s") + other.__enter__() + + ctx.__enter__() + + self.assertEqual( + messages, + [ + "Expected previous context %r, found %r" + % (ctx.previous_context, other) + ], + ) + + def test_exit_context_was_lost(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + ctx.__enter__() + # Simulate the context being lost from under us before exit. + set_current_context(SENTINEL_CONTEXT) + + ctx.__exit__(None, None, None) + + self.assertEqual(messages, ["Expected logging context ctx was lost"]) + self.assertTrue(ctx.finished) + + def test_exit_found_different_context(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + ctx.__enter__() + other = LoggingContext(name="other", server_name="s") + other.__enter__() + + ctx.__exit__(None, None, None) + + self.assertEqual(messages, ["Expected logging context ctx but found other"]) + + def test_preserve_logging_context_was_lost(self) -> None: + with _capture_logcontext_errors() as messages: + target = LoggingContext(name="target", server_name="s") + preserve = PreserveLoggingContext(target) + preserve.__enter__() + set_current_context(SENTINEL_CONTEXT) + + preserve.__exit__(None, None, None) + + self.assertEqual(messages, ["Expected logging context target was lost"]) + + def test_preserve_logging_context_found_different_context(self) -> None: + with _capture_logcontext_errors() as messages: + target = LoggingContext(name="target", server_name="s") + preserve = PreserveLoggingContext(target) + preserve.__enter__() + other = LoggingContext(name="other", server_name="s") + other.__enter__() + + preserve.__exit__(None, None, None) + + self.assertEqual( + messages, ["Expected logging context target but found other"] + ) + + def test_start_on_different_thread(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + + # Call `start` on a different thread + thread = threading.Thread(target=ctx.start, args=(None,)) + thread.start() + thread.join() + + self.assertEqual(messages, ["Started logcontext ctx on different thread"]) + + def test_stop_on_different_thread(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + + # Call `stop` on a different thread + thread = threading.Thread(target=ctx.stop, args=(None,)) + thread.start() + thread.join() + + self.assertEqual(messages, ["Stopped logcontext ctx on different thread"]) + + def test_restart_finished_context(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + ctx.finished = True + + ctx.start(None) + + self.assertEqual(messages, ["Re-starting finished log context ctx"]) + + def test_restart_already_active_context(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + ctx.start(_TRUTHY_RUSAGE) + + ctx.start(_TRUTHY_RUSAGE) + + self.assertEqual(messages, ["Re-starting already-active log context ctx"]) + + def test_stop_without_start(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + + ctx.stop(_TRUTHY_RUSAGE) + + self.assertEqual( + messages, + ["Called stop on logcontext ctx without recording a start rusage"], + ) + + +class LoggingContextFilterTestCase(unittest.TestCase): + """Tests for `LoggingContextFilter`, which is how logcontexts reach log + output. Checks that it fills record attributes from a real logcontext, and + that under the sentinel it only sets defaults for attributes that aren't + already present.""" + + def setUp(self) -> None: + self.assertIs(current_context(), SENTINEL_CONTEXT) + self.filter = LoggingContextFilter() + + def _make_record(self, **attrs: object) -> logging.LogRecord: + record = logging.LogRecord("test", logging.INFO, "path", 1, "msg", None, None) + for key, value in attrs.items(): + setattr(record, key, value) + return record + + def test_fills_record_from_real_context(self) -> None: + request = ContextRequest( + request_id="req-1", + ip_address="1.2.3.4", + site_tag="site", + requester="@u:test", + authenticated_entity="@u:test", + method="GET", + url="/path", + protocol="1.1", + user_agent="UA", + ) + with LoggingContext(name="ctx", server_name="myserver", request=request): + record = self._make_record() + self.assertTrue(self.filter.filter(record)) + + # Read via `__dict__` since these attributes are only ever set + # dynamically (they aren't declared on `logging.LogRecord`). + attrs = record.__dict__ + self.assertEqual(attrs["server_name"], "myserver") + # For backwards compatibility, `str(context)` is stored as `request`. + self.assertEqual(attrs["request"], "ctx") + self.assertEqual(attrs["ip_address"], "1.2.3.4") + self.assertEqual(attrs["site_tag"], "site") + self.assertEqual(attrs["requester"], "@u:test") + self.assertEqual(attrs["authenticated_entity"], "@u:test") + self.assertEqual(attrs["method"], "GET") + self.assertEqual(attrs["url"], "/path") + self.assertEqual(attrs["protocol"], "1.1") + self.assertEqual(attrs["user_agent"], "UA") + + def test_sentinel_sets_defaults_when_absent(self) -> None: + record = self._make_record() + self.assertTrue(self.filter.filter(record)) + + self.assertEqual( + record.__dict__["server_name"], "unknown_server_from_sentinel_context" + ) + self.assertEqual(record.__dict__["request"], "sentinel") + + def test_sentinel_does_not_clobber_preset_attributes(self) -> None: + # Simulate a 3rd-party log record that already carries its own attributes; + # under the sentinel the filter must leave them untouched. + record = self._make_record( + server_name="third-party-server", request="third-party-request" + ) + self.assertTrue(self.filter.filter(record)) + + self.assertEqual(record.__dict__["server_name"], "third-party-server") + self.assertEqual(record.__dict__["request"], "third-party-request") + + # a function which returns a deferred which has been "called", but # which had a function which returned another incomplete deferred on # its callback list, so won't yet call any other new callbacks. diff --git a/tests/util/test_mutable_overlay_mapping.py b/tests/util/test_mutable_overlay_mapping.py index ed738919e4..85444635db 100644 --- a/tests/util/test_mutable_overlay_mapping.py +++ b/tests/util/test_mutable_overlay_mapping.py @@ -187,3 +187,76 @@ class TestMutableOverlayMapping(unittest.TestCase): self.assertNotIn("c", mapping) self.assertIn("d", mapping) self.assertNotIn("e", mapping) + + def test_len_after_reset_and_redelete(self) -> None: + """len() must follow keys that move between the underlying map, the + overrides and the deletions.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + # Delete an underlying key, then set it again. + del mapping["a"] + self.assertEqual(len(mapping), 2) + mapping["a"] = 10 + self.assertEqual(len(mapping), 3) + + # Add a key only the overlay knows about, delete it, and add it back. + mapping["d"] = 4 + self.assertEqual(len(mapping), 4) + del mapping["d"] + self.assertEqual(len(mapping), 3) + mapping["d"] = 40 + self.assertEqual(len(mapping), 4) + + # Override then delete an underlying key. + mapping["b"] = 20 + del mapping["b"] + self.assertEqual(len(mapping), 3) + + self.assertEqual(len(mapping), len(dict(mapping))) + + def test_len_nested(self) -> None: + """An overlay over an overlay reports the right length.""" + inner = MutableOverlayMapping({"a": 1, "b": 2}) + inner["c"] = 3 + del inner["a"] + + outer = MutableOverlayMapping(inner) + self.assertEqual(len(outer), 2) + + outer["a"] = 10 # deleted in inner, so new to outer + outer["b"] = 20 # present in inner's underlying map + outer["c"] = 30 # present in inner's overrides + outer["d"] = 40 # new + self.assertEqual(len(outer), 4) + + del outer["b"] + del outer["d"] + self.assertEqual(len(outer), 2) + self.assertEqual(len(outer), len(dict(outer))) + + def test_total_entries(self) -> None: + """total_entries() counts the base map plus every override and + deletion, unlike len().""" + mapping = MutableOverlayMapping({"a": 1, "b": 2, "c": 3}) + self.assertEqual(mapping.total_entries(), 3) + + mapping["a"] = 10 # override: +1 entry, same length + mapping["d"] = 4 # new key: +1 entry, +1 length + del mapping["b"] # deletion: +1 entry, -1 length + self.assertEqual(len(mapping), 3) + self.assertEqual(mapping.total_entries(), 6) + + # Deleting an override drops it from the overrides and records the + # deletion, so the entry count is unchanged. + del mapping["d"] + self.assertEqual(len(mapping), 2) + self.assertEqual(mapping.total_entries(), 6) + + # Nested overlays are counted all the way down. + outer = MutableOverlayMapping(mapping) + outer["e"] = 5 + self.assertEqual(outer.total_entries(), 7) + + mapping.clear() + self.assertEqual(mapping.total_entries(), 0) diff --git a/tests/util/test_task_scheduler.py b/tests/util/test_task_scheduler.py index cab9695d33..bdd4f21924 100644 --- a/tests/util/test_task_scheduler.py +++ b/tests/util/test_task_scheduler.py @@ -98,9 +98,10 @@ class TestTaskScheduler(HomeserverTestCase): return TaskStatus.COMPLETE, None, None def test_schedule_lot_of_tasks(self) -> None: - """Schedule more than `TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS` tasks and check the behavior.""" + """Schedule more than `self.task_scheduler._max_concurrent_tasks` tasks and check the behavior.""" + max_concurrent = self.task_scheduler._max_concurrent_tasks task_ids = [] - for i in range(TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS + 1): + for i in range(max_concurrent + 1): task_ids.append( self.get_success( self.task_scheduler.schedule_task( @@ -117,11 +118,11 @@ class TestTaskScheduler(HomeserverTestCase): ) return [t for t in tasks if t is not None and t.status == status] - # At this point, there should be MAX_CONCURRENT_RUNNING_TASKS active tasks and + # At this point, there should be max_concurrent active tasks and # one scheduled task. self.assertEqual( len(get_tasks_of_status(TaskStatus.ACTIVE)), - TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS, + max_concurrent, ) self.assertEqual( len(get_tasks_of_status(TaskStatus.SCHEDULED)), @@ -131,11 +132,70 @@ class TestTaskScheduler(HomeserverTestCase): # Give the time to the active tasks to finish self.reactor.advance(1) - # Check that MAX_CONCURRENT_RUNNING_TASKS tasks have run and that one + # Check that max_concurrent tasks have run and that one # is still scheduled. self.assertEqual( len(get_tasks_of_status(TaskStatus.COMPLETE)), - TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS, + max_concurrent, + ) + scheduled_tasks = get_tasks_of_status(TaskStatus.SCHEDULED) + self.assertEqual(len(scheduled_tasks), 1) + + # The scheduled task should start 0.1s after the first of the active tasks + # finishes + self.reactor.advance(0.1) + self.assertEqual(len(get_tasks_of_status(TaskStatus.ACTIVE)), 1) + + # ... and should finally complete after another second + self.reactor.advance(1) + prev_scheduled_task = self.get_success( + self.task_scheduler.get_task(scheduled_tasks[0].id) + ) + assert prev_scheduled_task is not None + self.assertEqual( + prev_scheduled_task.status, + TaskStatus.COMPLETE, + ) + + @override_config({"task_scheduler": {"max_concurrent_tasks": 4}}) + def test_schedule_lot_of_tasks_custom_concurrency(self) -> None: + """Test that configuring a higher concurrency limit allows more concurrent tasks.""" + self.assertEqual(self.task_scheduler._max_concurrent_tasks, 4) + max_concurrent = self.task_scheduler._max_concurrent_tasks + task_ids = [] + for i in range(max_concurrent + 1): + task_ids.append( + self.get_success( + self.task_scheduler.schedule_task( + "_sleeping_task", + params={"val": i}, + ) + ) + ) + + def get_tasks_of_status(status: TaskStatus) -> list[ScheduledTask]: + tasks = ( + self.get_success(self.task_scheduler.get_task(task_id)) + for task_id in task_ids + ) + return [t for t in tasks if t is not None and t.status == status] + + self.assertEqual( + len(get_tasks_of_status(TaskStatus.ACTIVE)), + 4, + ) + self.assertEqual( + len(get_tasks_of_status(TaskStatus.SCHEDULED)), + 1, + ) + + # Give the time to the active tasks to finish + self.reactor.advance(1) + + # Check that 4 tasks have run and that one is still scheduled. + self.assertEqual( + len(get_tasks_of_status(TaskStatus.COMPLETE)), + 4, ) scheduled_tasks = get_tasks_of_status(TaskStatus.SCHEDULED) self.assertEqual(len(scheduled_tasks), 1)