diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index 41b7be6192..a891802ac8 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout synapse codebase - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: path: synapse @@ -58,7 +58,7 @@ jobs: # We use `poetry` in `complement.sh` - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" # Matches the `path` where we checkout Synapse above working-directory: "synapse" @@ -76,7 +76,7 @@ jobs: run: | set -x DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx - pipx install poetry==2.2.1 + pipx install poetry==2.4.1 poetry remove -n twisted poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk @@ -92,13 +92,24 @@ jobs: # are underpowered and don't like running tons of Synapse instances at once. # -json: Output JSON format so that gotestfmt can parse it. # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. + # tee /tmp/gotest-sanity-check-complement.log: We tee the raw JSON to a + # file so we can re-process it later for better formatting with gotestfmt + # (and upload it as an artifact). + # + # We deliberately do not dump the raw JSON to the terminal as it is huge + # and unreadable. Instead we pipe it through `jq` to print a compact, + # real-time progress view: for each `go test -json` event we print only + # completed tests (`pass`/`skip`/`fail`, including subtests), dropping the + # noisy `run`/`output` events and package-level lines. The full detail + # lives in the raw log we tee to the file above. Non-JSON lines (e.g. the + # image build output and `set -x` traces that `go test -json` does not + # wrap) are passed through unchanged rather than parsed, so we still see + # them and the filter never aborts the pipeline. run: | set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 | tee /tmp/gotest-sanity-check-complement.log + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 \ + | tee /tmp/gotest-sanity-check-complement.log \ + | jq -rR --unbuffered '. as $raw | (try fromjson catch $raw) | if type == "object" then (select(.Action=="pass" or .Action=="fail" or .Action=="skip") | select(.Test) | "\(.Action|ascii_upcase) \(.Test) \(.Elapsed)s") else $raw end' shell: bash env: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} @@ -121,13 +132,16 @@ jobs: # are underpowered and don't like running tons of Synapse instances at once. # -json: Output JSON format so that gotestfmt can parse it. # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. + # tee /tmp/gotest-complement.log: We tee the raw JSON to a file so we can + # re-process it later for better formatting with gotestfmt (and upload it + # as an artifact), and pipe it through `jq` for a compact, real-time + # progress view. See the sanity check step above for details on the jq + # filter. run: | set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 \ + | tee /tmp/gotest-complement.log \ + | jq -rR --unbuffered '. as $raw | (try fromjson catch $raw) | if type == "object" then (select(.Action=="pass" or .Action=="fail" or .Action=="skip") | select(.Test) | "\(.Action|ascii_upcase) \(.Test) \(.Elapsed)s") else $raw end' shell: bash env: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} @@ -151,13 +165,16 @@ jobs: # are underpowered and don't like running tons of Synapse instances at once. # -json: Output JSON format so that gotestfmt can parse it. # - # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. + # tee /tmp/gotest-in-repo-complement.log: We tee the raw JSON to a file so + # we can re-process it later for better formatting with gotestfmt (and + # upload it as an artifact), and pipe it through `jq` for a compact, + # real-time progress view. See the sanity check step above for details on + # the jq filter. run: | set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 \ + | tee /tmp/gotest-in-repo-complement.log \ + | jq -rR --unbuffered '. as $raw | (try fromjson catch $raw) | if type == "object" then (select(.Action=="pass" or .Action=="fail" or .Action=="skip") | select(.Test) | "\(.Action|ascii_upcase) \(.Test) \(.Elapsed)s") else $raw end' shell: bash env: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} @@ -174,3 +191,18 @@ jobs: # it derives several values under `$settings` and passes them to our # custom `.ci/complement_package.gotpl` template to render the output. run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" + + - name: Upload Complement logs + # Always upload the logs (as artifacts) if we attempted to run the + # Complement tests, so we can debug failures without scrolling through + # the (very large) raw output in the build log. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ always() }} + with: + # The matrix values keep the name unique across the three matrix + # combinations (upload-artifact rejects duplicate names within a run). + name: Complement Logs - ${{ job.status }} - (${{ matrix.arrangement }}, ${{ matrix.database }}) + path: | + /tmp/gotest-sanity-check-complement.log + /tmp/gotest-complement.log + /tmp/gotest-in-repo-complement.log diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 801d369c48..39ddb61918 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,10 +28,10 @@ jobs: steps: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see @@ -41,13 +41,13 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -79,7 +79,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -87,7 +87,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 with: push: true labels: | @@ -136,14 +136,14 @@ jobs: merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io @@ -176,20 +176,20 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} password: ${{ steps.import-secrets.outputs.OCI_PASSWORD }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Install Cosign uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Calculate docker image tag - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 with: images: ${{ matrix.repository }} flavor: | diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index b2161146db..e1a5b7be89 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -13,7 +13,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -50,7 +50,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 31f3b05b8e..7236bf99d9 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,7 @@ jobs: needs: - pre steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -92,7 +92,7 @@ jobs: # Deploy to the target directory. - name: Deploy to gh pages - uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./book diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index aea55fe0ce..3e518c6267 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -31,7 +31,7 @@ jobs: uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: install-project: "false" - poetry-version: "2.2.1" + poetry-version: "2.4.1" - name: Run ruff check continue-on-error: true diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index fa3f31e2c9..815593ffcd 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -42,7 +42,7 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -54,7 +54,7 @@ jobs: - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: "all" # Dump installed versions for debugging. - run: poetry run pip list > before.txt @@ -77,7 +77,7 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -151,7 +151,7 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -201,7 +201,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 5042a564f2..06545bd18a 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -16,7 +16,7 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index da45bb2c03..6f4c966cdc 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,33 +33,33 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 if: github.event_name == 'push' with: ref: master # We use `poetry` in `complement.sh` - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" - name: Login to registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Work out labels for complement image id: meta - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 with: images: ghcr.io/${{ github.repository }}/complement-synapse tags: | diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 41a7cb5611..c6b9f60baf 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,7 +27,7 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -50,18 +50,24 @@ jobs: name: "Build .deb packages" runs-on: ubuntu-latest strategy: + # Prevent all Debian package builds from being cancelled + # when one of them fails. Especially helpful when things are + # flaky during a release. + # + # `fail-fast` defaults to true and applies to the entire test matrix + fail-fast: false matrix: distro: ${{ fromJson(needs.get-distros.outputs.distros) }} steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: path: src - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Set up docker layer caching uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 @@ -123,7 +129,7 @@ jobs: os: "ubuntu-24.04-arm" steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -161,7 +167,7 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 430ff8861a..e36114d354 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -14,7 +14,7 @@ jobs: name: Ensure Synapse config schema is valid runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -40,7 +40,7 @@ jobs: name: Ensure generated documentation is up-to-date runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c1277e3d5f..6ecdd88815 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,8 +60,10 @@ jobs: - 'poetry.lock' - 'docker/**' - '.ci/**' - - 'scripts-dev/complement.sh' - '.github/workflows/tests.yml' + - 'scripts-dev/complement.sh' + - '.github/workflows/complement_tests.yml' + - 'complement/**' linting: - 'synapse/**' @@ -86,7 +88,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -95,7 +97,7 @@ jobs: - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: "all" - run: poetry run scripts-dev/generate_sample_config.sh --check - run: poetry run scripts-dev/config-lint.sh @@ -106,7 +108,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -116,7 +118,7 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -129,12 +131,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" install-project: "false" - name: Run ruff check @@ -151,7 +153,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -169,7 +171,7 @@ jobs: # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775 # To make CI green, err towards caution and install the project. install-project: "true" - poetry-version: "2.2.1" + poetry-version: "2.4.1" # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 @@ -187,7 +189,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -196,7 +198,7 @@ jobs: if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 @@ -214,7 +216,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -233,7 +235,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -251,7 +253,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -265,7 +267,7 @@ jobs: # Install like a normal project from source with all optional dependencies extras: all install-project: "true" - poetry-version: "2.2.1" + poetry-version: "2.4.1" - name: Ensure `Cargo.lock` is up to date (no stray changes after install) # The `::error::` syntax is using GitHub Actions' error annotations, see @@ -287,7 +289,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -307,7 +309,7 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -354,7 +356,7 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -375,7 +377,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -398,7 +400,7 @@ jobs: - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.job.python-version }} - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: ${{ matrix.job.extras }} - name: Await PostgreSQL if: ${{ matrix.job.postgres-version }} @@ -431,7 +433,7 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -494,13 +496,13 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.python-version }} - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: ${{ matrix.extras }} - run: poetry run trial --jobs=2 tests - name: Dump logs @@ -544,7 +546,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers @@ -591,11 +593,11 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - run: sudo apt-get -qq install xmlsec1 postgresql-client - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: "postgres" - run: .ci/scripts/test_export_data_command.sh env: @@ -634,7 +636,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -648,7 +650,7 @@ jobs: - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.python-version }} - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: "postgres" - run: .ci/scripts/test_synapse_port_db.sh id: run_tester_script @@ -682,7 +684,7 @@ jobs: - changes steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -702,7 +704,7 @@ jobs: - changes steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index 31dddab012..85d7be7b34 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -22,7 +22,7 @@ jobs: # This field is case-sensitive. TARGET_STATUS: Needs info steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Only clone the script file we care about, instead of the whole repo. sparse-checkout: .ci/scripts/triage_labelled_issue.sh diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 8d26e73e37..1b906f7f44 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -54,7 +54,7 @@ jobs: with: python-version: "3.x" extras: "all" - poetry-version: "2.2.1" + poetry-version: "2.4.1" - run: | poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref || 'trunk' }} @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust @@ -82,7 +82,7 @@ jobs: with: python-version: "3.x" extras: "all test" - poetry-version: "2.2.1" + poetry-version: "2.4.1" - run: | poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk @@ -115,7 +115,7 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -172,7 +172,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index e333f2320b..f9873b6bbd 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ __pycache__/ !/.coveragerc /.coverage* /.mypy_cache/ +/.ruff_cache/ /.tox /.tox-pg-container /build/ diff --git a/CHANGES.md b/CHANGES.md index 13e25709cd..70a9b60e4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +# Synapse 1.155.0 (2026-06-16) + +## End of Life of Debian 12 Bookworm + +The next version of Synapse will not include Debian packages for Debian 12 Bookworm +as it reached end of life on the 10th of June 2026. + +## Internal Changes + +- When building releases, don't cancel Debian package builds when one of them fails. ([\#19842](https://github.com/element-hq/synapse/issues/19842)) + # Synapse 1.155.0rc1 (2026-06-09) ## Bugfixes diff --git a/Cargo.lock b/Cargo.lock index 761e99b3e6..a6d98135cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,9 +373,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -707,9 +707,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lru-slab" @@ -1208,9 +1208,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", diff --git a/changelog.d/19390.bugfix b/changelog.d/19390.bugfix new file mode 100644 index 0000000000..706f779174 --- /dev/null +++ b/changelog.d/19390.bugfix @@ -0,0 +1 @@ +Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. diff --git a/changelog.d/19591.feature b/changelog.d/19591.feature new file mode 100644 index 0000000000..f780030807 --- /dev/null +++ b/changelog.d/19591.feature @@ -0,0 +1 @@ +Expose [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) over [MSC4186 (Simplified) Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186). \ No newline at end of file diff --git a/changelog.d/19660.doc b/changelog.d/19660.doc new file mode 100644 index 0000000000..f03bc6090c --- /dev/null +++ b/changelog.d/19660.doc @@ -0,0 +1 @@ +Update `auto_join_rooms` config documentation to cover requirements for auto-joining invite-only rooms. diff --git a/changelog.d/19732.misc b/changelog.d/19732.misc new file mode 100644 index 0000000000..99fca0a018 --- /dev/null +++ b/changelog.d/19732.misc @@ -0,0 +1 @@ +Refactor `get_user_which_could_invite` logic to reuse `get_users_which_can_issue_invite`. Contributed by Noah Markert. diff --git a/changelog.d/19758.feature b/changelog.d/19758.feature new file mode 100644 index 0000000000..5a360c7695 --- /dev/null +++ b/changelog.d/19758.feature @@ -0,0 +1 @@ +Stabilize support for sending ephemeral events to application services, as per [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409). Contributed by @jason-famedly @ Famedly. diff --git a/changelog.d/19762.feature b/changelog.d/19762.feature new file mode 100644 index 0000000000..543db3184b --- /dev/null +++ b/changelog.d/19762.feature @@ -0,0 +1,2 @@ +Include `allowed_room_ids` in the `/summary` client-server API response for rooms with restricted join rules, as required by Matrix 1.15. +Contributed by @FrenchGithubUser @Famedly. diff --git a/changelog.d/19785.bugfix b/changelog.d/19785.bugfix new file mode 100644 index 0000000000..643481ee27 --- /dev/null +++ b/changelog.d/19785.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where the badge notification count for a room could become permanently inflated if a read receipt was sent before the room's notification counts were first summarised. diff --git a/changelog.d/19788.doc b/changelog.d/19788.doc new file mode 100644 index 0000000000..bfdd3a3d37 --- /dev/null +++ b/changelog.d/19788.doc @@ -0,0 +1 @@ +Add stable endpoint for [MSC3266: Room summary API](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) into worker docs. Contributed by @olmari. \ No newline at end of file diff --git a/changelog.d/19794.feature b/changelog.d/19794.feature new file mode 100644 index 0000000000..be373f9a77 --- /dev/null +++ b/changelog.d/19794.feature @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Allow authentication on delayed event management endpoints (such as `/restart`) to bypass ratelimits for unauthenticated requests based on the client IP address. diff --git a/changelog.d/19810.bugfix b/changelog.d/19810.bugfix new file mode 100644 index 0000000000..72592532f8 --- /dev/null +++ b/changelog.d/19810.bugfix @@ -0,0 +1 @@ +Fix startup listener logging to report the actual bound TCP port, so listeners configured with port `0` no longer log `Synapse now listening on TCP port 0`. diff --git a/changelog.d/19829.doc b/changelog.d/19829.doc new file mode 100644 index 0000000000..44dbc0eff4 --- /dev/null +++ b/changelog.d/19829.doc @@ -0,0 +1 @@ +Tweak wording of Rust crate dependency update policy. \ No newline at end of file diff --git a/changelog.d/19834.bugfix b/changelog.d/19834.bugfix new file mode 100644 index 0000000000..097a09bac6 --- /dev/null +++ b/changelog.d/19834.bugfix @@ -0,0 +1 @@ +Fix notification counts being inflated after a `/purge_history` when notifications had already been rotated into the summary table. diff --git a/changelog.d/19840.misc b/changelog.d/19840.misc new file mode 100644 index 0000000000..db56b65764 --- /dev/null +++ b/changelog.d/19840.misc @@ -0,0 +1 @@ +Upload Complement test logs as CI artifacts instead of printing the raw output to the build log. diff --git a/changelog.d/19843.misc b/changelog.d/19843.misc new file mode 100644 index 0000000000..633c11edb7 --- /dev/null +++ b/changelog.d/19843.misc @@ -0,0 +1 @@ +Fix release script considering any workflow completion as successful. \ No newline at end of file diff --git a/changelog.d/19845.bugfix b/changelog.d/19845.bugfix new file mode 100644 index 0000000000..c2009f0749 --- /dev/null +++ b/changelog.d/19845.bugfix @@ -0,0 +1 @@ +Fix `/sync` caching transient errors for the `sync_response_cache_duration`. \ No newline at end of file diff --git a/changelog.d/19847.doc b/changelog.d/19847.doc new file mode 100644 index 0000000000..225b03d7b3 --- /dev/null +++ b/changelog.d/19847.doc @@ -0,0 +1 @@ +Fixed the Admin API user endpoint documentation examples to use JSON booleans (true/false) instead of numeric (0/1) values. diff --git a/changelog.d/19849.misc b/changelog.d/19849.misc new file mode 100644 index 0000000000..08a3bb0e96 --- /dev/null +++ b/changelog.d/19849.misc @@ -0,0 +1 @@ +Force keyword-args for clear `default_config(server_name="test")` usage in test utilities. diff --git a/changelog.d/19850.bugfix b/changelog.d/19850.bugfix new file mode 100644 index 0000000000..357c2ee3f3 --- /dev/null +++ b/changelog.d/19850.bugfix @@ -0,0 +1 @@ +Fix local events being deleted by the [Purge History admin API](https://element-hq.github.io/synapse/v1.155/admin_api/purge_history_api.html) despite `delete_local_events` being set to false, in room versions other than 1 and 2. \ No newline at end of file diff --git a/changelog.d/19853.feature b/changelog.d/19853.feature new file mode 100644 index 0000000000..b11eaa16d7 --- /dev/null +++ b/changelog.d/19853.feature @@ -0,0 +1,3 @@ +The `GET /_matrix/client/unstable/org.matrix.msc1763/retention/configuration` endpoint is now provided when retention +is enabled and `experimental_features.msc1763_enabled` is enabled, based on +[MSC1763](https://github.com/matrix-org/matrix-spec-proposals/pull/1763). diff --git a/changelog.d/19854.misc b/changelog.d/19854.misc new file mode 100644 index 0000000000..501745ebb1 --- /dev/null +++ b/changelog.d/19854.misc @@ -0,0 +1 @@ +Add `.ruff_cache/` directory to `.gitignore`. diff --git a/changelog.d/19855.bugfix b/changelog.d/19855.bugfix new file mode 100644 index 0000000000..706f779174 --- /dev/null +++ b/changelog.d/19855.bugfix @@ -0,0 +1 @@ +Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. diff --git a/changelog.d/19856.bugfix b/changelog.d/19856.bugfix new file mode 100644 index 0000000000..706f779174 --- /dev/null +++ b/changelog.d/19856.bugfix @@ -0,0 +1 @@ +Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. diff --git a/changelog.d/19866.misc b/changelog.d/19866.misc new file mode 100644 index 0000000000..7bf704e84c --- /dev/null +++ b/changelog.d/19866.misc @@ -0,0 +1 @@ +Bump `poetry` in CI from `2.2.1` to `2.4.1`. \ No newline at end of file diff --git a/changelog.d/19868.misc b/changelog.d/19868.misc new file mode 100644 index 0000000000..cd182ea15c --- /dev/null +++ b/changelog.d/19868.misc @@ -0,0 +1 @@ +Split out `deferred` and `tokio_runtime` to their own Rust modules. diff --git a/changelog.d/19869.bugfix b/changelog.d/19869.bugfix new file mode 100644 index 0000000000..2772db850f --- /dev/null +++ b/changelog.d/19869.bugfix @@ -0,0 +1 @@ +Make simple_select_one_onecol_txn() more helpful by naming the table of the select - as all other query wrapper functions already did. diff --git a/changelog.d/19877.misc b/changelog.d/19877.misc new file mode 100644 index 0000000000..7bf704e84c --- /dev/null +++ b/changelog.d/19877.misc @@ -0,0 +1 @@ +Bump `poetry` in CI from `2.2.1` to `2.4.1`. \ No newline at end of file diff --git a/complement/go.mod b/complement/go.mod index 5ebf19bbf6..dfd8e73f22 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -31,6 +31,7 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 // indirect github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect diff --git a/complement/go.sum b/complement/go.sum index 25c6922669..325ad216c6 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -38,6 +38,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= diff --git a/complement/tests/federation_room_join_test.go b/complement/tests/federation_room_join_test.go new file mode 100644 index 0000000000..78526883e9 --- /dev/null +++ b/complement/tests/federation_room_join_test.go @@ -0,0 +1,187 @@ +package synapse_tests + +import ( + "context" + "io" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/matrix-org/complement" + + "github.com/matrix-org/gomatrixserverlib" + + "github.com/tidwall/gjson" + + "github.com/matrix-org/complement/b" + "github.com/matrix-org/complement/federation" + "github.com/matrix-org/complement/helpers" + "github.com/matrix-org/complement/must" +) + +// This test verifies that events sent into a room between a /make_join and +// /send_join are not lost to the joining server. When an event is created +// during the join handshake, the join event's prev_events (set at make_join +// time) won't reference it, creating two forward extremities. The server +// handling the join should ensure the joining server can discover the missed +// event, for example by sending a follow-up event that references both +// extremities, prompting the joining server to backfill. +// +// See https://github.com/element-hq/synapse/pull/19390 +// +// This test lives as a in-repo Synapse Complement test because the spec doesn't mandate +// which events should be resolvable after the `/make_join`/`/send_join` dance (or that +// a homeserver should send `m.dummy` events to tie things together). +// +// To be clear, resolving the events in the `/make_join`/`/send_join` gap would happen +// naturally as soon as someone else sends an event who is on a homeserver aware of the +// events in the gap (tie it into the DAG). The goal of the automatic dummy events is to +// make this happen more immediately by sending a `m.dummy` event that ties things in +// instead of waiting for another event to be sent naturally. +func TestEventBetweenMakeJoinAndSendJoinIsNotLost(t *testing.T) { + deployment := complement.Deploy(t, 1) + defer deployment.Destroy(t) + + alice := deployment.Register(t, "hs1", helpers.RegistrationOpts{}) + + // We track the message event ID sent between make_join and send_join. + // After send_join, we wait for hs1 to send us either: + // - the message event itself, or + // - any event whose prev_events reference the message (e.g. a dummy event) + // + // atomic.Value is used because messageEventID is written on the main goroutine and + // read on the HTTP handler goroutine, and needs synchronization (without + // synchronization, writes are not guaranteed to be observed by other goroutines) + var messageEventID atomic.Value + messageEventID.Store("") + messageDiscoverableWaiter := helpers.NewWaiter() + + srv := federation.NewServer(t, deployment, + // hs1 fetches our signing keys via /_matrix/key/v2/server to verify our + // identity before accepting federation requests. Without this handler, + // make_join is rejected with 401 M_UNAUTHORIZED. + federation.HandleKeyRequests(), + ) + // After send_join, hs1 will start sending us federation transactions via + // /_matrix/federation/v1/send/{txnID}. Since we handle /send manually + // below, any other requests (e.g. key fetches) that arrive unexpectedly + // should be tolerated rather than treated as test failures. + srv.UnexpectedRequestsAreErrors = false + + // Custom /send handler: hs1 will push new room events to us via federation + // transactions once we've joined. We use a raw handler because the + // Complement server is not fully in the room until send_join completes, so + // we can't use HandleTransactionRequests (which requires the room in + // srv.rooms). Instead we parse the raw transaction body ourselves. + srv.Mux(). + Handle("/_matrix/federation/v1/send/{transactionID}", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, err := io.ReadAll(req.Body) + must.NotError(t, "failed to read request body in /send handler: %v", err) + txn := gjson.ParseBytes(body) + txn.Get("pdus").ForEach(func(_, pdu gjson.Result) bool { + eventID := pdu.Get("event_id").String() + eventType := pdu.Get("type").String() + t.Logf("Received PDU via /send: type=%s id=%s", eventType, eventID) + + // messageEventID is set after make_join but before send_join. + // Transactions can arrive before that window, so skip PDUs that + // arrive before we know which event to look for. + msgID, _ := messageEventID.Load().(string) + if msgID == "" { + return true + } + + // Check if this IS the message event (server pushed it directly). + if eventID == msgID { + messageDiscoverableWaiter.Finish() + return true + } + + // Check if this event's prev_events directly reference the message (e.g. a dummy + // event tying the two forward extremities together). If so, the joining server + // can backfill from that event and will discover the message. + // + // XXX: We only check one level of prev_events: if the reference is deeper in the + // DAG, it's valid and the joining server can still reach the message through + // backfill but our checks don't account for that yet (feel free to edit this + // assertion if you run into this) + pdu.Get("prev_events").ForEach(func(_, prevEvent gjson.Result) bool { + if prevEvent.String() == msgID { + messageDiscoverableWaiter.Finish() + return false + } + return true + }) + + return true + }) + w.WriteHeader(200) + // Respond with an empty PDU error map, which is the federation /send + // success response format: each key would be a PDU ID whose processing + // failed; an empty object means all PDUs were accepted. + _, err = w.Write([]byte(`{"pdus":{}}`)) + must.NotError(t, "failed to write response body in /send handler: %v", err) + })). + Methods("PUT") + + cancel := srv.Listen() + defer cancel() + + // Alice creates a room on hs1. + roomID := alice.MustCreateRoom(t, map[string]interface{}{ + "preset": "public_chat", + }) + + charlie := srv.UserID("charlie") + origin := srv.ServerName() + fedClient := srv.FederationClient(deployment) + + // Step 1: make_join, hs1 returns a join event template whose prev_events + // reflect the current room DAG tips. + makeJoinResp, err := fedClient.MakeJoin( + context.Background(), origin, + deployment.GetFullyQualifiedHomeserverName(t, "hs1"), + roomID, charlie, + ) + must.NotError(t, "MakeJoin", err) + + // Step 2: Alice sends a message on hs1. This advances the DAG past the + // point captured by make_join's prev_events. The Complement server is not + // yet in the room, so it won't receive this event via normal federation. + messageEventID.Store(alice.SendEventSynced(t, roomID, b.Event{ + Type: "m.room.message", + Content: map[string]interface{}{ + "msgtype": "m.text", + "body": "Message sent between make_join and send_join", + }, + })) + t.Logf("Alice sent message %s between make_join and send_join", messageEventID.Load()) + + // Step 3: Build and sign the join event, then send_join. + // The join event's prev_events are from step 1 (before the message), + // so persisting it on hs1 creates two forward extremities: the message + // and the join. + verImpl, err := gomatrixserverlib.GetRoomVersion(makeJoinResp.RoomVersion) + must.NotError(t, "GetRoomVersion", err) + eb := verImpl.NewEventBuilderFromProtoEvent(&makeJoinResp.JoinEvent) + joinEvent, err := eb.Build(time.Now(), srv.ServerName(), srv.KeyID, srv.Priv) + must.NotError(t, "Build join event", err) + + _, err = fedClient.SendJoin( + context.Background(), origin, + deployment.GetFullyQualifiedHomeserverName(t, "hs1"), + joinEvent, + ) + must.NotError(t, "SendJoin", err) + + // Step 4: hs1 should make the missed message discoverable to the joining + // server. We accept either receiving the message event directly, or + // receiving any event whose prev_events reference it (allowing the + // joining server to backfill). + messageDiscoverableWaiter.Waitf(t, 5*time.Second, + "Timed out waiting for message event %s to become discoverable — "+ + "the event sent between make_join and send_join was lost to the "+ + "joining server", messageEventID.Load(), + ) +} diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 7bbf52ddd9..bc689e6caf 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -35,7 +35,7 @@ TEMP_VENV="$(mktemp -d)" python3 -m venv "$TEMP_VENV" source "$TEMP_VENV/bin/activate" pip install -U pip -pip install poetry==2.2.1 poetry-plugin-export==1.9.0 +pip install poetry==2.4.1 poetry-plugin-export==1.9.0 poetry export \ --extras all \ --extras test \ diff --git a/debian/changelog b/debian/changelog index 617bc73611..224916d2f6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +matrix-synapse-py3 (1.155.0+nmu1) UNRELEASED; urgency=medium + + * Bump poetry from 2.2.1 to 2.4.1. + + -- Synapse Packaging team Wed, 24 Jun 2026 13:10:00 +0100 + +matrix-synapse-py3 (1.155.0) stable; urgency=medium + + * New Synapse release 1.155.0. + + -- Synapse Packaging team Tue, 16 Jun 2026 14:44:04 +0100 + matrix-synapse-py3 (1.155.0~rc1) stable; urgency=medium * New Synapse release 1.155.0rc1. diff --git a/docker/Dockerfile b/docker/Dockerfile index 6070d5c355..f395127da0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ ARG DEBIAN_VERSION=trixie ARG PYTHON_VERSION=3.13 -ARG POETRY_VERSION=2.2.1 +ARG POETRY_VERSION=2.4.1 ### ### Stage 0: generate requirements.txt diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 14f86fa976..9e2b756208 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -34,11 +34,11 @@ It returns a JSON body like the following: } ], "avatar_url": "", // can be null if not set - "is_guest": 0, - "admin": 0, - "deactivated": 0, + "is_guest": false, + "admin": false, + "deactivated": false, "erased": false, - "shadow_banned": 0, + "shadow_banned": false, "creation_ts": 1560432506, "last_seen_ts": 1732919539393, "appservice_id": null, @@ -183,24 +183,24 @@ A response body like the following is returned: "users": [ { "name": "", - "is_guest": 0, - "admin": 0, + "is_guest": false, + "admin": false, "user_type": null, - "deactivated": 0, + "deactivated": false, "erased": false, - "shadow_banned": 0, + "shadow_banned": false, "displayname": "", "avatar_url": null, "creation_ts": 1560432668000, "locked": false }, { "name": "", - "is_guest": 0, - "admin": 1, + "is_guest": false, + "admin": true, "user_type": null, - "deactivated": 0, + "deactivated": false, "erased": false, - "shadow_banned": 0, + "shadow_banned": false, "displayname": "", "avatar_url": "", "creation_ts": 1561550621000, diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index 2a4301b4ee..3ce64f01d7 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -79,11 +79,17 @@ concerned about the criteria for selecting minimum versions. The only thing of c is making sure we're not making it unnecessarily difficult for downstream package maintainers. Generally, this just means avoiding the bleeding edge for a few months. -The situation for Rust dependencies is fundamentally different. For packagers, the -concerns around Python dependency versions do not apply. The `cargo` tool handles -downloading and building all libraries to satisfy dependencies, and these libraries are -statically linked into the final binary. This means that from a packager's perspective, -the Rust dependency versions are an internal build detail, not a runtime dependency to -be managed on the target system. Consequently, we have even greater flexibility to -upgrade Rust dependencies as needed for the project. Some distros (e.g. Fedora) do -package Rust libraries, but this appears to be the outlier rather than the norm. +The situation for Rust dependencies is typically different and +the concerns around Python dependency versions typically do not apply. +For example, for packagers of Debian, the packagers have the choice of either +using a crate packaged in the distro, or vendoring crates in the source package for the +application. + +This freedom to vendor a dependency crate for a specific application consequently gives +us even greater flexibility to upgrade Rust dependencies as needed for the project. + +(This is in contrast with Python dependencies, which are generally +installed system-wide by mainstream distributions' official packages.) + +Some distros (e.g. Fedora) do not vendor Rust dependencies in their +official application packages, but these cases appear to be less common. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index d028d65fe3..92eca4a7ff 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1971,7 +1971,7 @@ rc_presence: *(object)* Ratelimiting settings for delayed event management. -This is a ratelimiting option that ratelimits attempts to restart, cancel, or view delayed events based on the sending client's account and device ID. +This is a ratelimiting option that ratelimits attempts to restart, cancel, or view delayed events based on the sending client's account, or its source IP when requests are unauthenticated. Attempts to create or send delayed events are ratelimited not by this setting, but by `rc_message`. @@ -2870,6 +2870,8 @@ enable_3pid_changes: false By default, any room aliases included in this list will be created as a publicly joinable room when the first user registers for the homeserver. If the room already exists, make certain it is a publicly joinable room, i.e. the join rule of the room must be set to `public`. You can find more options relating to auto-joining rooms below. +Invite-only rooms can also be auto-joined when setting `auto_join_mxid_localpart` to a user who's part of the invite-only rooms. + As Spaces are just rooms under the hood, Space aliases may also be used. Defaults to `[]`. diff --git a/docs/workers.md b/docs/workers.md index 8d3aad19c6..d987ff8980 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -239,6 +239,7 @@ information. ^/_matrix/client/(v1|unstable)/rooms/.*/relations/ ^/_matrix/client/v1/rooms/.*/threads$ ^/_matrix/client/unstable/im.nheko.summary/summary/.*$ + ^/_matrix/client/v1/room_summary/.*$ ^/_matrix/client/(r0|v3|unstable)/account/3pid$ ^/_matrix/client/(r0|v3|unstable)/account/whoami$ ^/_matrix/client/(r0|v3|unstable)/account/deactivate$ diff --git a/poetry.lock b/poetry.lock index 4afbaad2c1..21a84142ec 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -531,7 +531,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +556,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +606,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -889,7 +889,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -930,7 +930,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1122,7 +1122,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1239,7 +1239,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, @@ -1547,7 +1547,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1828,7 +1828,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1883,14 +1883,14 @@ tests = ["pytest (>=9)", "typing-extensions (>=4.15)"] [[package]] name = "phonenumbers" -version = "9.0.26" +version = "9.0.28" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = ">=2.5" groups = ["main"] files = [ - {file = "phonenumbers-9.0.26-py2.py3-none-any.whl", hash = "sha256:ff473da5712965b6c7f7a31cbff8255864df694eb48243771133ecb761e807c1"}, - {file = "phonenumbers-9.0.26.tar.gz", hash = "sha256:9e582c827f0f5503cddeebef80099475a52ffa761551d8384099c7ec71298cbf"}, + {file = "phonenumbers-9.0.28-py2.py3-none-any.whl", hash = "sha256:ee8caabab4fd554efb6119e7b95cb69da40e0f04050611730eed839d93f39920"}, + {file = "phonenumbers-9.0.28.tar.gz", hash = "sha256:f1d810aaa43fbf3a5cb1ee54733218f8333a2a92c85c4d579a810403d6260a8c"}, ] [[package]] @@ -2026,7 +2026,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2044,7 +2044,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2060,7 +2060,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2342,7 +2342,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2474,7 +2474,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2499,7 +2499,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2510,14 +2510,14 @@ six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.26" +version = "0.0.27" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "python_multipart-0.0.26-py3-none-any.whl", hash = "sha256:c0b169f8c4484c13b0dcf2ef0ec3a4adb255c4b7d18d8e420477d2b1dd03f185"}, - {file = "python_multipart-0.0.26.tar.gz", hash = "sha256:08fadc45918cd615e26846437f50c5d6d23304da32c341f289a617127b081f17"}, + {file = "python_multipart-0.0.27-py3-none-any.whl", hash = "sha256:6fccfad17a27334bd0193681b369f476eda3409f17381a2d65aa7df3f7275645"}, + {file = "python_multipart-0.0.27.tar.gz", hash = "sha256:9870a6a8c5a20a5bf4f07c017bd1489006ff8836cff097b6933355ee2b49b602"}, ] [[package]] @@ -2527,7 +2527,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2668,14 +2668,14 @@ typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} [[package]] name = "requests" -version = "2.33.0" +version = "2.33.1" description = "Python HTTP for Humans." optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b"}, - {file = "requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652"}, + {file = "requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a"}, + {file = "requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517"}, ] [package.dependencies] @@ -2686,7 +2686,6 @@ urllib3 = ">=1.26,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -test = ["PySocks (>=1.5.6,!=1.5.7)", "pytest (>=3)", "pytest-cov", "pytest-httpbin (==2.1.0)", "pytest-mock", "pytest-xdist"] use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] [[package]] @@ -2932,7 +2931,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3038,14 +3037,14 @@ type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.deve [[package]] name = "setuptools-rust" -version = "1.12.0" +version = "1.12.1" description = "Setuptools Rust extension plugin" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "setuptools_rust-1.12.0-py3-none-any.whl", hash = "sha256:7e7db90547f224a835b45f5ad90c983340828a345554a9a660bdb2de8605dcdd"}, - {file = "setuptools_rust-1.12.0.tar.gz", hash = "sha256:d94a93f0c97751c17014565f07bdc324bee45d396cd1bba83d8e7af92b945f0c"}, + {file = "setuptools_rust-1.12.1-py3-none-any.whl", hash = "sha256:b7ebd6a182e7aefa97a072e880530c9b0ec8fcca8617e0bb8ff299c1a064f693"}, + {file = "setuptools_rust-1.12.1.tar.gz", hash = "sha256:85ae70989d96c9cfeb5ef79cf3bac2d5200bc5564f720a06edceedbdf6664640"}, ] [package.dependencies] @@ -3132,7 +3131,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3148,7 +3147,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3160,60 +3159,61 @@ twisted = ["twisted"] [[package]] name = "tomli" -version = "2.4.0" +version = "2.4.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "tomli-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867"}, - {file = "tomli-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5572e41282d5268eb09a697c89a7bee84fae66511f87533a6f88bd2f7b652da9"}, - {file = "tomli-2.4.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:551e321c6ba03b55676970b47cb1b73f14a0a4dce6a3e1a9458fd6d921d72e95"}, - {file = "tomli-2.4.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e3f639a7a8f10069d0e15408c0b96a2a828cfdec6fca05296ebcdcc28ca7c76"}, - {file = "tomli-2.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1b168f2731796b045128c45982d3a4874057626da0e2ef1fdd722848b741361d"}, - {file = "tomli-2.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:133e93646ec4300d651839d382d63edff11d8978be23da4cc106f5a18b7d0576"}, - {file = "tomli-2.4.0-cp311-cp311-win32.whl", hash = "sha256:b6c78bdf37764092d369722d9946cb65b8767bfa4110f902a1b2542d8d173c8a"}, - {file = "tomli-2.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:d3d1654e11d724760cdb37a3d7691f0be9db5fbdaef59c9f532aabf87006dbaa"}, - {file = "tomli-2.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:cae9c19ed12d4e8f3ebf46d1a75090e4c0dc16271c5bce1c833ac168f08fb614"}, - {file = "tomli-2.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:920b1de295e72887bafa3ad9f7a792f811847d57ea6b1215154030cf131f16b1"}, - {file = "tomli-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d6d9a4aee98fac3eab4952ad1d73aee87359452d1c086b5ceb43ed02ddb16b8"}, - {file = "tomli-2.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36b9d05b51e65b254ea6c2585b59d2c4cb91c8a3d91d0ed0f17591a29aaea54a"}, - {file = "tomli-2.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c8a885b370751837c029ef9bc014f27d80840e48bac415f3412e6593bbc18c1"}, - {file = "tomli-2.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8768715ffc41f0008abe25d808c20c3d990f42b6e2e58305d5da280ae7d1fa3b"}, - {file = "tomli-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b438885858efd5be02a9a133caf5812b8776ee0c969fea02c45e8e3f296ba51"}, - {file = "tomli-2.4.0-cp312-cp312-win32.whl", hash = "sha256:0408e3de5ec77cc7f81960c362543cbbd91ef883e3138e81b729fc3eea5b9729"}, - {file = "tomli-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:685306e2cc7da35be4ee914fd34ab801a6acacb061b6a7abca922aaf9ad368da"}, - {file = "tomli-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:5aa48d7c2356055feef06a43611fc401a07337d5b006be13a30f6c58f869e3c3"}, - {file = "tomli-2.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84d081fbc252d1b6a982e1870660e7330fb8f90f676f6e78b052ad4e64714bf0"}, - {file = "tomli-2.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9a08144fa4cba33db5255f9b74f0b89888622109bd2776148f2597447f92a94e"}, - {file = "tomli-2.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c73add4bb52a206fd0c0723432db123c0c75c280cbd67174dd9d2db228ebb1b4"}, - {file = "tomli-2.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fb2945cbe303b1419e2706e711b7113da57b7db31ee378d08712d678a34e51e"}, - {file = "tomli-2.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bbb1b10aa643d973366dc2cb1ad94f99c1726a02343d43cbc011edbfac579e7c"}, - {file = "tomli-2.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4cbcb367d44a1f0c2be408758b43e1ffb5308abe0ea222897d6bfc8e8281ef2f"}, - {file = "tomli-2.4.0-cp313-cp313-win32.whl", hash = "sha256:7d49c66a7d5e56ac959cb6fc583aff0651094ec071ba9ad43df785abc2320d86"}, - {file = "tomli-2.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:3cf226acb51d8f1c394c1b310e0e0e61fecdd7adcb78d01e294ac297dd2e7f87"}, - {file = "tomli-2.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:d20b797a5c1ad80c516e41bc1fb0443ddb5006e9aaa7bda2d71978346aeb9132"}, - {file = "tomli-2.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:26ab906a1eb794cd4e103691daa23d95c6919cc2fa9160000ac02370cc9dd3f6"}, - {file = "tomli-2.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:20cedb4ee43278bc4f2fee6cb50daec836959aadaf948db5172e776dd3d993fc"}, - {file = "tomli-2.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:39b0b5d1b6dd03684b3fb276407ebed7090bbec989fa55838c98560c01113b66"}, - {file = "tomli-2.4.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a26d7ff68dfdb9f87a016ecfd1e1c2bacbe3108f4e0f8bcd2228ef9a766c787d"}, - {file = "tomli-2.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:20ffd184fb1df76a66e34bd1b36b4a4641bd2b82954befa32fe8163e79f1a702"}, - {file = "tomli-2.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75c2f8bbddf170e8effc98f5e9084a8751f8174ea6ccf4fca5398436e0320bc8"}, - {file = "tomli-2.4.0-cp314-cp314-win32.whl", hash = "sha256:31d556d079d72db7c584c0627ff3a24c5d3fb4f730221d3444f3efb1b2514776"}, - {file = "tomli-2.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:43e685b9b2341681907759cf3a04e14d7104b3580f808cfde1dfdb60ada85475"}, - {file = "tomli-2.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:3d895d56bd3f82ddd6faaff993c275efc2ff38e52322ea264122d72729dca2b2"}, - {file = "tomli-2.4.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5b5807f3999fb66776dbce568cc9a828544244a8eb84b84b9bafc080c99597b9"}, - {file = "tomli-2.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c084ad935abe686bd9c898e62a02a19abfc9760b5a79bc29644463eaf2840cb0"}, - {file = "tomli-2.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f2e3955efea4d1cfbcb87bc321e00dc08d2bcb737fd1d5e398af111d86db5df"}, - {file = "tomli-2.4.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e0fe8a0b8312acf3a88077a0802565cb09ee34107813bba1c7cd591fa6cfc8d"}, - {file = "tomli-2.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:413540dce94673591859c4c6f794dfeaa845e98bf35d72ed59636f869ef9f86f"}, - {file = "tomli-2.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0dc56fef0e2c1c470aeac5b6ca8cc7b640bb93e92d9803ddaf9ea03e198f5b0b"}, - {file = "tomli-2.4.0-cp314-cp314t-win32.whl", hash = "sha256:d878f2a6707cc9d53a1be1414bbb419e629c3d6e67f69230217bb663e76b5087"}, - {file = "tomli-2.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2add28aacc7425117ff6364fe9e06a183bb0251b03f986df0e78e974047571fd"}, - {file = "tomli-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2b1e3b80e1d5e52e40e9b924ec43d81570f0e7d09d11081b797bc4692765a3d4"}, - {file = "tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a"}, - {file = "tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c"}, + {file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"}, + {file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc"}, + {file = "tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049"}, + {file = "tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e"}, + {file = "tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1"}, + {file = "tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917"}, + {file = "tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9"}, + {file = "tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5"}, + {file = "tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd"}, + {file = "tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36"}, + {file = "tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba"}, + {file = "tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6"}, + {file = "tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7"}, + {file = "tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f"}, + {file = "tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8"}, + {file = "tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26"}, + {file = "tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396"}, + {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, + {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, ] +markers = {main = "python_version < \"3.14\""} [[package]] name = "tornado" @@ -3222,7 +3222,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3354,7 +3354,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3502,14 +3502,14 @@ files = [ [[package]] name = "types-requests" -version = "2.32.4.20260107" +version = "2.33.0.20260408" description = "Typing stubs for requests" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "types_requests-2.32.4.20260107-py3-none-any.whl", hash = "sha256:b703fe72f8ce5b31ef031264fe9395cac8f46a04661a79f7ed31a80fb308730d"}, - {file = "types_requests-2.32.4.20260107.tar.gz", hash = "sha256:018a11ac158f801bfa84857ddec1650750e393df8a004a8a9ae2a9bec6fcb24f"}, + {file = "types_requests-2.33.0.20260408-py3-none-any.whl", hash = "sha256:81f31d5ea4acb39f03be7bc8bed569ba6d5a9c5d97e89f45ac43d819b68ca50f"}, + {file = "types_requests-2.33.0.20260408.tar.gz", hash = "sha256:95b9a86376807a216b2fb412b47617b202091c3ea7c078f47cc358d5528ccb7b"}, ] [package.dependencies] @@ -3615,7 +3615,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3636,7 +3636,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, diff --git a/pyproject.toml b/pyproject.toml index 3500b4f516..fa70791f7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.155.0rc1" +version = "1.155.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs new file mode 100644 index 0000000000..91d5a08f7d --- /dev/null +++ b/rust/src/deferred.rs @@ -0,0 +1,138 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::{future::Future, sync::OnceLock}; + +use once_cell::sync::OnceCell; +use pyo3::{create_exception, exceptions::PyException, prelude::*}; + +use crate::tokio_runtime::runtime; + +create_exception!( + synapse.synapse_rust.http_client, + RustPanicError, + PyException, + "A panic which happened in a Rust future" +); + +impl RustPanicError { + fn from_panic(panic_err: &(dyn std::any::Any + Send + 'static)) -> PyErr { + // Apparently this is how you extract the panic message from a panic + let panic_message = if let Some(str_slice) = panic_err.downcast_ref::<&str>() { + str_slice + } else if let Some(string) = panic_err.downcast_ref::() { + string + } else { + "unknown error" + }; + Self::new_err(panic_message.to_owned()) + } +} + +/// A reference to the `twisted.internet.defer` module. +static DEFER: OnceCell> = OnceCell::new(); + +/// Access to the `twisted.internet.defer` module. +fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(DEFER + .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))? + .bind(py)) +} + +/// Creates a twisted deferred from the given future, spawning the task on the +/// tokio runtime. +/// +/// Does not handle deferred cancellation or contextvars. +pub fn create_deferred<'py, F, O>( + py: Python<'py>, + reactor: &Bound<'py, PyAny>, + fut: F, +) -> PyResult> +where + F: Future> + Send + 'static, + for<'a> O: IntoPyObject<'a> + Send + 'static, +{ + let deferred = defer(py)?.call_method0("Deferred")?; + let deferred_callback = deferred.getattr("callback")?.unbind(); + let deferred_errback = deferred.getattr("errback")?.unbind(); + + let rt = runtime(reactor)?; + let handle = rt.handle()?; + let task = handle.spawn(fut); + + // Unbind the reactor so that we can pass it to the task + let reactor = reactor.clone().unbind(); + handle.spawn(async move { + let res = task.await; + + Python::attach(move |py| { + // Flatten the panic into standard python error + let res = match res { + Ok(r) => r, + Err(join_err) => match join_err.try_into_panic() { + Ok(panic_err) => Err(RustPanicError::from_panic(&panic_err)), + Err(err) => Err(PyException::new_err(format!("Task cancelled: {err}"))), + }, + }; + + // Re-bind the reactor + let reactor = reactor.bind(py); + + // Send the result to the deferred, via `.callback(..)` or `.errback(..)` + match res { + Ok(obj) => { + reactor + .call_method("callFromThread", (deferred_callback, obj), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + Err(err) => { + reactor + .call_method("callFromThread", (deferred_errback, err), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + } + }); + }); + + // Make the deferred follow the Synapse logcontext rules + make_deferred_yieldable(py, &deferred) +} + +static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); + +/// Given a deferred, make it follow the Synapse logcontext rules +fn make_deferred_yieldable<'py>( + py: Python<'py>, + deferred: &Bound<'py, PyAny>, +) -> PyResult> { + let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { + let sys = PyModule::import(py, "synapse.logging.context").unwrap(); + let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); + func + }); + + make_deferred_yieldable + .call1(py, (deferred,))? + .extract(py) + .map_err(Into::into) +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, _m: &Bound<'_, PyModule>) -> PyResult<()> { + // Make sure we fail early if we can't load some modules + defer(py)?; + + Ok(()) +} diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index 398ba9041f..aaa1066a67 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -12,145 +12,22 @@ * . */ -use std::{collections::HashMap, future::Future, sync::OnceLock}; +use std::collections::HashMap; use anyhow::Context; use http_body_util::BodyExt; -use once_cell::sync::OnceCell; -use pyo3::{create_exception, exceptions::PyException, prelude::*}; +use pyo3::prelude::*; use reqwest::RequestBuilder; -use tokio::runtime::Runtime; +use crate::deferred::create_deferred; use crate::errors::HttpResponseException; - -create_exception!( - synapse.synapse_rust.http_client, - RustPanicError, - PyException, - "A panic which happened in a Rust future" -); - -impl RustPanicError { - fn from_panic(panic_err: &(dyn std::any::Any + Send + 'static)) -> PyErr { - // Apparently this is how you extract the panic message from a panic - let panic_message = if let Some(str_slice) = panic_err.downcast_ref::<&str>() { - str_slice - } else if let Some(string) = panic_err.downcast_ref::() { - string - } else { - "unknown error" - }; - Self::new_err(panic_message.to_owned()) - } -} - -/// This is the name of the attribute where we store the runtime on the reactor -static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; - -/// A Python wrapper around a Tokio runtime. -/// -/// This allows us to 'store' the runtime on the reactor instance, starting it -/// when the reactor starts, and stopping it when the reactor shuts down. -#[pyclass] -struct PyTokioRuntime { - runtime: Option, -} - -#[pymethods] -impl PyTokioRuntime { - fn start(&mut self) -> PyResult<()> { - // TODO: allow customization of the runtime like the number of threads - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .enable_all() - .build()?; - - self.runtime = Some(runtime); - - Ok(()) - } - - fn shutdown(&mut self) -> PyResult<()> { - let runtime = self - .runtime - .take() - .context("Runtime was already shutdown")?; - - // Dropping the runtime will shut it down - drop(runtime); - - Ok(()) - } -} - -impl PyTokioRuntime { - /// Get the handle to the Tokio runtime, if it is running. - fn handle(&self) -> PyResult<&tokio::runtime::Handle> { - let handle = self - .runtime - .as_ref() - .context("Tokio runtime is not running")? - .handle(); - - Ok(handle) - } -} - -/// Get a handle to the Tokio runtime stored on the reactor instance, or create -/// a new one. -fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { - install_runtime(reactor)?; - } - - get_runtime(reactor) -} - -/// Install a new Tokio runtime on the reactor instance. -fn install_runtime(reactor: &Bound) -> PyResult<()> { - let py = reactor.py(); - let runtime = PyTokioRuntime { runtime: None }; - let runtime = runtime.into_pyobject(py)?; - - // Attach the runtime to the reactor, starting it when the reactor is - // running, stopping it when the reactor is shutting down - reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; - reactor.call_method1( - "addSystemEventTrigger", - ("after", "shutdown", runtime.getattr("shutdown")?), - )?; - reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; - - Ok(()) -} - -/// Get a reference to a Tokio runtime handle stored on the reactor instance. -fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is - // not a `Runtime`. Careful that this could happen if the user sets it - // manually, or if multiple versions of `pyo3-twisted` are used! - let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; - Ok(runtime.borrow()) -} - -/// A reference to the `twisted.internet.defer` module. -static DEFER: OnceCell> = OnceCell::new(); - -/// Access to the `twisted.internet.defer` module. -fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { - Ok(DEFER - .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))? - .bind(py)) -} +use crate::tokio_runtime::runtime; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module: Bound<'_, PyModule> = PyModule::new(py, "http_client")?; child_module.add_class::()?; - // Make sure we fail early if we can't load some modules - defer(py)?; - m.add_submodule(&child_module)?; // We need to manually add the module to sys.modules to make `from @@ -262,81 +139,3 @@ impl HttpClient { }) } } - -/// Creates a twisted deferred from the given future, spawning the task on the -/// tokio runtime. -/// -/// Does not handle deferred cancellation or contextvars. -fn create_deferred<'py, F, O>( - py: Python<'py>, - reactor: &Bound<'py, PyAny>, - fut: F, -) -> PyResult> -where - F: Future> + Send + 'static, - for<'a> O: IntoPyObject<'a> + Send + 'static, -{ - let deferred = defer(py)?.call_method0("Deferred")?; - let deferred_callback = deferred.getattr("callback")?.unbind(); - let deferred_errback = deferred.getattr("errback")?.unbind(); - - let rt = runtime(reactor)?; - let handle = rt.handle()?; - let task = handle.spawn(fut); - - // Unbind the reactor so that we can pass it to the task - let reactor = reactor.clone().unbind(); - handle.spawn(async move { - let res = task.await; - - Python::attach(move |py| { - // Flatten the panic into standard python error - let res = match res { - Ok(r) => r, - Err(join_err) => match join_err.try_into_panic() { - Ok(panic_err) => Err(RustPanicError::from_panic(&panic_err)), - Err(err) => Err(PyException::new_err(format!("Task cancelled: {err}"))), - }, - }; - - // Re-bind the reactor - let reactor = reactor.bind(py); - - // Send the result to the deferred, via `.callback(..)` or `.errback(..)` - match res { - Ok(obj) => { - reactor - .call_method("callFromThread", (deferred_callback, obj), None) - .expect("callFromThread should not fail"); // There's nothing we can really do with errors here - } - Err(err) => { - reactor - .call_method("callFromThread", (deferred_errback, err), None) - .expect("callFromThread should not fail"); // There's nothing we can really do with errors here - } - } - }); - }); - - // Make the deferred follow the Synapse logcontext rules - make_deferred_yieldable(py, &deferred) -} - -static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); - -/// Given a deferred, make it follow the Synapse logcontext rules -fn make_deferred_yieldable<'py>( - py: Python<'py>, - deferred: &Bound<'py, PyAny>, -) -> PyResult> { - let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { - let sys = PyModule::import(py, "synapse.logging.context").unwrap(); - let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); - func - }); - - make_deferred_yieldable - .call1(py, (deferred,))? - .extract(py) - .map_err(Into::into) -} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index bf96422cbb..eb2d8cecdd 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -6,6 +6,7 @@ use pyo3_log::ResetHandle; pub mod acl; pub mod canonical_json; +pub mod deferred; pub mod duration; pub mod errors; pub mod events; @@ -19,6 +20,7 @@ pub mod push; pub mod rendezvous; pub mod room_versions; pub mod segmenter; +pub mod tokio_runtime; pub mod types; lazy_static! { @@ -65,6 +67,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?; acl::register_module(py, m)?; + deferred::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; http_client::register_module(py, m)?; diff --git a/rust/src/tokio_runtime.rs b/rust/src/tokio_runtime.rs new file mode 100644 index 0000000000..f239040f30 --- /dev/null +++ b/rust/src/tokio_runtime.rs @@ -0,0 +1,107 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use anyhow::Context; +use pyo3::prelude::*; +use tokio::runtime::Runtime; + +/// This is the name of the attribute where we store the runtime on the reactor +static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; + +/// A Python wrapper around a Tokio runtime. +/// +/// This allows us to 'store' the runtime on the reactor instance, starting it +/// when the reactor starts, and stopping it when the reactor shuts down. +#[pyclass] +pub struct PyTokioRuntime { + runtime: Option, +} + +#[pymethods] +impl PyTokioRuntime { + fn start(&mut self) -> PyResult<()> { + // TODO: allow customization of the runtime like the number of threads + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build()?; + + self.runtime = Some(runtime); + + Ok(()) + } + + fn shutdown(&mut self) -> PyResult<()> { + let runtime = self + .runtime + .take() + .context("Runtime was already shutdown")?; + + // Dropping the runtime will shut it down + drop(runtime); + + Ok(()) + } +} + +impl PyTokioRuntime { + /// Get the handle to the Tokio runtime, if it is running. + pub fn handle(&self) -> PyResult<&tokio::runtime::Handle> { + let handle = self + .runtime + .as_ref() + .context("Tokio runtime is not running")? + .handle(); + + Ok(handle) + } +} + +/// Get a handle to the Tokio runtime stored on the reactor instance, or create +/// a new one. +pub fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { + install_runtime(reactor)?; + } + + get_runtime(reactor) +} + +/// Install a new Tokio runtime on the reactor instance. +fn install_runtime(reactor: &Bound) -> PyResult<()> { + let py = reactor.py(); + let runtime = PyTokioRuntime { runtime: None }; + let runtime = runtime.into_pyobject(py)?; + + // Attach the runtime to the reactor, starting it when the reactor is + // running, stopping it when the reactor is shutting down + reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; + reactor.call_method1( + "addSystemEventTrigger", + ("after", "shutdown", runtime.getattr("shutdown")?), + )?; + reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; + + Ok(()) +} + +/// Get a reference to a Tokio runtime handle stored on the reactor instance. +fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is + // not a `Runtime`. Careful that this could happen if the user sets it + // manually, or if multiple versions of `pyo3-twisted` are used! + let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; + Ok(runtime.borrow()) +} diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 1a2caddbfd..f58b923add 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2244,8 +2244,8 @@ properties: This is a ratelimiting option that ratelimits attempts to restart, cancel, - or view delayed events based on the sending client's account and device - ID. + or view delayed events based on the sending client's account, + or its source IP when requests are unauthenticated. Attempts to create or send delayed events are ratelimited not by this @@ -3227,6 +3227,10 @@ properties: options relating to auto-joining rooms below. + Invite-only rooms can also be auto-joined when setting `auto_join_mxid_localpart` + to a user who's part of the invite-only rooms. + + As Spaces are just rooms under the hood, Space aliases may also be used. items: type: string diff --git a/scripts-dev/release.py b/scripts-dev/release.py index f429f7e048..ea4fb0f142 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -611,18 +611,28 @@ def _wait_for_actions(gh_token: str | None) -> None: if len(resp["workflow_runs"]) == 0: continue - if all( - workflow["status"] != "in_progress" for workflow in resp["workflow_runs"] - ): - success = all( - workflow["status"] == "completed" for workflow in resp["workflow_runs"] - ) - if success: - _notify("Workflows successful. You can now continue the release.") - else: - _notify("Workflows failed.") - click.confirm("Continue anyway?", abort=True) + # Notify early if any workflow run has already failed. + failed_workflows = [ + workflow + for workflow in resp["workflow_runs"] + if workflow["status"] == "completed" and workflow["conclusion"] != "success" + ] + if failed_workflows: + for workflow in failed_workflows: + print( + f"Workflow run failed ({workflow['conclusion']}): {workflow['name']}" + ) + print(f" see {workflow['html_url']}") + _notify("A workflow run has failed.") + click.confirm("Continue anyway?", abort=True) + break + # If every run has completed successfully, we are done. + if all( + workflow["status"] == "completed" and workflow["conclusion"] == "success" + for workflow in resp["workflow_runs"] + ): + _notify("Workflows successful. You can now continue the release.") break diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 55df4c382e..4814370684 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -497,7 +497,8 @@ def listen_http( reactor=reactor, ) logger.info( - "Synapse now listening on TCP port %d (TLS)", listener_config.port + "Synapse now listening on TCP port %d (TLS)", + ports[0].getHost().port, ) else: ports = listen_tcp( @@ -507,7 +508,7 @@ def listen_http( reactor=reactor, ) logger.info( - "Synapse now listening on TCP port %d", listener_config.port + "Synapse now listening on TCP port %d", ports[0].getHost().port ) elif isinstance(listener_config, UnixListenerConfig): diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 620aa29dfc..c55a83a879 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -101,6 +101,7 @@ class ApplicationService: rate_limited: bool = True, ip_range_whitelist: IPSet | None = None, supports_ephemeral: bool = False, + supports_unstable_ephemeral: bool = False, msc3202_transaction_extensions: bool = False, msc4190_device_management: bool = False, ): @@ -125,6 +126,7 @@ class ApplicationService: self.namespaces = self._check_namespaces(namespaces) self.id = id self.ip_range_whitelist = ip_range_whitelist + self.supports_unstable_ephemeral = supports_unstable_ephemeral self.supports_ephemeral = supports_ephemeral self.msc3202_transaction_extensions = msc3202_transaction_extensions self.msc4190_device_management = msc4190_device_management diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index cae0101d52..5f6fb4971c 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -357,8 +357,21 @@ class ApplicationServiceApi(SimpleHttpClient): if service.supports_ephemeral: body.update( { - # TODO: Update to stable prefixes once MSC2409 completes FCP merge. + "ephemeral": ephemeral, + } + ) + if service.supports_unstable_ephemeral: + body.update( + { "de.sorunome.msc2409.ephemeral": ephemeral, + } + ) + if service.supports_ephemeral or service.supports_unstable_ephemeral: + body.update( + { + # TODO: Update to stable prefixes once MSC4203 completes FCP merge. + # Previously, this was part of MSC2409 which is why it has the + # mismatched unstable identifier "de.sorunome.msc2409.to_device": to_device_messages, } ) diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index b9ed1a702c..7a629d10bf 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -169,7 +169,12 @@ def _load_appservice( if as_info.get("ip_range_whitelist"): ip_range_whitelist = IPSet(as_info.get("ip_range_whitelist")) - supports_ephemeral = as_info.get("de.sorunome.msc2409.push_ephemeral", False) + # TODO: remove push_ephemeral handling at some point in the future. It was part of + # MSC2409 which changed the identifier near the end of the review cycle. + supports_unstable_ephemeral = as_info.get( + "de.sorunome.msc2409.push_ephemeral", False + ) + supports_ephemeral = as_info.get("receive_ephemeral", False) # Opt-in flag for the MSC3202-specific transactional behaviour. # When enabled, appservice transactions contain the following information: @@ -204,6 +209,7 @@ def _load_appservice( protocols=protocols, rate_limited=rate_limited, ip_range_whitelist=ip_range_whitelist, + supports_unstable_ephemeral=supports_unstable_ephemeral, supports_ephemeral=supports_ephemeral, msc3202_transaction_extensions=msc3202_transaction_extensions, msc4190_device_management=msc4190_enabled, diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index c958a278fc..120b31cee5 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -380,6 +380,9 @@ class ExperimentalConfig(Config): ) -> None: experimental = config.get("experimental_features") or {} + # MSC1763 (retention policy configuration endpoint) + self.msc1763_enabled: bool = experimental.get("msc1763_enabled", False) + # MSC3026 (busy presence state) self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False) @@ -389,6 +392,8 @@ class ExperimentalConfig(Config): # MSC2409 (this setting only relates to optionally sending to-device messages). # Presence, typing and read receipt EDUs are already sent to application services that # have opted in to receive them. If enabled, this adds to-device messages to that list. + # This is also for MSC4203 which was broken off of MSC2409 but kept the same unstable + # identifier. self.msc2409_to_device_messages_enabled: bool = experimental.get( "msc2409_to_device_messages_enabled", False ) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 6069287975..2fc78c012e 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -816,6 +816,15 @@ class FederationServer(FederationBase): event, context = await self._on_send_membership_event( origin, content, Membership.JOIN, room_id ) + # Use the join event's own stream ordering as the upper bound when fetching + # forward extremities (below), so we only consider extremities that existed at + # or before the join rather than those introduced by concurrent writes that + # occur while we prepare the response. + # Note: in workers mode the event is persisted on a separate worker, so + # event.internal_metadata.stream_ordering is not populated here; query the DB. + stream_ordering_of_join = ( + await self.store.get_position_for_event(event.event_id) + ).stream prev_state_ids = await context.get_prev_state_ids() @@ -856,6 +865,30 @@ class FederationServer(FederationBase): "members_omitted": caller_supports_partial_state, } + # Check the forward extremities for the room here. If there is more than one, it + # is likely that another event was created in the room during the + # make_join/send_join handshake. The joining server is likely to thus miss this event + # until a second event is created that references it - which could be some time. + # In that case, we proactively send a dummy extensible event that ties these + # forward extremities together. The remote server will then attempt to backfill + # the missing event on its own. + # + # By not sending the 'missing event' directly, but instead having the joining + # homeserver backfill it, the stream ordering for the missing event will be + # "before" the join (which is what we expect). + + forward_extremities = ( + await self.store.get_forward_extremities_for_room_at_stream_ordering( + room_id, stream_ordering_of_join + ) + ) + + if len(forward_extremities) > 1: + # The likelihood of this being used is extremely low, thus only build the handler + # when necessary. + _creation_handler = self.hs.get_event_creation_handler() + await _creation_handler._send_dummy_event_after_room_join(room_id) + if servers_in_room is not None: resp["servers_in_room"] = list(servers_in_room) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index c91d2adbe1..36b2f63e41 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -255,9 +255,8 @@ class ApplicationServicesHandler: will cause this function to return early. Ephemeral events will only be pushed to appservices that have opted into - receiving them by setting `push_ephemeral` to true in their registration - file. Note that while MSC2409 is experimental, this option is called - `de.sorunome.msc2409.push_ephemeral`. + receiving them by setting `receive_ephemeral` to true in their + registration file(previously this was `push_ephemeral`). Appservices will only receive ephemeral events that fall within their registered user and room namespaces. @@ -270,7 +269,6 @@ class ApplicationServicesHandler: # Notify appservices of updates in ephemeral event streams. # Only the following streams are currently supported. - # FIXME: We should use constants for these values. if stream_key not in ( StreamKeyType.TYPING, StreamKeyType.RECEIPT, diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 4a9f646d4d..f016d95e31 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -36,6 +36,7 @@ from synapse.storage.databases.main.delayed_events import ( ) from synapse.storage.databases.main.state_deltas import StateDelta from synapse.types import ( + Absent, JsonDict, Requester, RoomID, @@ -45,7 +46,6 @@ from synapse.types import ( from synapse.util.duration import Duration from synapse.util.events import generate_fake_event_id from synapse.util.metrics import Measure -from synapse.util.sentinel import Sentinel if TYPE_CHECKING: from synapse.server import HomeServer @@ -61,13 +61,13 @@ class DelayedEventsHandler: self._storage_controllers = hs.get_storage_controllers() self._config = hs.config self._clock = hs.get_clock() + self._auth = hs.get_auth() self._event_creation_handler = hs.get_event_creation_handler() self._room_member_handler = hs.get_room_member_handler() self._request_ratelimiter = hs.get_request_ratelimiter() - # Ratelimiter for management of existing delayed events, - # keyed by the sending user ID & device ID. + # Ratelimiter for management of existing delayed events self._delayed_event_mgmt_ratelimiter = Ratelimiter( store=self._store, clock=self._clock, @@ -273,9 +273,7 @@ class DelayedEventsHandler: ) continue - sender_str = event_id_and_sender_dict.get( - delta.event_id, Sentinel.UNSET_SENTINEL - ) + sender_str = event_id_and_sender_dict.get(delta.event_id, Absent) if sender_str is None: # An event exists, but the `sender` field was "null" and Synapse # incorrectly accepted the event. This is not expected. @@ -285,7 +283,7 @@ class DelayedEventsHandler: delta.event_id, ) continue - if sender_str is Sentinel.UNSET_SENTINEL: + if sender_str is Absent: # We have an event ID, but the event was not found in the # datastore. This can happen if a room, or its history, is # purged. State deltas related to the room are left behind, but @@ -413,9 +411,7 @@ class DelayedEventsHandler: NotFoundError: if no matching delayed event could be found. """ assert self._is_master - await self._delayed_event_mgmt_ratelimiter.ratelimit( - None, request.getClientAddress().host - ) + await self._mgmt_ratelimit(request) await make_deferred_yieldable(self._initialized_from_db) next_send_ts = await self._store.cancel_delayed_event(delay_id) @@ -430,9 +426,7 @@ class DelayedEventsHandler: Raises: NotFoundError: if no matching delayed event could be found. """ - await self._delayed_event_mgmt_ratelimiter.ratelimit( - None, request.getClientAddress().host - ) + await self._mgmt_ratelimit(request) # Note: We don't need to wait on `self._initialized_from_db` here as the # events that deals with are already marked as processed. @@ -456,9 +450,7 @@ class DelayedEventsHandler: NotFoundError: if no matching delayed event could be found. """ assert self._is_master - await self._delayed_event_mgmt_ratelimiter.ratelimit( - None, request.getClientAddress().host - ) + await self._mgmt_ratelimit(request) await make_deferred_yieldable(self._initialized_from_db) event, next_send_ts = await self._store.process_target_delayed_event(delay_id) @@ -468,6 +460,19 @@ class DelayedEventsHandler: await self._send_event(event) + async def _mgmt_ratelimit(self, request: SynapseRequest) -> None: + """ + Ratelimit requests with the `_delayed_event_mgmt_ratelimiter` keyed on the + user making the request, or the request's IP address if unauthed. + """ + if self._auth.has_access_token(request): + requester = await self._auth.get_user_by_req(request) + key = None + else: + requester = None + key = request.getClientAddress().host + await self._delayed_event_mgmt_ratelimiter.ratelimit(requester, key) + async def _send_on_timeout(self) -> None: self._next_delayed_event_call = None @@ -527,10 +532,7 @@ class DelayedEventsHandler: async def get_all_for_user(self, requester: Requester) -> list[JsonDict]: """Return all pending delayed events requested by the given user.""" - await self._delayed_event_mgmt_ratelimiter.ratelimit( - requester, - (requester.user.to_string(), requester.device_id), - ) + await self._delayed_event_mgmt_ratelimiter.ratelimit(requester) return await self._store.get_all_delayed_events_for_user( requester.user.localpart ) diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index daa8dd5eba..a9a6990581 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -23,8 +23,6 @@ from typing import TYPE_CHECKING, Mapping from synapse import event_auth from synapse.api.constants import ( - CREATOR_POWER_LEVEL, - EventContentFields, EventTypes, JoinRules, Membership, @@ -38,6 +36,7 @@ from synapse.event_auth import ( ) from synapse.events import EventBase from synapse.events.builder import EventBuilder +from synapse.handlers.room_member import get_users_which_can_issue_invite from synapse.types import StateMap, StrCollection if TYPE_CHECKING: @@ -143,53 +142,20 @@ class EventAuthHandler: Raises: SynapseError if no appropriate user is found. """ - create_event_id = current_state_ids[(EventTypes.Create, "")] - create_event = await self._store.get_event(create_event_id) - power_level_event_id = current_state_ids.get((EventTypes.PowerLevels, "")) - invite_level = 0 - users_default_level = 0 - if power_level_event_id: - power_level_event = await self._store.get_event(power_level_event_id) - invite_level = power_level_event.content.get("invite", invite_level) - users_default_level = power_level_event.content.get( - "users_default", users_default_level - ) - users = power_level_event.content.get("users", {}) - else: - users = {} - - # Find the user with the highest power level (only interested in local - # users). - user_power_level = 0 - chosen_user = None local_users_in_room = await self._store.get_local_users_in_room(room_id) - if create_event.room_version.msc4289_creator_power_enabled: - creators = set( - create_event.content.get(EventContentFields.ADDITIONAL_CREATORS, []) - ) - creators.add(create_event.sender) - local_creators = creators.intersection(set(local_users_in_room)) - if len(local_creators) > 0: - chosen_user = local_creators.pop() # random creator - user_power_level = CREATOR_POWER_LEVEL - if chosen_user is None: - chosen_user = max( - local_users_in_room, - key=lambda user: users.get(user, users_default_level), - default=None, - ) - # Return the chosen if they can issue invites. - if chosen_user: - user_power_level = users.get(chosen_user, users_default_level) + current_state = await self._store.get_events(current_state_ids.values()) + auth_events = { + state_key: event + for state_key, event_id in current_state_ids.items() + if (event := current_state.get(event_id)) is not None + } - if chosen_user and user_power_level >= invite_level: - logger.debug( - "Found a user who can issue invites %s with power level %d >= invite level %d", - chosen_user, - user_power_level, - invite_level, - ) - return chosen_user + users_which_can_invite = get_users_which_can_issue_invite(auth_events) + local_users_which_can_invite = set(users_which_can_invite).intersection( + local_users_in_room + ) + if local_users_which_can_invite: + return local_users_which_can_invite.pop() # No user was found. raise SynapseError( diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9785e8efc0..b34ee9d50f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -2294,7 +2294,32 @@ class EventCreationHandler: now = self.clock.time_msec() self._rooms_to_exclude_from_dummy_event_insertion[room_id] = now - async def _send_dummy_event_for_room(self, room_id: str) -> bool: + async def _send_dummy_event_after_room_join(self, room_id: str) -> None: + """ + Creates and sends a dummy event into the given room, referencing the + current forward extremities (via `prev_events`). + This should only be triggered when handling a remote join while events + were sent during the make_join/send_join handshake. The joining + homeserver would otherwise not immediately know to backfill those events + and would "miss" them. + """ + async with self._worker_lock_handler.acquire_read_write_lock( + NEW_EVENT_DURING_PURGE_LOCK_NAME, room_id, write=False + ): + dummy_event_sent = await self._send_dummy_event_for_room( + room_id, proactively_send=True + ) + + if not dummy_event_sent: + logger.warning( + "Failed to send dummy event into room %s after remote join; " + "no local user with permission was found", + room_id, + ) + + async def _send_dummy_event_for_room( + self, room_id: str, proactively_send: bool = False + ) -> bool: """Attempt to send a dummy event for the given room. Args: @@ -2326,8 +2351,7 @@ class EventCreationHandler: }, ) context = await unpersisted_context.persist(event) - - event.internal_metadata.proactively_send = False + event.internal_metadata.proactively_send = proactively_send # Since this is a dummy-event it is OK if it is sent by a # shadow-banned user. diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index bbcdc0877e..0bc6021fa7 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -513,7 +513,7 @@ class RoomSummaryHandler: ): return None - room_entry = await self._build_room_entry(room_id, for_federation=bool(origin)) + room_entry = await self._build_room_entry(room_id) # If the room is not a space return just the room information. if room_entry.get("room_type") != RoomTypes.SPACE or not include_children: @@ -769,14 +769,12 @@ class RoomSummaryHandler: # pending invite, etc. return await self._is_local_room_accessible(room_id, requester) - async def _build_room_entry(self, room_id: str, for_federation: bool) -> JsonDict: + async def _build_room_entry(self, room_id: str) -> JsonDict: """ Generate en entry summarising a single room. Args: room_id: The room ID to summarize. - for_federation: True if this is a summary requested over federation - (which includes additional fields). Returns: The JSON dictionary for the room. @@ -805,24 +803,30 @@ class RoomSummaryHandler: "encryption": stats.encryption, } - # Federation requests need to provide additional information so the - # requested server is able to filter the response appropriately. - if for_federation: - current_state_ids = ( - await self._storage_controllers.state.get_current_state_ids(room_id) + # Include allowed_room_ids for rooms with restricted join rules so that + # clients can determine which memberships grant access. + # Only the join rules event is needed for both has_restricted_join_rules + # and get_rooms_that_allow_join, so avoid fetching full state. + join_rules_state_ids = ( + await self._storage_controllers.state.get_current_state_ids( + room_id, + state_filter=StateFilter.from_types([(EventTypes.JoinRules, "")]), ) - room_version = await self._store.get_room_version(room_id) + ) - if await self._event_auth_handler.has_restricted_join_rules( - current_state_ids, room_version - ): - allowed_rooms = ( - await self._event_auth_handler.get_rooms_that_allow_join( - current_state_ids - ) - ) - if allowed_rooms: - entry["allowed_room_ids"] = allowed_rooms + try: + room_version = await self._store.get_room_version(room_id) + except UnsupportedRoomVersionError: + room_version = None + + if room_version and await self._event_auth_handler.has_restricted_join_rules( + join_rules_state_ids, room_version + ): + allowed_rooms = await self._event_auth_handler.get_rooms_that_allow_join( + join_rules_state_ids + ) + if allowed_rooms: + entry["allowed_room_ids"] = allowed_rooms # Filter out Nones – rather omit the field altogether room_entry = {k: v for k, v in entry.items() if v is not None} @@ -932,7 +936,6 @@ class RoomSummaryHandler: raise NotFoundError("Room not found or is not accessible") room = dict(room_entry.room) - room.pop("allowed_room_ids", None) # If there was a requester, add their membership. # We keep the membership in the local membership table unless the diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index e7f0e7340b..10ca3ddea0 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -287,7 +287,6 @@ class SlidingSyncHandler: lists = interested_rooms.lists relevant_room_map = interested_rooms.relevant_room_map - all_rooms = interested_rooms.all_rooms room_membership_for_user_map = interested_rooms.room_membership_for_user_map relevant_rooms_to_send_map = interested_rooms.relevant_rooms_to_send_map @@ -328,6 +327,7 @@ class SlidingSyncHandler: actual_lists=lists, previous_connection_state=previous_connection_state, new_connection_state=new_connection_state, + all_interested_room_ids=interested_rooms.all_rooms, # We're purposely using `relevant_room_map` instead of # `relevant_rooms_to_send_map` here. This needs to be all room_ids we could # send regardless of whether they have an event update or not. The @@ -350,7 +350,7 @@ class SlidingSyncHandler: if from_token: # The set of rooms that the client (may) care about, but aren't # in any list range (or subscribed to). - missing_rooms = all_rooms - relevant_room_map.keys() + missing_rooms = interested_rooms.all_rooms - relevant_room_map.keys() # We now just go and try fetching any events in the above rooms # to see if anything has happened since the `from_token`. diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 4a324e9661..b3342de778 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -11,7 +11,6 @@ # See the GNU Affero General Public License for more details: # . # - import itertools import logging from collections import ChainMap @@ -26,11 +25,13 @@ from typing import ( from typing_extensions import TypeAlias, assert_never -from synapse.api.constants import AccountDataTypes, EduTypes +from synapse.api.constants import AccountDataTypes, EduTypes, StickyEvent +from synapse.events.utils import FilteredEvent from synapse.handlers.receipts import ReceiptEventSource from synapse.logging.opentracing import trace from synapse.storage.databases.main.receipts import ReceiptInRoom from synapse.types import ( + Absent, DeviceListUpdates, JsonMapping, MultiWriterStreamToken, @@ -47,10 +48,12 @@ from synapse.types.handlers.sliding_sync import ( SlidingSyncConfig, SlidingSyncResult, ) +from synapse.types.rest.client import SlidingSyncStickyEventsToken from synapse.util.async_helpers import ( concurrently_execute, gather_optional_coroutines, ) +from synapse.visibility import filter_and_transform_events_for_client _ThreadSubscription: TypeAlias = ( SlidingSyncResult.Extensions.ThreadSubscriptionsExtension.ThreadSubscription @@ -73,7 +76,10 @@ class SlidingSyncExtensionHandler: self.event_sources = hs.get_event_sources() self.device_handler = hs.get_device_handler() self.push_rules_handler = hs.get_push_rules_handler() + self.clock = hs.get_clock() + self._storage_controllers = hs.get_storage_controllers() self._enable_thread_subscriptions = hs.config.experimental.msc4306_enabled + self._enable_sticky_events = hs.config.experimental.msc4354_enabled @trace async def get_extensions_response( @@ -81,6 +87,7 @@ class SlidingSyncExtensionHandler: sync_config: SlidingSyncConfig, previous_connection_state: "PerConnectionState", new_connection_state: "MutablePerConnectionState", + all_interested_room_ids: set[str], actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], actual_room_ids: set[str], actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], @@ -91,9 +98,12 @@ class SlidingSyncExtensionHandler: Args: sync_config: Sync configuration - new_connection_state: Snapshot of the current per-connection state - new_per_connection_state: A mutable copy of the per-connection + previous_connection_state: Snapshot of the current per-connection state + new_connection_state: A mutable copy of the per-connection state, used to record updates to the state during this request. + all_interested_room_ids: The IDs of all rooms that the client is interested in, + even if they don't appear in the current limited window. + See `SlidingSyncInterestedRooms.all_rooms`. actual_lists: Sliding window API. A map of list key to list results in the Sliding Sync response. actual_room_ids: The actual room IDs in the the Sliding Sync response. @@ -174,6 +184,19 @@ class SlidingSyncExtensionHandler: from_token=from_token, ) + sticky_events_coro = None + if ( + sync_config.extensions.sticky_events is not Absent + and self._enable_sticky_events + ): + sticky_events_coro = self.get_sticky_events_extension_response( + sync_config=sync_config, + sticky_events_request=sync_config.extensions.sticky_events, + all_interested_room_ids=all_interested_room_ids, + to_token=to_token, + from_token=from_token, + ) + ( to_device_response, e2ee_response, @@ -181,6 +204,7 @@ class SlidingSyncExtensionHandler: receipts_response, typing_response, thread_subs_response, + sticky_events_response, ) = await gather_optional_coroutines( to_device_coro, e2ee_coro, @@ -188,6 +212,7 @@ class SlidingSyncExtensionHandler: receipts_coro, typing_coro, thread_subs_coro, + sticky_events_coro, ) return SlidingSyncResult.Extensions( @@ -197,6 +222,7 @@ class SlidingSyncExtensionHandler: receipts=receipts_response, typing=typing_response, thread_subscriptions=thread_subs_response, + sticky_events=sticky_events_response, ) def find_relevant_room_ids_for_extension( @@ -967,3 +993,65 @@ class SlidingSyncExtensionHandler: unsubscribed=unsubscribed_threads, prev_batch=prev_batch, ) + + async def get_sticky_events_extension_response( + self, + sync_config: SlidingSyncConfig, + sticky_events_request: SlidingSyncConfig.Extensions.StickyEventsExtension, + all_interested_room_ids: set[str], + to_token: StreamToken, + from_token: SlidingSyncStreamToken | None, + ) -> SlidingSyncResult.Extensions.StickyEventsExtension | None: + if not sticky_events_request.enabled: + return None + now = self.clock.time_msec() + # If there is no `since` token specified, start from the beginning of the stream + # to make sure the client receives all visible (unexpired) sticky events + since_token = sticky_events_request.since or SlidingSyncStickyEventsToken.START + ( + sticky_events_to_id, + room_to_event_ids, + ) = await self.store.get_sticky_events_in_rooms( + all_interested_room_ids, + from_id=since_token.sticky_events_stream_id, + to_id=to_token.sticky_events_key, + now=now, + limit=min(sticky_events_request.limit, StickyEvent.MAX_EVENTS_IN_SYNC), + ) + # No need to preserve sticky event order here because we will + # reassemble it in the right order after. + all_sticky_event_ids = { + ev_id for evs in room_to_event_ids.values() for ev_id in evs + } + unfiltered_events = await self.store.get_events_as_list(all_sticky_event_ids) + filtered_events = await filter_and_transform_events_for_client( + self._storage_controllers, + sync_config.user.to_string(), + unfiltered_events, + # As per MSC4354: + # > History visibility checks MUST NOT be applied to sticky events. + # > Any joined user is authorised to see sticky events for the duration they remain sticky. + always_include_ids=frozenset(all_sticky_event_ids), + ) + filtered_event_map = {ev.event.event_id: ev for ev in filtered_events} + + room_id_to_sticky_events: dict[str, list[FilteredEvent]] = {} + for room_id, sticky_event_ids in room_to_event_ids.items(): + filtered_events_for_room = [ + filtered_event_map[event_id] + # This reintroduces the correct order + # (by the sticky events stream) + for event_id in sticky_event_ids + if event_id in filtered_event_map + ] + if len(filtered_events_for_room) == 0: + continue + + room_id_to_sticky_events[room_id] = filtered_events_for_room + + return SlidingSyncResult.Extensions.StickyEventsExtension( + room_id_to_sticky_events=room_id_to_sticky_events, + next_batch=SlidingSyncStickyEventsToken( + sticky_events_stream_id=sticky_events_to_id + ), + ) diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 216ef3b071..836cee6c20 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -52,6 +52,7 @@ from synapse.storage.roommember import ( RoomsForUserStateReset, ) from synapse.types import ( + Absent, MutableStateMap, RoomStreamToken, StateMap, @@ -71,7 +72,6 @@ from synapse.types.handlers.sliding_sync import ( from synapse.types.state import StateFilter from synapse.util import MutableOverlayMapping from synapse.util.duration import Duration -from synapse.util.sentinel import Sentinel if TYPE_CHECKING: from synapse.server import HomeServer @@ -112,31 +112,55 @@ class SlidingSyncInterestedRooms: sliding sync request. Returned by `compute_interested_rooms`. - - Attributes: - lists: A mapping from list name to the list result for the response - relevant_room_map: A map from rooms that match the sync request to - their room sync config. - relevant_rooms_to_send_map: Subset of `relevant_room_map` that - includes the rooms that *may* have relevant updates. Rooms not - in this map will definitely not have room updates (though - extensions may have updates in these rooms). - newly_joined_rooms: The set of rooms that were joined in the token range - and the user is still joined to at the end of this range. - newly_left_rooms: The set of rooms that we left in the token range - and are still "leave" at the end of this range. - dm_room_ids: The set of rooms the user consider as direct-message (DM) rooms """ lists: Mapping[str, SlidingSyncResult.SlidingWindowList] + """ + A mapping from list name to the list result for the response + """ + relevant_room_map: Mapping[str, RoomSyncConfig] + """ + A map from rooms that match the sync request to + their room sync config. + """ + relevant_rooms_to_send_map: Mapping[str, RoomSyncConfig] + """ + Subset of `relevant_room_map` that + includes the rooms that *may* have relevant updates. Rooms not + in this map will definitely not have room updates (though + extensions may have updates in these rooms). + """ + all_rooms: set[str] + """ + The set of room IDs of all rooms that could appear in any list. + This set includes rooms that are outside the list ranges. + In other words, this is the set of all rooms that the client is + _interested_ in (in a pure sense), + even if these rooms are omitted from the current window (which + is, in a sense, just a computational optimisation). + """ + room_membership_for_user_map: Mapping[str, RoomsForUserType] newly_joined_rooms: AbstractSet[str] + """ + The set of rooms that were joined in the token range + and the user is still joined to at the end of this range. + """ + newly_left_rooms: AbstractSet[str] + """ + The set of rooms that we left in the token range + and are still "leave" at the end of this range. + """ + dm_room_ids: AbstractSet[str] + """ + The set of rooms the user consider as direct-message (DM) rooms + """ @staticmethod def empty() -> "SlidingSyncInterestedRooms": @@ -1703,10 +1727,8 @@ class SlidingSyncRoomLists: # (applies to invite/knock rooms) rooms_ids_without_stripped_state: set[str] = set() for room_id in room_ids_without_results: - stripped_state_map = room_id_to_stripped_state_map.get( - room_id, Sentinel.UNSET_SENTINEL - ) - assert stripped_state_map is not Sentinel.UNSET_SENTINEL, ( + stripped_state_map = room_id_to_stripped_state_map.get(room_id, Absent) + assert stripped_state_map is not Absent, ( f"Stripped state left unset for room {room_id}. " + "Make sure you're calling `_bulk_get_stripped_state_for_rooms_from_sync_room_map(...)` " + "with that room_id. (this is a problem with Synapse itself)" diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 947be24d3e..3341e49b85 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -142,6 +142,8 @@ from synapse.storage.background_updates import ( from synapse.storage.database import DatabasePool, LoggingTransaction from synapse.storage.databases.main.roommember import ProfileInfo from synapse.types import ( + Absent, + AbsentType, DomainSpecificString, JsonDict, JsonMapping, @@ -160,7 +162,6 @@ from synapse.util.caches.descriptors import CachedFunction, cached as _cached from synapse.util.clock import Clock from synapse.util.duration import Duration from synapse.util.frozenutils import freeze -from synapse.util.sentinel import Sentinel if TYPE_CHECKING: # Old versions don't have `LiteralString` @@ -1990,7 +1991,7 @@ class ModuleApi: self, user_id: UserID, new_displayname: str, - deactivation: bool | Sentinel = Sentinel.UNSET_SENTINEL, + deactivation: bool | AbsentType = Absent, ) -> None: """Sets a user's display name. @@ -2020,7 +2021,7 @@ class ModuleApi: """ requester = create_requester(user_id) - if deactivation is not Sentinel.UNSET_SENTINEL: + if deactivation is not Absent: logger.error( "Deprecated `deactivation` parameter passed to `set_displayname` Module API (value: %r). This will break in 2027.", deactivation, diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index fe66494d82..a56a81a8e9 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -57,6 +57,7 @@ from synapse.rest.client import ( relations, rendezvous, reporting, + retention, room, room_keys, room_upgrade_rest_servlet, @@ -107,6 +108,7 @@ CLIENT_SERVLET_FUNCTIONS: tuple[RegisterServletsFunc, ...] = ( tags.register_servlets, account_data.register_servlets, reporting.register_servlets, + retention.register_servlets, openid.register_servlets, notifications.register_servlets, devices.register_servlets, diff --git a/synapse/rest/client/retention.py b/synapse/rest/client/retention.py new file mode 100644 index 0000000000..922b6cfb69 --- /dev/null +++ b/synapse/rest/client/retention.py @@ -0,0 +1,102 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from http import HTTPStatus +from typing import TYPE_CHECKING + +from typing_extensions import TypedDict + +from synapse.http.server import HttpServer +from synapse.http.servlet import RestServlet +from synapse.http.site import SynapseRequest + +from ._base import client_patterns + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +class RetentionPolicyDict(TypedDict, total=False): + min_lifetime: int + max_lifetime: int + + +class LifetimeBoundsDict(TypedDict, total=False): + min: int + max: int + + +class RetentionLimitsDict(TypedDict, total=False): + max_lifetime: LifetimeBoundsDict + + +class RetentionConfigurationResponse(TypedDict): + policies: dict[str, RetentionPolicyDict] + limits: RetentionLimitsDict + + +class RetentionConfigurationServlet(RestServlet): + """Implements MSC1763: /_matrix/client/unstable/org.matrix.msc1763/retention/configuration""" + + PATTERNS = client_patterns( + "/org.matrix.msc1763/retention/configuration$", + releases=[], + v1=False, + unstable=True, + ) + CATEGORY = "Client API requests" + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self._retention_config = hs.config.retention + + async def on_GET( + self, request: SynapseRequest + ) -> tuple[int, RetentionConfigurationResponse]: + await self.auth.get_user_by_req(request) + + default_policy: RetentionPolicyDict = {} + if self._retention_config.retention_default_min_lifetime is not None: + default_policy["min_lifetime"] = ( + self._retention_config.retention_default_min_lifetime + ) + if self._retention_config.retention_default_max_lifetime is not None: + default_policy["max_lifetime"] = ( + self._retention_config.retention_default_max_lifetime + ) + + max_lifetime_limits: LifetimeBoundsDict = {} + if self._retention_config.retention_allowed_lifetime_min is not None: + max_lifetime_limits["min"] = ( + self._retention_config.retention_allowed_lifetime_min + ) + if self._retention_config.retention_allowed_lifetime_max is not None: + max_lifetime_limits["max"] = ( + self._retention_config.retention_allowed_lifetime_max + ) + + limits: RetentionLimitsDict = {} + if max_lifetime_limits: + limits["max_lifetime"] = max_lifetime_limits + + policies: dict[str, RetentionPolicyDict] = {} + if default_policy: + policies["*"] = default_policy + + return HTTPStatus.OK, {"policies": policies, "limits": limits} + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + if hs.config.retention.retention_enabled and hs.config.experimental.msc1763_enabled: + RetentionConfigurationServlet(hs).register(http_server) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 0488f3c8f7..6776c7172b 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -20,7 +20,7 @@ # import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, Mapping +from typing import TYPE_CHECKING, Any, Literal, Mapping import attr @@ -671,6 +671,7 @@ class SlidingSyncRestServlet(RestServlet): - receipts (MSC3960) - account data (MSC3959) - thread subscriptions (MSC4308) + - sticky events (MSC4354) Request query parameters: timeout: How long to wait for new events in milliseconds. @@ -894,7 +895,7 @@ class SlidingSyncRestServlet(RestServlet): requester, sliding_sync_result.rooms ) response["extensions"] = await self.encode_extensions( - requester, sliding_sync_result.extensions + requester, sliding_sync_result.extensions, sliding_sync_result.rooms ) return response @@ -1044,8 +1045,18 @@ class SlidingSyncRestServlet(RestServlet): @trace_with_opname("sliding_sync.encode_extensions") async def encode_extensions( - self, requester: Requester, extensions: SlidingSyncResult.Extensions + self, + requester: Requester, + extensions: SlidingSyncResult.Extensions, + ref_rooms_results: Mapping[str, SlidingSyncResult.RoomResult], ) -> JsonDict: + """ + Args: + ref_rooms_results: + Map of room ID -> RoomResult that was serialised as the `room` section + of the Sliding Sync response. + Will not be mutated, only used for reading. + """ serialized_extensions: JsonDict = {} if extensions.to_device is not None: @@ -1114,8 +1125,80 @@ class SlidingSyncRestServlet(RestServlet): _serialise_thread_subscriptions(extensions.thread_subscriptions) ) + if extensions.sticky_events: + serialized_extensions[ + "org.matrix.msc4354.sticky_events" + ] = await self._serialise_sticky_events( + requester, extensions.sticky_events, ref_rooms_results + ) + return serialized_extensions + async def _serialise_sticky_events( + self, + requester: Requester, + sticky_events: SlidingSyncResult.Extensions.StickyEventsExtension, + ref_rooms_results: Mapping[str, SlidingSyncResult.RoomResult], + ) -> JsonDict: + """ + Serialise the sticky events extension response. + + This includes deduplicating by filtering out sticky events + from this extension that already appeared in the timeline + section. + + Args: + ref_rooms_results: + Map of room ID -> RoomResult that was serialised as the `room` section + of the Sliding Sync response. + Will not be mutated, only used for reading. + """ + + time_now = self.clock.time_msec() + # Same as SSS timelines. + # + serialize_options = SerializeEventConfig( + event_format=format_event_for_client_v2_without_room_id, + requester=requester, + ) + + rooms_out: dict[str, dict[Literal["events"], list[JsonDict]]] = {} + for ( + room_id, + possibly_duplicated_sticky_events, + ) in sticky_events.room_id_to_sticky_events.items(): + # As per MSC4354: + # Remove sticky events that are already in the timeline, else we will needlessly duplicate + # events. + # There is no purpose in including sticky events in the sticky section if they're already in + # the timeline, as either way the client becomes aware of them. + # This is particularly important given the risk of sticky events spam since + # anyone can send sticky events, so halving the bandwidth on average for each sticky + # event is helpful. + room_result = ref_rooms_results.get(room_id) + if room_result is None: + # Nothing to deduplicate + sticky_events_to_write = possibly_duplicated_sticky_events + else: + sent_event_ids_in_room_section = { + ev.event.event_id for ev in room_result.timeline_events + } + sticky_events_to_write = [ + ev + for ev in possibly_duplicated_sticky_events + if ev.event.event_id not in sent_event_ids_in_room_section + ] + rooms_out[room_id] = { + "events": await self.event_serializer.serialize_events( + sticky_events_to_write, time_now, config=serialize_options + ) + } + + return { + "rooms": rooms_out, + "next_batch": sticky_events.next_batch.serialise(), + } + def _serialise_thread_subscriptions( thread_subscriptions: SlidingSyncResult.Extensions.ThreadSubscriptionsExtension, diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 39c0631d50..023014276b 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -1879,7 +1879,7 @@ class DatabasePool: if allow_none: return None else: - raise StoreError(404, "No row found") + raise StoreError(404, f"No row found ({table})") @staticmethod def simple_select_onecol_txn( diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index a66caa672c..9c5fd35906 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -1509,6 +1509,43 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas "last_receipt_stream_ordering": stream_ordering, }, ) + # If no summary row exists yet for a thread that has pending push + # actions (room active but not yet through a rotation cycle), the + # UPDATE above is a silent no-op for that thread and + # last_receipt_stream_ordering is never persisted. + # _rotate_notifs_before_txn would then INSERT the row with + # last_receipt_stream_ordering=NULL, causing the badge query to + # include every event before the receipt as unread. Pre-populate + # rows for every thread with pending push actions so rotation + # only counts events that arrive after this receipt. + txn.execute( + """ + SELECT DISTINCT thread_id + FROM event_push_actions + WHERE user_id = ? AND room_id = ? + """, + (user_id, room_id), + ) + pending_thread_ids = [row[0] for row in txn] + self.db_pool.simple_upsert_many_txn( + txn, + table="event_push_summary", + key_names=("user_id", "room_id", "thread_id"), + key_values=[ + (user_id, room_id, pending_thread_id) + for pending_thread_id in pending_thread_ids + ], + value_names=( + "notif_count", + "unread_count", + "stream_ordering", + "last_receipt_stream_ordering", + ), + value_values=[ + (0, 0, old_rotate_stream_ordering, stream_ordering) + for _ in pending_thread_ids + ], + ) # For a threaded receipt, we *always* want to update that receipt, # event if there are no new notifications in that thread. This ensures @@ -1517,8 +1554,10 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas unread_counts = [(0, 0, thread_id)] # Then any updated threads get their notification count and unread - # count updated. - self.db_pool.simple_update_many_txn( + # count updated. Use upsert so that a row is created if none exists + # yet (same race as the unthreaded case above: without this, rotation + # would INSERT with last_receipt_stream_ordering=NULL). + self.db_pool.simple_upsert_many_txn( txn, table="event_push_summary", key_names=("room_id", "user_id", "thread_id"), diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index fe8079c201..1accf207be 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -210,7 +210,7 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): should_delete_expr = "state_events.state_key IS NULL" should_delete_params: tuple[Any, ...] = () if not delete_local_events: - should_delete_expr += " AND event_id NOT LIKE ?" + should_delete_expr += " AND sender NOT LIKE ?" # We include the parameter twice since we use the expression twice should_delete_params += ("%:" + self.hs.hostname, "%:" + self.hs.hostname) @@ -326,6 +326,65 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): ")" % (table,) ) + # Some of the `event_push_actions` we're about to delete may have already + # been rotated into the aggregate `event_push_summary` counts. Deleting + # the rows without adjusting those counts would leave the summary + # over-counting, inflating users' notification counts. So first work out + # how much of each summary is attributable to the events being deleted and + # decrement it. + # + # We only count rows that have already been rotated into the summary: + # those at or before the rotated-up-to position + # (`event_push_summary_stream_ordering`) and after the receipt used to + # compute the summary. Rows beyond that position aren't in the summary + # yet (they're still counted live from `event_push_actions`), so deleting + # them needs no adjustment here. This mirrors how rotation counts them in + # `_rotate_notifs_before_txn`. + logger.info("[purge] adjusting event_push_summary for deleted events") + txn.execute( + """ + SELECT epa.user_id, epa.thread_id, + COUNT(CASE WHEN epa.notif = 1 THEN 1 END), + COUNT(CASE WHEN epa.unread = 1 THEN 1 END) + FROM event_push_actions AS epa + INNER JOIN event_push_summary AS eps USING (user_id, room_id, thread_id) + WHERE epa.room_id = ? + AND epa.event_id IN ( + SELECT event_id FROM events_to_purge WHERE should_delete + ) + AND epa.stream_ordering <= ( + SELECT stream_ordering FROM event_push_summary_stream_ordering + ) + AND ( + eps.last_receipt_stream_ordering IS NULL + OR epa.stream_ordering > eps.last_receipt_stream_ordering + ) + GROUP BY epa.user_id, epa.thread_id + """, + (room_id,), + ) + summary_decrements = cast(list[tuple[str, str, int, int]], txn.fetchall()) + + # `unread_count` is nullable, so `COALESCE` it before subtracting (else + # the result would be NULL). Clamp both counts at 0 via `GREATEST`/`MAX` + # to guard against ever driving a count negative if the summary is + # somehow out of sync with `event_push_actions`. + greatest_func = ( + "GREATEST" if isinstance(self.database_engine, PostgresEngine) else "MAX" + ) + txn.execute_batch( + f""" + UPDATE event_push_summary + SET notif_count = {greatest_func}(notif_count - ?, 0), + unread_count = {greatest_func}(COALESCE(unread_count, 0) - ?, 0) + WHERE room_id = ? AND user_id = ? AND thread_id = ? + """, + [ + (notif_count, unread_count, room_id, user_id, thread_id) + for user_id, thread_id, notif_count, unread_count in summary_decrements + ], + ) + # event_push_actions lacks an index on event_id, and has one on # (room_id, event_id) instead. for table in ("event_push_actions",): diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index f2b1c3e42b..a6fc806701 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -27,8 +27,10 @@ from enum import Enum from typing import ( TYPE_CHECKING, AbstractSet, + Annotated, Any, ClassVar, + Final, Literal, Mapping, Match, @@ -41,8 +43,12 @@ from typing import ( overload, ) +import annotated_types import attr +import pydantic_core.core_schema from immutabledict import immutabledict +from pydantic import GetCoreSchemaHandler, StrictInt +from pydantic_core import CoreSchema from signedjson.key import decode_verify_key_bytes from signedjson.types import VerifyKey from typing_extensions import Self @@ -109,6 +115,110 @@ StrCollection = tuple[str, ...] | list[str] | AbstractSet[str] StrSequence = tuple[str, ...] | list[str] +class AbsentType(Enum): + """ + Type of a sentinel to use as an alternative to `None` + for when we really mean 'absent' and not JSON null. + + Generally suitable for distinguishing a default state from user-suppliable values. + Has no meaning on its own. + + It is falsy (like None is), so shorthand forms like `x or 0` can be used. + """ + + # Making this an Enum member makes this compatible with type narrowing, + # meaning `x is not Absent` will narrow `x: int | AbsentType` to `x: int` etc. + _Absent = object() + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: object, handler: GetCoreSchemaHandler + ) -> CoreSchema: + """ + This function is checked for and used by Pydantic when + attempting to deserialise/validate a field of this type. + + As the `Absent` type has no valid value when deserialising + from JSON (as that's the point; `Absent` is a marker representing + a lack of any JSON value), we always reject any value. + Instead of deserialising from this type, we rely on the struct class + we are in having field defaults that provide an `Absent`, which does not + go through the JSON validation. + + When validating Python, we accept the absent marker itself. + """ + + def _reject_from_json(v: object) -> "AbsentType": + """ + Reject the JSON value, no matter what it is, since absent values + are meant to be ... absent, thus have nothing they can be deserialised + from. + """ + raise ValueError("AbsentType cannot be deserialized from JSON") + + # `json_or_python_schema` wrapper needed for Pydantic < 2.10 + # but can be replaced with just the `is_instance_schema` after that version. + return pydantic_core.core_schema.json_or_python_schema( + json_schema=pydantic_core.core_schema.no_info_plain_validator_function( + _reject_from_json + ), + python_schema=pydantic_core.core_schema.is_instance_schema(cls), + ) + + def __copy__(self) -> "AbsentType": + """ + Copy implementation used by `copy.copy()`. + Always use the same instance. + + Without this and the deep version `__deepcopy__`, + `copy.copy(Absent)` on Python 3.10 (olddeps) + had a problem where it tried to construct a new Absent + as part of a deepcopy operation and resulted in: + ValueError: is not a valid AbsentType + """ + return self + + def __deepcopy__(self, memo: object) -> "AbsentType": + """ + Copy implementation used by `copy.deepcopy()`. + Always use the same instance. + """ + return self + + def __bool__(self) -> Literal[False]: + return False + + def __str__(self) -> str: + return "Absent" + + def __repr__(self) -> str: + return "Absent" + + +Absent: Final = AbsentType._Absent +""" +Sentinel to use as an alternative to `None` +for when we really mean 'absent' and not JSON null. + +Generally suitable for distinguishing a default state from user-suppliable values. +Has no meaning on its own. + +It is falsy (like None is), so shorthand forms like `x or 0` can be used. + +(Previously known as `Sentinel.UNSET_SENTINEL`.) +""" + + +NonNegativeStrictInt = Annotated[StrictInt, annotated_types.Ge(0)] +"""A strict integer that must be greater than or equal to zero. + +Should be preferred in place of Pydantic's own (lax) NonNegativeInt, +which will coerce strings to integers in a way that does not agree with +the Matrix specification (and would risk backing us into a backward compatibility +hole where we had to support input forms we didn't intend). +""" + + # Note that this seems to require inheriting *directly* from Interface in order # for mypy-zope to realize it is an interface. class ISynapseThreadlessReactor( diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index e73533d296..dd913250ba 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -48,7 +48,7 @@ from synapse.types import ( ThreadSubscriptionsToken, UserID, ) -from synapse.types.rest.client import SlidingSyncBody +from synapse.types.rest.client import SlidingSyncBody, SlidingSyncStickyEventsToken from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -424,12 +424,31 @@ class SlidingSyncResult: or bool(self.prev_batch) ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class StickyEventsExtension: + """The Sticky Events extension (MSC4354) + + Attributes: + room_id_to_sticky_events: map (room_id -> [unexpired_sticky_events]) + The events are ordered by the sticky events stream. + + The events haven't yet been deduplicated to remove + events that also appear in the timeline. + """ + + room_id_to_sticky_events: Mapping[str, list[FilteredEvent]] + next_batch: SlidingSyncStickyEventsToken + + def __bool__(self) -> bool: + return bool(self.room_id_to_sticky_events) + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None receipts: ReceiptsExtension | None = None typing: TypingExtension | None = None thread_subscriptions: ThreadSubscriptionsExtension | None = None + sticky_events: StickyEventsExtension | None = None def __bool__(self) -> bool: """Are there any updates that should be returned immediately to @@ -441,6 +460,7 @@ class SlidingSyncResult: or self.receipts or self.typing or self.thread_subscriptions + or self.sticky_events ) next_pos: SlidingSyncStreamToken diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index 49782b5234..a7cb4d8b08 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -18,9 +18,14 @@ # [This file includes modifications made by New Vector Limited] # # +import re +from typing import ClassVar + +import pydantic_core.core_schema from pydantic import ( ConfigDict, Field, + GetCoreSchemaHandler, StrictBool, StrictInt, StrictStr, @@ -28,9 +33,10 @@ from pydantic import ( field_validator, model_validator, ) -from pydantic_core import PydanticCustomError +from pydantic_core import CoreSchema, PydanticCustomError from typing_extensions import Annotated, Self +from synapse.types import Absent, AbsentType, NonNegativeStrictInt from synapse.types.rest import RequestBodyModel from synapse.util.threepids import validate_email @@ -107,6 +113,82 @@ class MsisdnRequestTokenBody(ThreepidRequestTokenBody): phone_number: StrictStr +class SlidingSyncStickyEventsToken: + """ + A token returned by `next_batch` of the MSC4354 Sticky Events extension to Sliding Sync + and then accepted as the `since` parameter in the requests of the same extension. + + Current format: + SlidingSyncStickyEventsToken ::= 'sticky_' DIGIT+ + DIGIT ::= '0'-'9' + + The `sticky_` prefix allows us to make sure it's not swapped for another token + or to evolve the type of token accepted with backwards compatibility in the future. + """ + + PATTERN = re.compile(r"^sticky_([0-9]+)$") + START: ClassVar["SlidingSyncStickyEventsToken"] + + def __init__(self, *, sticky_events_stream_id: int) -> None: + # FIXME: We should use MultiWriterStreamToken here + # Track: https://github.com/element-hq/synapse/issues/19661 + self.sticky_events_stream_id = sticky_events_stream_id + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: object, handler: GetCoreSchemaHandler + ) -> CoreSchema: + """ + This function is checked for and used by Pydantic when + attempting to deserialise/validate a field of this type. + + This returns a schema that will parse a string into an + instance of `SlidingSyncStickyEventsToken`. + """ + + return pydantic_core.core_schema.no_info_plain_validator_function( + cls._validate, + serialization=pydantic_core.core_schema.plain_serializer_function_ser_schema( + cls.serialise, + info_arg=False, + ), + ) + + @classmethod + def _validate(cls, v: object) -> Self: + """ + Create an instance from serialised string form. + + The inverse of `serialise`. + """ + if isinstance(v, cls): + return v + if isinstance(v, str): + match = cls.PATTERN.match(v) + if match is None: + raise ValueError(f"Invalid SlidingSyncStickyEventsToken format: {v!r}") + return cls(sticky_events_stream_id=int(match.group(1))) + raise ValueError(f"Cannot parse SlidingSyncStickyEventsToken from {type(v)}") + + def serialise(self) -> str: + """ + Convert this instance to string. + + The inverse of `_validate`. + """ + return f"sticky_{self.sticky_events_stream_id}" + + def __repr__(self) -> str: + # Use the serialised form as debug output. + return self.serialise() + + +# Starting reading a stream at 0 ensures all stream fact rows will be read +SlidingSyncStickyEventsToken.START = SlidingSyncStickyEventsToken( + sticky_events_stream_id=0 +) + + class SlidingSyncBody(RequestBodyModel): """ Sliding Sync API request body. @@ -383,6 +465,19 @@ class SlidingSyncBody(RequestBodyModel): enabled: StrictBool | None = False limit: StrictInt = 100 + class StickyEventsExtension(RequestBodyModel): + """The Sticky Events extension (MSC4354) + + Attributes: + enabled + limit: maximum number of sticky events to return in the extension (default 100) + since: either a string with the Sticky Events since token or absent + """ + + enabled: StrictBool = False + limit: NonNegativeStrictInt = 100 + since: SlidingSyncStickyEventsToken | AbsentType = Absent + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None @@ -391,6 +486,9 @@ class SlidingSyncBody(RequestBodyModel): thread_subscriptions: ThreadSubscriptionsExtension | None = Field( None, alias="io.element.msc4308.thread_subscriptions" ) + sticky_events: StickyEventsExtension | AbsentType = Field( + Absent, alias="org.matrix.msc4354.sticky_events" + ) conn_id: StrictStr | None = None lists: ( diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 1c3cd48a9c..5211bcc8e1 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -98,6 +98,8 @@ class ObservableDeferred(Generic[_T], AbstractObservableDeferred[_T]): deferred. If consumeErrors is true errors will be captured from the origin deferred. + In that case, errors will NOT be propagated onto any errbacks added to this + `ObservableDeferred`; instead the success callbacks will be called with `None`. Cancelling or otherwise resolving an observer will not affect the original ObservableDeferred. @@ -388,6 +390,7 @@ T3 = TypeVar("T3") T4 = TypeVar("T4") T5 = TypeVar("T5") T6 = TypeVar("T6") +T7 = TypeVar("T7") @overload @@ -517,6 +520,30 @@ async def gather_optional_coroutines( ) -> tuple[T1 | None, T2 | None, T3 | None, T4 | None, T5 | None, T6 | None]: ... +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + tuple[ + Coroutine[Any, Any, T1] | None, + Coroutine[Any, Any, T2] | None, + Coroutine[Any, Any, T3] | None, + Coroutine[Any, Any, T4] | None, + Coroutine[Any, Any, T5] | None, + Coroutine[Any, Any, T6] | None, + Coroutine[Any, Any, T7] | None, + ] + ], +) -> tuple[ + T1 | None, + T2 | None, + T3 | None, + T4 | None, + T5 | None, + T6 | None, + T7 | None, +]: ... + + async def gather_optional_coroutines( *coroutines: Unpack[tuple[Coroutine[Any, Any, T1] | None, ...]], ) -> tuple[T1 | None, ...]: diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 70cea9b77c..afa5ca20c1 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -32,6 +32,7 @@ from typing import ( import attr from twisted.internet import defer +from twisted.python.failure import Failure from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import ( @@ -226,14 +227,19 @@ class ResponseCache(Generic[KV]): Returns: The cache entry object. """ - result = ObservableDeferred(deferred, consumeErrors=True) + result = ObservableDeferred( + deferred, + # We set `consumeErrors=False` as we want to handle errors ourselves (`on_fail`) instead of + # replacing them with a `None` successful result that would go to `on_succeed` + consumeErrors=False, + ) key = context.cache_key entry = ResponseCacheEntry( result, opentracing_span_context, cancellable=cancellable ) self._result_cache[key] = entry - def on_complete(r: RV) -> RV: + def on_succeed(r: RV) -> RV: # if this cache has a non-zero timeout, and the callback has not cleared # the should_cache bit, we leave it in the cache for now and schedule # its removal later. @@ -254,10 +260,29 @@ class ResponseCache(Generic[KV]): self.unset(key) return r - # make sure we do this *after* adding the entry to result_cache, - # in case the result is already complete (in which case flipping the order would - # leave us with a stuck entry in the cache). - result.addBoth(on_complete) + def on_fail(failure: Failure) -> None: + """ + If the deferred fails, unset the cache entry. + """ + self.unset(key) + + # Consider the Failure handled so they don't get thrown by the reactor + return None + + # Two ordering constraints to be aware of for registering the callback and errback: + # 1. Both of them must be registered _after_ adding the entry to the result_cache, + # otherwise it is possible for them to trigger immediately (before the entry + # is added to the result_cache), the net effect of which is that it would leave + # us with a stuck entry in the cache. + # 2. We must register the errback after callback. + # If the errback was registered first, `on_fail` returning `None` would + # cause `on_succeed` to be called with that `None` as argument. + # This would start a timer to remove a cache entry, even though there isn't a + # valid cache entry yet. + # (This could prematurely remove a future cache entry with the same key.) + result.addCallback(on_succeed) + result.addErrback(on_fail) + return entry def unset(self, key: KV) -> None: diff --git a/synapse/util/sentinel.py b/synapse/util/sentinel.py deleted file mode 100644 index c8434fc97a..0000000000 --- a/synapse/util/sentinel.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2025 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# - -import enum - - -class Sentinel(enum.Enum): - # defining a sentinel in this way allows mypy to correctly handle the - # type of a dictionary lookup and subsequent type narrowing. - UNSET_SENTINEL = object() diff --git a/synapse/visibility.py b/synapse/visibility.py index fc3e9dfa49..4eeba14a6c 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -103,6 +103,7 @@ async def filter_and_transform_events_for_client( Returns: The filtered events, wrapped in FilteredEvent with the requesting user's membership at each event annotated for use during serialization (MSC4115). + The events are returned in the same order. """ # Filter out events that have been soft failed so that we don't relay them # to clients, unless they're a server admin and want that to happen. diff --git a/tests/config/test_oauth_delegation.py b/tests/config/test_oauth_delegation.py index 6105ca2b04..e0698764a1 100644 --- a/tests/config/test_oauth_delegation.py +++ b/tests/config/test_oauth_delegation.py @@ -60,7 +60,7 @@ class MSC3861OAuthDelegation(TestCase): def setUp(self) -> None: self.config_dict: JsonDict = { - **default_config("test"), + **default_config(server_name="test"), "public_baseurl": BASE_URL, "enable_registration": False, "experimental_features": { @@ -287,7 +287,7 @@ class MasAuthDelegation(TestCase): def setUp(self) -> None: self.config_dict: JsonDict = { - **default_config("test"), + **default_config(server_name="test"), "public_baseurl": BASE_URL, "enable_registration": False, "matrix_authentication_service": { diff --git a/tests/config/test_ratelimiting.py b/tests/config/test_ratelimiting.py index 0d52a96858..c01b0da286 100644 --- a/tests/config/test_ratelimiting.py +++ b/tests/config/test_ratelimiting.py @@ -57,7 +57,7 @@ class ParseRatelimitSettingsTestcase(TestCase): class RatelimitConfigTestCase(TestCase): def test_parse_rc_federation(self) -> None: - config_dict = default_config("test") + config_dict = default_config(server_name="test") config_dict["rc_federation"] = { "window_size": 20000, "sleep_limit": 693, diff --git a/tests/config/test_registration_config.py b/tests/config/test_registration_config.py index 9eb5323a24..139a067833 100644 --- a/tests/config/test_registration_config.py +++ b/tests/config/test_registration_config.py @@ -37,7 +37,7 @@ class RegistrationConfigTestCase(ConfigFileTestCase): Test that the user is faced with configuration errors if they make it smaller, as that configuration doesn't make sense. """ - config_dict = default_config("test") + config_dict = default_config(server_name="test") # First test all the error conditions with self.assertRaises(ConfigError): diff --git a/tests/handlers/test_room_list.py b/tests/handlers/test_room_list.py index e7c4436d1d..da5cd656c6 100644 --- a/tests/handlers/test_room_list.py +++ b/tests/handlers/test_room_list.py @@ -31,7 +31,7 @@ class RoomListHandlerTestCase(unittest.HomeserverTestCase): return room_id def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config["room_list_publication_rules"] = [{"action": "allow"}] return config diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index 49076e69d8..0f8de6e7b9 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -1219,6 +1219,60 @@ class RoomSummaryTestCase(unittest.HomeserverTestCase): result = self.get_success(self.handler.get_room_summary(user2, self.room)) self.assertEqual(result.get("room_id"), self.room) + def test_allowed_room_ids_local(self) -> None: + """allowed_room_ids is returned for a local room with restricted join rules.""" + # Create a space that the restricted room will allow membership from. + space = self.helper.create_room_as( + self.user, + tok=self.token, + extra_content={ + "creation_content": {"type": RoomTypes.SPACE}, + "initial_state": [ + { + "type": EventTypes.JoinRules, + "state_key": "", + "content": {"join_rule": JoinRules.PUBLIC}, + } + ], + }, + ) + + # Create a room version 8 room with join_rule=restricted allowing members + # of the space above. + restricted_room = self.helper.create_room_as( + self.user, + room_version=RoomVersions.V8.identifier, + tok=self.token, + extra_content={ + "initial_state": [ + { + "type": EventTypes.JoinRules, + "state_key": "", + "content": { + "join_rule": JoinRules.RESTRICTED, + "allow": [ + { + "type": RestrictedJoinRuleTypes.ROOM_MEMBERSHIP, + "room_id": space, + } + ], + }, + } + ] + }, + ) + + result = self.get_success( + self.handler.get_room_summary(self.user, restricted_room) + ) + self.assertEqual(result.get("room_id"), restricted_room) + self.assertEqual(result.get("allowed_room_ids"), [space]) + + def test_allowed_room_ids_absent_without_restricted_join_rules(self) -> None: + """allowed_room_ids is absent for rooms that do not use restricted join rules.""" + result = self.get_success(self.handler.get_room_summary(self.user, self.room)) + self.assertNotIn("allowed_room_ids", result) + def test_fed(self) -> None: """ Return data over federation and ensure that it is handled properly. diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 49ecaa30ff..0b2eabaf98 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -77,7 +77,7 @@ class MatrixFederationAgentTests(unittest.TestCase): self.mock_resolver = AsyncMock(spec=SrvResolver) - config_dict = default_config("test", parse=False) + config_dict = default_config(server_name="test", parse=False) config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()] self._config = config = HomeServerConfig() @@ -1019,7 +1019,7 @@ class MatrixFederationAgentTests(unittest.TestCase): self.mock_resolver.resolve_service.return_value = [] self.reactor.lookups["testserv"] = "1.2.3.4" - config = default_config("test", parse=True) + config = default_config(server_name="test", parse=True) # Build a new agent and WellKnownResolver with a different tls factory tls_factory = FederationPolicyForHTTPS(config) diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 631718a366..f25b507aac 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -930,7 +930,7 @@ class SpamCheckerTestCaseLegacy(unittest.HomeserverTestCase): return resources def default_config(self) -> dict[str, Any]: - config = default_config("test") + config = default_config(server_name="test") config.update( { diff --git a/tests/rest/client/sliding_sync/test_extension_sticky_events.py b/tests/rest/client/sliding_sync/test_extension_sticky_events.py new file mode 100644 index 0000000000..aaea6d6793 --- /dev/null +++ b/tests/rest/client/sliding_sync/test_extension_sticky_events.py @@ -0,0 +1,676 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import logging +import sqlite3 + +from twisted.internet.testing import MemoryReactor + +import synapse.rest.admin +import synapse.rest.client.account_data +from synapse.api.constants import EventTypes, EventUnsignedContentFields +from synapse.rest.client import account_data, login, register, room, sync +from synapse.server import HomeServer +from synapse.types import JsonDict, StreamKeyType +from synapse.util.clock import Clock +from synapse.util.duration import Duration + +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.server import TimedOutException +from tests.utils import USE_POSTGRES_FOR_TESTS + +logger = logging.getLogger(__name__) + + +DUMMY_LISTS = { + "main": { + # Don't include any rooms in the top-N window + "ranges": [[0, 0]], + "required_state": [], + "timeline_limit": 0, + } +} +""" +Subscription lists that can be used in the Sliding Sync request `lists` field, +which sets up a subscription that is interested in all rooms but does not let any rooms into the window, +thus does not return any timelines. + +Sufficient to get sticky event updates as per MSC4354: + +> The server MUST include sticky events across all rooms that would be matched by at least one subscription list +> (i.e. all rooms that the client is interested in), even if the room does not appear in top-N window for that +> subscription list at this time. +> Rooms that would not be matched by a list are not included, as this means the client is not interested +> in those rooms. +> +> — https://github.com/matrix-org/matrix-spec-proposals/pull/4354/changes#diff-d76bc1a1d612c6da37d024f5b57f7b8352939b8db8a7ee9c6b71c1a848359afdR213-R217 +""" + + +class SlidingSyncStickyEventsExtensionTestCase(SlidingSyncBase): + """Tests for the sticky events sliding sync extension""" + + if not USE_POSTGRES_FOR_TESTS and sqlite3.sqlite_version_info < (3, 40, 0): + # We need the JSON functionality in SQLite + skip = f"SQLite version is too old to support sticky events: {sqlite3.sqlite_version_info} (See https://github.com/element-hq/synapse/issues/19428)" + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + register.register_servlets, + room.register_servlets, + sync.register_servlets, + account_data.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync and sticky events MSCs + config["experimental_features"] = { + "msc3575_enabled": True, + "msc4354_enabled": True, + } + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + super().prepare(reactor, clock, hs) + + def _assert_sticky_events_response( + self, + response_body: JsonDict, + expected_events_by_room: dict[str, list[str]] | None, + ) -> str | None: + """Assert the sliding sync response was successful and has the expected + sticky events. + + Args: + response_body: Sliding Sync response body + expected_events_by_room: + map of room ID to list of event IDs to expect (in the order we expect them), + or None if we expect an empty sticky events extension response + + Returns the next_batch token from the sticky events section, + unless we're expecting an empty response. + """ + extensions = response_body["extensions"] + sticky_events = extensions.get("org.matrix.msc4354.sticky_events") + + # If there are no expected events, we shouldn't get anything in the response + if expected_events_by_room is None: + self.assertIsNone(sticky_events) + return None + + self.assertIsNotNone(sticky_events) + self.assertIsInstance(sticky_events["next_batch"], str) + + actual_rooms = sticky_events["rooms"] + # Check that we have the expected rooms + self.assertIncludes( + set(actual_rooms.keys()), set(expected_events_by_room.keys()), exact=True + ) + + # Check the events in each room + for room_id, expected_events in expected_events_by_room.items(): + actual_events = actual_rooms[room_id]["events"] + actual_event_ids = [e["event_id"] for e in actual_events] + self.assertEqual(actual_event_ids, expected_events) + for actual_event in actual_events: + # Check the sticky TTL is sent + self.assertIn("unsigned", actual_event) + ttl = actual_event["unsigned"][EventUnsignedContentFields.STICKY_TTL] + self.assertIsInstance(ttl, int) + + self.assertIn("next_batch", sticky_events) + return sticky_events["next_batch"] + + def test_empty_sync(self) -> None: + """Test that enabling sticky events extension works on initial and incremental sync, + even if there is no data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + # No sticky events in initial sync. + self._assert_sticky_events_response(response_body, None) + + # Incremental sync should also have no sticky events + response_body, _ = self.do_sync( + sync_body, since=response_body["pos"], tok=user1_tok + ) + self._assert_sticky_events_response(response_body, None) + + def test_initial_sync(self) -> None: + """Test that we get sticky events when we don't specify a since token + (initial sync). + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room and join both users + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send a sticky event from user2 + sticky_event_id: str = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Initial sync should return the sticky event + sync_body: JsonDict = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Assert the response and then get the next_batch for the next sliding sync request + next_batch = self._assert_sticky_events_response( + response_body, {room_id: [sticky_event_id]} + ) + assert next_batch is not None + + # Do an incremental sync immediately again + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + "since": next_batch, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Check we don't get that event again + self._assert_sticky_events_response(response_body, None) + + # Send another sticky event + sticky_event_id2: str = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "another sticky message", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + + # Now the incremental sync should give us that event + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + self._assert_sticky_events_response( + response_body, {room_id: [sticky_event_id2]} + ) + + def test_expired_events_not_returned(self) -> None: + """Test that expired sticky events are not returned.""" + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send a sticky event with a short duration + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(seconds=2), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Initial sync should return the sticky event + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # We should still get the event for now + self._assert_sticky_events_response(response_body, {room_id: [sticky_event_id]}) + + # Advance time past the sticky duration + self.reactor.advance(3) + + # A second initial sync should not return the expired sticky event + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + self._assert_sticky_events_response(response_body, None) + + def test_wait_for_new_data(self) -> None: + """Test that the sliding sync request waits for new sticky events to arrive. + (Only applies to incremental syncs with a `timeout` specified). + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Initial sync with no sticky events + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make the sliding sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + "?timeout=10000" + f"&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + + # Block for 5 seconds to make sure we are in `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + + # Send a sticky event to trigger new results + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Should respond before the 10 second timeout + channel.await_result(timeout_ms=100) + self.assertEqual(channel.code, 200, channel.json_body) + + self._assert_sticky_events_response( + channel.json_body, + {room_id: [sticky_event_id]}, + ) + + def test_wait_for_new_data_timeout(self) -> None: + """ + Test that the sliding sync request waits for new sticky events to arrive + and times out when no data arrives before the deadline. + (Only applies to incremental syncs with a `timeout` specified). + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Initial sync with no sticky events + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make the sliding sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + "?timeout=10000" + f"&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + + # Block for 5 seconds to make sure we are `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + # Wake-up `notifier.wait_for_events(...)` that will cause us test + # `SlidingSyncResult.__bool__` for new results. + self._bump_notifier_wait_for_events( + # wake key is intentionally unrelated to sticky events + user1_id, + wake_stream_key=StreamKeyType.ACCOUNT_DATA, + ) + # Block for a little bit more to ensure we don't see any new results. + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=4000) + # Wait for the sync to complete (wait for the rest of the 10 second timeout, + # 5000 + 4000 + 1200 > 10000) + channel.await_result(timeout_ms=1200) + self.assertEqual(channel.code, 200, channel.json_body) + + self._assert_sticky_events_response( + channel.json_body, + None, + ) + + def test_ignored_users_sticky_events(self) -> None: + """ + Test that sticky events from ignored users are not delivered to clients. + + > As with normal events, sticky events sent by ignored users MUST NOT be + > delivered to clients. + > — https://github.com/matrix-org/matrix-spec-proposals/blob/4340903c15e9eab1bfb2f6a31cfa08fd535f7e7c/proposals/4354-sticky-events.md#sync-api-changes + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # User1 ignores user2 + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/user/{user1_id}/account_data/m.ignored_user_list", + {"ignored_users": {user2_id: {}}}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # User2 sends a sticky event + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky from ignored user", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Initial sync for user1 + sync_body = { + "lists": { + "main": { + "ranges": [[0, 10]], + "required_state": [], + # In this test we ask for 10 events of timeline. + "timeline_limit": 10, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Timeline events should not include sticky event from ignored user + timeline_events = response_body["rooms"][room_id]["timeline"] + timeline_event_ids = [e["event_id"] for e in timeline_events] + + self.assertNotIn( + sticky_event_id, + timeline_event_ids, + "Sticky event from ignored user should not be in timeline", + ) + + # Sticky events section should also not include the event from ignored user + self._assert_sticky_events_response(response_body, None) + + def test_history_visibility_bypass_for_sticky_events(self) -> None: + """ + Test that joined users can see sticky events even when history visibility + is set to "joined" and they joined after the event was sent. + + > History visibility checks MUST NOT be applied to sticky events. + > Any joined user is authorised to see sticky events for the duration they remain sticky. + > — https://github.com/matrix-org/matrix-spec-proposals/blob/4340903c15e9eab1bfb2f6a31cfa08fd535f7e7c/proposals/4354-sticky-events.md#proposal + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # Create a room with restrictive history visibility + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + extra_content={ + # Anyone can join + "preset": "public_chat", + # But you can't see history before you joined + "initial_state": [ + { + "type": EventTypes.RoomHistoryVisibility, + "state_key": "", + "content": {"history_visibility": "joined"}, + } + ], + }, + is_public=False, + ) + + # User1 sends a sticky event + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + + # User1 also sends a regular event, to verify our test setup + regular_event_id = self.helper.send( + room_id=room_id, + body="regular message", + tok=user1_tok, + )["event_id"] + + # Register and join a second user after the sticky event was sent + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + self.helper.join(room_id, user2_id, tok=user2_tok) + + # User2 syncs - they should see sticky event even though + # history visibility is "joined" and they joined after it was sent + sync_body = { + "lists": { + "main": { + "ranges": [[0, 10]], + "required_state": [], + # In this test, we ask for 10 events of timeline. + "timeline_limit": 10, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user2_tok) + + # The sticky event is fully visible in its own right, + # but AFAICT the timeline only includes events since we join the room + # (regardless of history visibility), + # so this comes down in the sticky extension + self._assert_sticky_events_response(response_body, {room_id: [sticky_event_id]}) + + # Instead the sticky event is in the timeline + timeline_events = response_body["rooms"][room_id]["timeline"] + timeline_event_ids = [e["event_id"] for e in timeline_events] + self.assertNotIn( + regular_event_id, + timeline_event_ids, + f"Expecting to not see regular event ({regular_event_id}) before user1 joined.", + ) + + def test_sticky_event_pagination(self) -> None: + """ + Test that pagination works correctly when there are many sticky events. + Also check they are delivered in stream order. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send 4 sticky events (more than our limit of 2) + sticky_event_ids: list[str] = [] + for i in range(4): + event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": f"sticky message {i}", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + sticky_event_ids.append(event_id) + + # Initial sync + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": {"enabled": True, "limit": 2} + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # We expect to see the first 2 sticky events by stream order + # and they should be in that stream order + next_batch = self._assert_sticky_events_response( + response_body, {room_id: sticky_event_ids[0:2]} + ) + + # Incremental sync to get remaining sticky events + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + # This makes it incremental + "since": next_batch, + "limit": 3, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Should get remaining events, in stream order again + self._assert_sticky_events_response( + response_body, {room_id: sticky_event_ids[2:4]} + ) + + def test_deduplication_with_timeline(self) -> None: + """ + Test that sticky events are not included in the sticky event extension of sliding sync + if they are included in the main timeline section. + + Send 3 events: + 1. sticky + 2. sticky + 3. regular + + We then will sync with a timeline limit of 2 and a sticky event limit of 2. + We should then see (2) and (3) included in the timeline + and (1) in the sticky event response (but not (2) because it's already + included in the timeline.) + + 1. sticky [in sticky section] + + ------------->>> Timeline section + 2. sticky + 3. regular + -------------<<< + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + + sticky_event_ids: list[str] = [] + for i in range(2): + event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": f"sticky message {i}", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + sticky_event_ids.append(event_id) + + non_sticky_event_id = self.helper.send_event( + room_id, + EventTypes.Message, + content={"body": "regular message", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + + # Sync + sync_body = { + "lists": { + "main": { + "ranges": [[0, 10]], + "required_state": [], + # In this test, we want a timeline window of the 2 latest messages + "timeline_limit": 2, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + "limit": 2, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + events_in_sticky_section = response_body["extensions"][ + "org.matrix.msc4354.sticky_events" + ]["rooms"][room_id]["events"] + event_ids_in_sticky_section = [e["event_id"] for e in events_in_sticky_section] + + events_in_timeline_section = response_body["rooms"][room_id]["timeline"] + event_ids_in_timeline_section = [ + e["event_id"] for e in events_in_timeline_section + ] + + self.assertEqual( + event_ids_in_sticky_section, + [sticky_event_ids[0]], + ) + self.assertEqual( + event_ids_in_timeline_section, [sticky_event_ids[1], non_sticky_event_id] + ) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index fc7d6a279c..2fd18f0e54 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -374,7 +374,7 @@ class SlidingSyncBase(unittest.HomeserverTestCase): user_id: The user ID to wake up the notifier for wake_stream_key: The stream key to wake up. This will create an actual new entity in that stream so it's best to choose one that won't affect the - Sliding Sync results you're testing for. In other words, if your testing + Sliding Sync results you're testing for. In other words, if you're testing account data, choose `StreamKeyType.PRESENCE` instead. We support two possible stream keys because you're probably testing one or the other so one is always a "safe" option. diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index da904ce1f5..c3bfdf7c8d 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -317,7 +317,7 @@ class DelayedEventsTestCase(HomeserverTestCase): ) def test_cancel_delayed_event_ratelimit(self, action_in_path: bool) -> None: delay_ids = [] - for _ in range(2): + for _ in range(3): channel = self.make_request( "POST", _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), @@ -329,12 +329,39 @@ class DelayedEventsTestCase(HomeserverTestCase): assert delay_id is not None delay_ids.append(delay_id) - channel = self._update_delayed_event(delay_ids.pop(0), "cancel", action_in_path) + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event(delay_id, "cancel", action_in_path) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) - channel = self._update_delayed_event(delay_ids.pop(0), "cancel", action_in_path) + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event(delay_id, "cancel", action_in_path) self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + # Using auth should bypass ratelimit applied against source IP + channel = self._update_delayed_event( + delay_id, "cancel", action_in_path, self.user1_access_token + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event( + delay_id, "cancel", action_in_path, self.user1_access_token + ) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) + ) + + # Test that the request isn't ratelimited anymore. + channel = self._update_delayed_event( + delay_id, "cancel", action_in_path, self.user1_access_token + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + @parameterized.expand( ( (content_property_value, action_in_path) @@ -475,7 +502,7 @@ class DelayedEventsTestCase(HomeserverTestCase): ) def test_restart_delayed_event_ratelimit(self, action_in_path: bool) -> None: delay_ids = [] - for _ in range(2): + for _ in range(3): channel = self.make_request( "POST", _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), @@ -487,16 +514,39 @@ class DelayedEventsTestCase(HomeserverTestCase): assert delay_id is not None delay_ids.append(delay_id) + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event(delay_id, "restart", action_in_path) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event(delay_id, "restart", action_in_path) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Using auth should bypass ratelimit applied against source IP channel = self._update_delayed_event( - delay_ids.pop(0), "restart", action_in_path + delay_id, "restart", action_in_path, self.user1_access_token ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + delay_id = delay_ids.pop(0) channel = self._update_delayed_event( - delay_ids.pop(0), "restart", action_in_path + delay_id, "restart", action_in_path, self.user1_access_token ) self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) + ) + + # Test that the request isn't ratelimited anymore. + channel = self._update_delayed_event( + delay_id, "restart", action_in_path, self.user1_access_token + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + def test_delayed_state_is_not_cancelled_by_new_state_from_same_user( self, ) -> None: @@ -612,7 +662,11 @@ class DelayedEventsTestCase(HomeserverTestCase): return content def _update_delayed_event( - self, delay_id: str, action: str, action_in_path: bool + self, + delay_id: str, + action: str, + action_in_path: bool, + access_token: str | None = None, ) -> FakeChannel: path = f"{PATH_PREFIX}/{delay_id}" body = {} @@ -620,7 +674,7 @@ class DelayedEventsTestCase(HomeserverTestCase): path += f"/{action}" else: body["action"] = action - return self.make_request("POST", path, body) + return self.make_request("POST", path, body, access_token) def _find_sent_delayed_event( self, access_token: str, delay_id: str, should_find: bool diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py index 1f8d9154ca..3fe1c1bc89 100644 --- a/tests/rest/client/test_retention.py +++ b/tests/rest/client/test_retention.py @@ -2,6 +2,7 @@ # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -25,7 +26,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes from synapse.events.utils import FilteredEvent from synapse.rest import admin -from synapse.rest.client import login, room +from synapse.rest.client import login, retention, room from synapse.server import HomeServer from synapse.types import JsonDict, create_requester from synapse.util.clock import Clock @@ -394,3 +395,122 @@ class RetentionNoDefaultPolicyTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, expected_code, channel.result) return channel.json_body + + +RETENTION_CONFIGURATION_URL = ( + "/_matrix/client/unstable/org.matrix.msc1763/retention/configuration" +) + + +class RetentionConfigurationEndpointTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + retention.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.register_user("user", "password") + self.token = self.login("user", "password") + + @override_config( + { + "experimental_features": {"msc1763_enabled": True}, + } + ) + def test_disabled_returns_404_no_retention(self) -> None: + """The endpoint must 404 when retention is not enabled.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 404, channel.result) + + @override_config( + { + "retention": { + "enabled": True, + "default_policy": { + "min_lifetime": one_day_ms, + "max_lifetime": one_day_ms * 3, + }, + "allowed_lifetime_min": one_day_ms, + "allowed_lifetime_max": one_day_ms * 3, + }, + "experimental_features": {"msc1763_enabled": False}, + } + ) + def test_disabled_returns_404_no_msc1763_enabled(self) -> None: + """The endpoint must 404 when retention is not enabled.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 404, channel.result) + + @override_config( + { + "retention": { + "enabled": True, + "default_policy": { + "min_lifetime": one_day_ms, + "max_lifetime": one_day_ms * 3, + }, + "allowed_lifetime_min": one_day_ms, + "allowed_lifetime_max": one_day_ms * 3, + }, + "experimental_features": {"msc1763_enabled": True}, + } + ) + def test_full_config(self) -> None: + """Returns default policy and max_lifetime limits when fully configured.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 200, channel.result) + body = channel.json_body + self.assertEqual( + body["policies"]["*"], + {"min_lifetime": one_day_ms, "max_lifetime": one_day_ms * 3}, + ) + self.assertEqual( + body["limits"]["max_lifetime"], + {"min": one_day_ms, "max": one_day_ms * 3}, + ) + + @override_config( + { + "retention": {"enabled": True}, + "experimental_features": {"msc1763_enabled": True}, + } + ) + def test_no_default_policy_no_limits(self) -> None: + """Returns empty policies and limits when nothing is configured.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 200, channel.result) + body = channel.json_body + self.assertEqual(body["policies"], {}) + self.assertEqual(body["limits"], {}) + + @override_config( + { + "retention": { + "enabled": True, + "allowed_lifetime_min": one_day_ms, + "allowed_lifetime_max": one_day_ms * 7, + }, + "experimental_features": {"msc1763_enabled": True}, + } + ) + def test_limits_only(self) -> None: + """Returns limits but no default policy entry when only limits are set.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 200, channel.result) + body = channel.json_body + self.assertNotIn("*", body["policies"]) + self.assertEqual( + body["limits"]["max_lifetime"], + {"min": one_day_ms, "max": one_day_ms * 7}, + ) diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 10325c536a..8d4892ae91 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2785,7 +2785,7 @@ class PublicRoomsRoomTypeFilterTestCase(unittest.HomeserverTestCase): ) def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config["room_list_publication_rules"] = [{"action": "allow"}] return config diff --git a/tests/server.py b/tests/server.py index 20fcc42081..ce5eaad63d 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1108,7 +1108,7 @@ def setup_test_homeserver( reactor = ThreadedMemoryReactorClock() if config is None: - config = default_config(server_name, parse=True) + config = default_config(server_name=server_name, parse=True) server_name = config.server.server_name if not isinstance(server_name, str): diff --git a/tests/server_notices/__init__.py b/tests/server_notices/__init__.py index 19bda218e3..355e105da8 100644 --- a/tests/server_notices/__init__.py +++ b/tests/server_notices/__init__.py @@ -44,7 +44,7 @@ class ServerNoticesTests(unittest.HomeserverTestCase): ] def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config.update({"server_notices": DEFAULT_SERVER_NOTICES_CONFIG}) diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py index a0c5582496..06302e7776 100644 --- a/tests/server_notices/test_resource_limits_server_notices.py +++ b/tests/server_notices/test_resource_limits_server_notices.py @@ -40,7 +40,7 @@ from tests.utils import default_config class TestResourceLimitsServerNotices(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config.update( { diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index d5ed947094..bfe7049fb2 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -286,6 +286,192 @@ class EventPushActionsStoreTestCase(HomeserverTestCase): _rotate() _assert_counts(0, 0) + def test_count_aggregation_after_purge(self) -> None: + """Purging history must not leave stale counts in event_push_summary. + + Regression test: if notifications had been rotated into + `event_push_summary` before a history purge, deleting the purged rows + from `event_push_actions` left the summary counts unchanged, inflating + the notification count. + """ + user_id, _, _, other_token, room_id = self._create_users_and_room() + + def _assert_count(notif_count: int) -> None: + aggregate_counts = self.get_success( + self.store.db_pool.runInteraction( + "get-aggregate-unread-counts", + self.store._get_unread_counts_by_room_for_user_txn, + user_id, + ) + ) + self.assertEqual(aggregate_counts.get(room_id, 0), notif_count) + + def _create_event() -> str: + result = self.helper.send_event( + room_id, + type="m.room.message", + content={"msgtype": "m.text", "body": "msg"}, + tok=other_token, + ) + return result["event_id"] + + def _mark_read(event_id: str) -> None: + self.get_success( + self.store.insert_receipt( + room_id, + "m.read", + user_id=user_id, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + + def _purge_before(event_id: str) -> None: + token = self.get_success( + self.store.get_topological_token_for_event(event_id) + ) + token_str = self.get_success(token.to_string(self.store)) + self.get_success( + self.store.purge_history(room_id, token_str, delete_local_events=True) + ) + + # Mark an initial event as read so that the summary tracks a receipt. + read_event_id = _create_event() + _mark_read(read_event_id) + _assert_count(0) + + # Send some events and rotate them into the summary table. + _create_event() + _create_event() + cutoff_event_id = _create_event() + self.get_success(self.store._rotate_notifs()) + _assert_count(3) + + # Purge history before the most recent event, which deletes the earlier + # events (including the one before the read receipt). + _purge_before(cutoff_event_id) + + # Only the most recent event survives, so the count should be 1. + _assert_count(1) + + # A subsequent rotation must not resurrect the purged counts. + self.get_success(self.store._rotate_notifs()) + _assert_count(1) + + # Reading the surviving event clears the count entirely. + _mark_read(cutoff_event_id) + _assert_count(0) + + def test_count_aggregation_receipt_before_first_rotation(self) -> None: + """ + Regression test: reading a highlight before the first rotation must not + permanently inflate the badge count. + + Highlights survive the receipt-triggered DELETE (highlight=1), so if + last_receipt_stream_ordering is NULL when rotation creates the summary + row, the badge query re-counts them forever. + """ + user_id, token, _, other_token, room_id = self._create_users_and_room() + + def _assert_badge(expected: int) -> None: + counts = self.get_success( + self.store.db_pool.runInteraction( + "get-aggregate-unread-counts", + self.store._get_unread_counts_by_room_for_user_txn, + user_id, + ) + ) + self.assertEqual(counts.get(room_id, 0), expected) + + def _send(highlight: bool = False) -> str: + return self.helper.send_event( + room_id, + type="m.room.message", + content={"msgtype": "m.text", "body": user_id if highlight else "msg"}, + tok=other_token, + )["event_id"] + + def _read(event_id: str) -> None: + self.get_success( + self.store.insert_receipt( + room_id, + "m.read", + user_id=user_id, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + + # Highlight arrives; user reads it before any rotation (no summary row exists). + _read(_send(highlight=True)) + # One new event after the receipt makes stream_ordering > max_clause true. + _send() + # Without the fix: badge = 2 (highlight re-counted). With fix: badge = 1. + self.get_success(self.store._rotate_notifs()) + _assert_badge(1) + + def test_count_aggregation_receipt_before_first_rotation_in_thread(self) -> None: + """ + Same regression as test_count_aggregation_receipt_before_first_rotation, + but for a highlight inside a thread cleared by an unthreaded receipt. + + The fix must pre-populate event_push_summary for every thread with + pending push actions, not just MAIN_TIMELINE. + """ + user_id, token, _, other_token, room_id = self._create_users_and_room() + + def _assert_badge(expected: int) -> None: + counts = self.get_success( + self.store.db_pool.runInteraction( + "get-aggregate-unread-counts", + self.store._get_unread_counts_by_room_for_user_txn, + user_id, + ) + ) + self.assertEqual(counts.get(room_id, 0), expected) + + def _send(thread_root: str | None = None, highlight: bool = False) -> str: + content: JsonDict = { + "msgtype": "m.text", + "body": user_id if highlight else "msg", + } + if thread_root is not None: + content["m.relates_to"] = { + "rel_type": RelationTypes.THREAD, + "event_id": thread_root, + } + return self.helper.send_event( + room_id, + type="m.room.message", + content=content, + tok=other_token, + )["event_id"] + + def _read(event_id: str) -> None: + self.get_success( + self.store.insert_receipt( + room_id, + "m.read", + user_id=user_id, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + + # A thread root, then a highlight inside that thread. + thread_root = _send() + thread_highlight = _send(thread_root=thread_root, highlight=True) + # User reads everything with an unthreaded receipt before any rotation. + _read(thread_highlight) + # A new event in the same thread makes stream_ordering > max_clause true. + _send(thread_root=thread_root) + # Without the fix: badge = 2 (thread highlight re-counted). With: badge = 1. + self.get_success(self.store._rotate_notifs()) + _assert_badge(1) + def test_count_aggregation_threads(self) -> None: """ This is essentially the same test as test_count_aggregation, but adds diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index 4f97d89f78..d8ee2d8b2c 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -41,7 +41,7 @@ def gen_3pids(count: int) -> list[dict[str, Any]]: class MonthlyActiveUsersTestCase(unittest.HomeserverTestCase): def default_config(self) -> dict[str, Any]: - config = default_config("test") + config = default_config(server_name="test") config.update({"limit_usage_by_mau": True, "max_mau_value": 50}) diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py index 2894530d52..b41bd7cc2a 100644 --- a/tests/storage/test_purge.py +++ b/tests/storage/test_purge.py @@ -18,15 +18,25 @@ # # +from parameterized import parameterized + from twisted.internet.testing import MemoryReactor +from synapse.api.constants import EventTypes from synapse.api.errors import NotFoundError, SynapseError +from synapse.api.room_versions import ( + EventFormatVersions, + RoomVersion, + RoomVersions, +) +from synapse.events import EventBase from synapse.rest.client import room from synapse.server import HomeServer from synapse.types.state import StateFilter from synapse.types.storage import _BackgroundUpdates from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import HomeserverTestCase @@ -457,3 +467,88 @@ class PurgeTests(HomeserverTestCase): ) ) self.assertEqual(len(state_groups), 210) + + +class PurgeLocalEventsTests(HomeserverTestCase): + """Tests that purging history with ``delete_local_events=False`` preserves + locally-originated events while still deleting remote ones. + """ + + user_id = "@red:server" + servlets = [room.register_servlets] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver("server") + return hs + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.state = hs.get_state_handler() + self._storage_controllers = hs.get_storage_controllers() + + @parameterized.expand([(RoomVersions.V2,), (RoomVersions.V12,)]) + def test_purge_history_keeps_local_events(self, room_version: RoomVersion) -> None: + """ + Purging with `delete_local_events=False` should keep local events. + """ + room_id = self.helper.create_room_as( + self.user_id, room_version=room_version.identifier + ) + + def inject_remote_event(*, depth: int, prev_event_id: str) -> EventBase: + pdu = { + "type": EventTypes.Message, + "sender": "@bob:other.example.com", + "room_id": room_id, + "content": {"msgtype": "m.text", "body": "this is a message"}, + "prev_events": [prev_event_id], + "auth_events": [], + "depth": depth, + "origin_server_ts": 42, + } + if room_version.event_format == EventFormatVersions.ROOM_V1_V2: + # For v1/v2 rooms we need to assign our own arbitrary event ID at creation time + # For later versions, the event ID is computed (it's a hash) + pdu["event_id"] = "$remote_abc:other.example.com" + # The format of prev_events was also different + pdu["prev_events"] = [[prev_event_id, {}]] + + remote_event = make_test_pdu_event(pdu, room_version) + context = self.get_success(self.state.compute_event_context(remote_event)) + persistence = self._storage_controllers.persistence + assert persistence is not None + self.get_success(persistence.persist_event(remote_event, context)) + return remote_event + + # A local event before the purge threshold + local_event_id = self.helper.send(room_id, body="local event")["event_id"] + local_event = self.get_success(self.store.get_event(local_event_id)) + + # A remote event before the purge threshold + remote_event = inject_remote_event( + prev_event_id=local_event_id, + depth=local_event.depth + 1, + ) + + # A final local event, after the cutoff, to serve as the new forward + # extremity (so the purge doesn't refuse to delete the extremities). + last_event_id = self.helper.send(room_id, body="new fwd extrem")["event_id"] + + token = self.get_success( + self.store.get_topological_token_for_event(last_event_id) + ) + token_str = self.get_success(token.to_string(self.store)) + + # Purge everything before the last event, keeping local events. + self.get_success( + self._storage_controllers.purge_events.purge_history( + room_id, token_str, delete_local_events=False + ) + ) + + # The remote event below the cutoff should have been deleted. + self.get_failure(self.store.get_event(remote_event.event_id), NotFoundError) + + # Both local event must be preserved + self.get_success(self.store.get_event(local_event_id)) + self.get_success(self.store.get_event(last_event_id)) diff --git a/tests/test_mau.py b/tests/test_mau.py index 2d5c4c5d1c..0590628fc9 100644 --- a/tests/test_mau.py +++ b/tests/test_mau.py @@ -39,7 +39,7 @@ class TestMauLimit(unittest.HomeserverTestCase): servlets = [register.register_servlets, sync.register_servlets] def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config.update( { diff --git a/tests/test_state.py b/tests/test_state.py index 0a6720d6ef..b69a09dab5 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -244,7 +244,7 @@ class StateTestCase(unittest.TestCase): ] ) reactor, clock = get_clock() - hs.config = default_config("tesths", True) + hs.config = default_config(server_name="tesths", parse=True) hs.get_datastores.return_value = Mock( main=self.dummy_store, state_deletion=dummy_deletion_store, diff --git a/tests/test_types.py b/tests/test_types.py index 1802f0fae3..43fd96d6f5 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -18,16 +18,20 @@ # [This file includes modifications made by New Vector Limited] # # - +import copy from unittest import skipUnless from immutabledict import immutabledict from parameterized import parameterized_class +from pydantic import BaseModel, PydanticInvalidForJsonSchema, ValidationError from synapse.api.errors import SynapseError from synapse.types import ( + Absent, + AbsentType, AbstractMultiWriterStreamToken, MultiWriterStreamToken, + NonNegativeStrictInt, RoomAlias, RoomStreamToken, UserID, @@ -199,3 +203,150 @@ class MultiWriterTokenTestCase(unittest.HomeserverTestCase): parsed_token = self.get_success(self.token_type.parse(store, "m5~")) self.assertEqual(parsed_token, self.token_type(stream=5)) + + +class AbsentTestCase(unittest.TestCase): + """ + Tests for the `Absent` utility, which is meant to be like `None` except + explicitly signalling absence rather than JSON null. + """ + + def test_cant_create_second_absent(self) -> None: + """ + Tests that we aren't allowed to instantiate a second `Absent`. + """ + with self.assertRaises(TypeError): + AbsentType() # type: ignore[call-arg] + + def test_is_falsy(self) -> None: + """ + Tests `Absent` is falsy and can therefore be used a bit like `None`. + """ + if Absent: + self.fail("Absent is truthy!") + + self.assertEqual(Absent or "something", "something") + + def test_pydantic_jsonschema(self) -> None: + """ + Tests that `Absent` can't be used to produce JSONSchema in Pydantic models. + + In the future, it may be useful to produce correct JSONSchema, but for now + I was mostly interested in making sure we don't produce weird/invalid JSONSchema. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + with self.assertRaises(PydanticInvalidForJsonSchema): + MyModel.model_json_schema() + + def test_pydantic_reject_null(self) -> None: + """ + Tests that `Absent` rejects `None` (JSON null) when used in Pydantic models. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + with self.assertRaises(ValidationError): + MyModel.model_validate({"absent": None}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"absent": null}') + + def test_pydantic_accept_absence(self) -> None: + """ + Tests that `Absent` accepts the absence of a value when used in Pydantic models. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + self.assertEqual(MyModel.model_validate({}), MyModel(absent=Absent)) + self.assertEqual(MyModel.model_validate_json("{}"), MyModel(absent=Absent)) + + def test_copy(self) -> None: + """ + Tests that the `copy` module always uses the same instance of Absent. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + a = MyModel.model_validate({}) + b = copy.deepcopy(a) + + self.assertIs(copy.copy(Absent), Absent) + self.assertIs(a.absent, b.absent) + + +class NonNegativeStrictIntTestCase(unittest.TestCase): + """ + Tests for the `NonNegativeStrictInt` utility. + """ + + def test_pydantic_jsonschema(self) -> None: + """ + Tests that `NonNegativeStrictInt` produces sensible JSONSchema. + """ + + class MyModel(BaseModel): + limit: NonNegativeStrictInt = 100 + + self.assertEqual( + MyModel.model_json_schema(), + { + "properties": { + "limit": { + "default": 100, + "minimum": 0, + "title": "Limit", + "type": "integer", + } + }, + "title": "MyModel", + "type": "object", + }, + f"JSONSchema actually is:\n{MyModel.model_json_schema()!r}", + ) + + def test_pydantic_reject(self) -> None: + """ + Tests that `NonNegativeStrictInt` rejects negative numbers + and non-ints. + """ + + class MyModel(BaseModel): + limit: NonNegativeStrictInt = 100 + + with self.assertRaises(ValidationError): + MyModel.model_validate({"limit": -1}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"limit": -1}') + + # StrictInt, so don't accept floats... + with self.assertRaises(ValidationError): + MyModel.model_validate({"limit": 1.5}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"limit": 1.5}') + + # ...and don't accept stringy ints either. + with self.assertRaises(ValidationError): + MyModel.model_validate({"limit": "42"}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"limit": "42"}') + + def test_pydantic_accept(self) -> None: + """ + Tests that `Absent` accepts the absence of a value when used in Pydantic models. + """ + + class MyModel(BaseModel): + limit: NonNegativeStrictInt = 100 + + self.assertEqual(MyModel.model_validate_json('{"limit": 0}'), MyModel(limit=0)) + self.assertEqual(MyModel.model_validate({"limit": 42}), MyModel(limit=42)) diff --git a/tests/unittest.py b/tests/unittest.py index 03db9a4282..93131521d0 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -545,7 +545,7 @@ class HomeserverTestCase(TestCase): """ Get a default HomeServer config dict. """ - config = default_config("test") + config = default_config(server_name="test") # apply any additional config which was specified via the override_config # decorator. diff --git a/tests/util/caches/test_response_cache.py b/tests/util/caches/test_response_cache.py index af42769890..79b6968ff0 100644 --- a/tests/util/caches/test_response_cache.py +++ b/tests/util/caches/test_response_cache.py @@ -3,6 +3,7 @@ # # Copyright 2021 The Matrix.org Foundation C.I.C. # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -15,7 +16,8 @@ # Originally licensed under the Apache License, Version 2.0: # . # -# [This file includes modifications made by New Vector Limited] +# [This file includes modifications made by New Vector Limited +# and Element Creations Ltd] # # @@ -234,9 +236,9 @@ class ResponseCacheTestCase(TestCase): [], cache.keys(), "cache should not have the result now" ) - def test_cache_func_errors(self) -> None: + def test_errors_raised_to_all_waiters(self) -> None: """If the callback raises an error, the error should be raised to all - callers and the result should not be cached""" + concurrent callers that were waiting on the same in-flight result.""" cache = self.with_cache("error_cache", ms=3000) expected_error = Exception("oh no") @@ -259,6 +261,60 @@ class ResponseCacheTestCase(TestCase): self.assertFailure(wrap_d, Exception) self.assertFailure(wrap2_d, Exception) + def test_errors_are_not_cached(self) -> None: + """If the callback raises an error, the error is not cached and + served to any subsequent requests. + """ + cache = self.with_cache("error_not_cached", ms=3000) + + return_error = True + + REQUEST_FAKE_WORK_SLEEP_TIME = Duration(seconds=1) + + async def erring_then_fine(_: str) -> str: + """ + This function raises an error the first time it is called, + then is fine the next time it is called. + """ + nonlocal return_error + + # pretend to do some work + await self.clock.sleep(REQUEST_FAKE_WORK_SLEEP_TIME) + + if return_error: + return_error = False + raise RuntimeError("this is a temporary error!") + return "fine" + + # First call: should get an error + wrap_d = defer.ensureDeferred(cache.wrap(0, erring_then_fine, "ignored")) + + # Should be pending + self.assertNoResult(wrap_d) + + # Wait for the time it takes for the request to resolve to an error + self.reactor.advance(REQUEST_FAKE_WORK_SLEEP_TIME.as_secs()) + + # Check that the Deferred resolved to an error of the correct type + self.failureResultOf(wrap_d, RuntimeError) + + # Second call: the error shouldn't be replayed + # and the next call should succeed, so we should get a successful response + wrap2_d = defer.ensureDeferred(cache.wrap(0, erring_then_fine, "ignored")) + + # Since this is a new request (not coalesced with the previous failed one), + # this should still be pending + self.assertNoResult(wrap2_d) + + # Wait for the time it takes for the request to resolve + self.reactor.advance(REQUEST_FAKE_WORK_SLEEP_TIME.as_secs()) + + self.assertEqual( + self.successResultOf(wrap2_d), + "fine", + "should get the fresh result, not the cached error", + ) + def test_cache_cancel_first_wait(self) -> None: """Test that cancellation of the deferred returned by wrap() on the first call does not immediately cause a cancellation error to be raised diff --git a/tests/util/test_ratelimitutils.py b/tests/util/test_ratelimitutils.py index d3b123c778..07b31852e9 100644 --- a/tests/util/test_ratelimitutils.py +++ b/tests/util/test_ratelimitutils.py @@ -139,7 +139,7 @@ def _await_resolution(reactor: ThreadedMemoryReactorClock, d: Deferred) -> float def build_rc_config(settings: dict | None = None) -> FederationRatelimitSettings: - config_dict = default_config("test") + config_dict = default_config(server_name="test") config_dict.update(settings or {}) config = HomeServerConfig() config.parse_config_dict(config_dict, "", "") diff --git a/tests/utils.py b/tests/utils.py index 0cf97a7e8d..f3d5129097 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -129,16 +129,16 @@ def setupdb() -> None: @overload def default_config( - server_name: str, parse: Literal[False] = ... + *, server_name: str, parse: Literal[False] = ... ) -> dict[str, object]: ... @overload -def default_config(server_name: str, parse: Literal[True]) -> HomeServerConfig: ... +def default_config(*, server_name: str, parse: Literal[True]) -> HomeServerConfig: ... def default_config( - server_name: str, parse: bool = False + *, server_name: str, parse: bool = False ) -> dict[str, object] | HomeServerConfig: """ Create a reasonable test config.