diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e31b6aea90..67d7df0b23 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: run: docker buildx inspect - name: Install Cosign - uses: sigstore/cosign-installer@fb28c2b6339dcd94da6e4cbcbc5e888961f6f8c3 # v3.9.0 + uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1 - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 923e96a624..845e90d27c 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -6,6 +6,11 @@ name: Attempt to automatically fix linting errors on: workflow_dispatch: +env: + # We use nightly so that `fmt` correctly groups together imports, and + # clippy correctly fixes up the benchmarks. + RUST_VERSION: nightly-2025-06-24 + jobs: fixup: name: Fix up @@ -16,13 +21,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: - # We use nightly so that `fmt` correctly groups together imports, and - # clippy correctly fixes up the benchmarks. - toolchain: nightly-2022-12-01 + toolchain: ${{ env.RUST_VERSION }} components: clippy, rustfmt - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -44,6 +47,6 @@ jobs: - run: cargo fmt continue-on-error: true - - uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5.2.0 + - uses: stefanzweifel/git-auto-commit-action@778341af668090896ca464160c2def5d1d1a3eb0 # v6.0.1 with: commit_message: "Attempt to fix linting" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index ee0dac3beb..702f1a5381 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -21,6 +21,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: 1.87.0 + jobs: check_repo: # Prevent this workflow from running on any fork of Synapse other than element-hq/synapse, as it is @@ -41,8 +44,10 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. @@ -75,8 +80,10 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.postgres-version }} @@ -148,8 +155,10 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Ensure sytest runs `pip install` # Delete the lockfile so sytest will `pip install` rather than `poetry install` diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 3e2e76d18d..e80f4f4d71 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -111,7 +111,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, macos-13] + os: [ubuntu-24.04, macos-13] arch: [x86_64, aarch64] # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. @@ -139,7 +139,7 @@ jobs: python-version: "3.x" - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.23.0 + run: python -m pip install cibuildwheel==3.0.0 - name: Set up QEMU to emulate aarch64 if: matrix.arch == 'aarch64' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f9156e81ba..f6250c8ed2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: 1.87.0 + jobs: # Job to detect what has changed so we don't run e.g. Rust checks on PRs that # don't modify Rust code. @@ -85,8 +88,10 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" @@ -149,8 +154,10 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -210,8 +217,10 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: poetry-version: "2.1.1" @@ -227,10 +236,11 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@0d72692bcfbf448b1e2afa01a67f71b455a9dcec # 1.86.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: components: clippy - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo clippy -- -D warnings @@ -245,11 +255,11 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: toolchain: nightly-2025-04-23 components: clippy - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo clippy --all-features -- -D warnings @@ -262,12 +272,12 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: # We use nightly so that it correctly groups together imports toolchain: nightly-2025-04-23 components: rustfmt - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo fmt --check @@ -362,8 +372,10 @@ jobs: postgres:${{ matrix.job.postgres-version }} - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -404,8 +416,10 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 # There aren't wheels for some of the older deps, so we need to install # their build dependencies @@ -519,8 +533,10 @@ jobs: run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Run SyTest run: /bootstrap.sh synapse @@ -663,8 +679,10 @@ jobs: path: synapse - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh @@ -695,8 +713,10 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo test @@ -713,10 +733,10 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: toolchain: nightly-2022-12-01 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo bench --no-run diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 5638029b39..15aa1886af 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -20,6 +20,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: 1.87.0 + jobs: check_repo: # Prevent this workflow from running on any fork of Synapse other than element-hq/synapse, as it is @@ -43,8 +46,10 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -69,8 +74,10 @@ jobs: - run: sudo apt-get -qq install xmlsec1 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -113,8 +120,10 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Patch dependencies # Note: The poetry commands want to create a virtualenv in /src/.venv/, diff --git a/CHANGES.md b/CHANGES.md index 23debccd63..f3e541563f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,21 @@ +# Synapse 1.133.0 (2025-07-01) + +Pre-built wheels are now built using the [manylinux_2_28](https://github.com/pypa/manylinux#manylinux_2_28-almalinux-8-based) base, which is expected to be compatible with distros using glibc 2.28 or later, including: + + - Debian 10+ + - Ubuntu 18.10+ + - Fedora 29+ + - CentOS/RHEL 8+ + +Previously, wheels were built using the [manylinux2014](https://github.com/pypa/manylinux#manylinux2014-centos-7-based-glibc-217) base, which was expected to be compatible with distros using glibc 2.17 or later. + +### Bugfixes + +- Bump `cibuildwheel` to 3.0.0 to fix the `manylinux` wheel builds. ([\#18615](https://github.com/element-hq/synapse/issues/18615)) + + + + # Synapse 1.133.0rc1 (2025-06-24) ### Features diff --git a/Cargo.lock b/Cargo.lock index b85f5f8b98..24994225e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -150,6 +150,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -516,11 +525,59 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" dependencies = [ "displaydoc", - "yoke", + "yoke 0.7.5", "zerofrom", - "zerovec", + "zerovec 0.10.4", ] +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke 0.8.0", + "zerofrom", + "zerovec 0.11.2", +] + +[[package]] +name = "icu_locale" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ae5921528335e91da1b6c695dbf1ec37df5ac13faa3f91e5640be93aa2fbefd" +dependencies = [ + "displaydoc", + "icu_collections 2.0.0", + "icu_locale_core", + "icu_locale_data", + "icu_provider 2.0.0", + "potential_utf", + "tinystr 0.8.1", + "zerovec 0.11.2", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap 0.8.0", + "tinystr 0.8.1", + "writeable 0.6.1", + "zerovec 0.11.2", +] + +[[package]] +name = "icu_locale_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fdef0c124749d06a743c69e938350816554eb63ac979166590e2b4ee4252765" + [[package]] name = "icu_locid" version = "1.5.0" @@ -528,10 +585,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" dependencies = [ "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", + "litemap 0.7.5", + "tinystr 0.7.6", + "writeable 0.5.5", + "zerovec 0.10.4", ] [[package]] @@ -543,9 +600,9 @@ dependencies = [ "displaydoc", "icu_locid", "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", + "icu_provider 1.5.0", + "tinystr 0.7.6", + "zerovec 0.10.4", ] [[package]] @@ -561,15 +618,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" dependencies = [ "displaydoc", - "icu_collections", + "icu_collections 1.5.0", "icu_normalizer_data", "icu_properties", - "icu_provider", + "icu_provider 1.5.0", "smallvec", "utf16_iter", "utf8_iter", "write16", - "zerovec", + "zerovec 0.10.4", ] [[package]] @@ -585,12 +642,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" dependencies = [ "displaydoc", - "icu_collections", + "icu_collections 1.5.0", "icu_locid_transform", "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", + "icu_provider 1.5.0", + "tinystr 0.7.6", + "zerovec 0.10.4", ] [[package]] @@ -609,11 +666,28 @@ dependencies = [ "icu_locid", "icu_provider_macros", "stable_deref_trait", - "tinystr", - "writeable", - "yoke", + "tinystr 0.7.6", + "writeable 0.5.5", + "yoke 0.7.5", "zerofrom", - "zerovec", + "zerovec 0.10.4", +] + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr 0.8.1", + "writeable 0.6.1", + "yoke 0.8.0", + "zerofrom", + "zerotrie", + "zerovec 0.11.2", ] [[package]] @@ -627,6 +701,30 @@ dependencies = [ "syn", ] +[[package]] +name = "icu_segmenter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e185fc13b6401c138cf40db12b863b35f5edf31b88192a545857b41aeaf7d3d3" +dependencies = [ + "core_maths", + "displaydoc", + "icu_collections 2.0.0", + "icu_locale", + "icu_locale_core", + "icu_provider 2.0.0", + "icu_segmenter_data", + "potential_utf", + "utf8_iter", + "zerovec 0.11.2", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5360a2fbe97f617c4f8b944356dedb36d423f7da7f13c070995cf89e59f01220" + [[package]] name = "idna" version = "1.0.3" @@ -664,6 +762,17 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -708,12 +817,24 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "litemap" version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + [[package]] name = "log" version = "0.4.27" @@ -806,6 +927,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "serde", + "zerovec 0.11.2", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1055,9 +1186,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.20" +version = "0.12.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" +checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" dependencies = [ "base64 0.22.1", "bytes", @@ -1341,6 +1472,7 @@ dependencies = [ "hex", "http", "http-body-util", + "icu_segmenter", "lazy_static", "log", "mime", @@ -1409,7 +1541,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ "displaydoc", - "zerovec", + "zerovec 0.10.4", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec 0.11.2", ] [[package]] @@ -1429,15 +1571,17 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "1140bb80481756a8cbe10541f37433b459c5aa1e727b4c020fbfebdc25bf3ec4" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "pin-project-lite", + "slab", "socket2", "windows-sys 0.52.0", ] @@ -1829,6 +1973,12 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + [[package]] name = "yoke" version = "0.7.5" @@ -1837,7 +1987,19 @@ checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", - "yoke-derive", + "yoke-derive 0.7.5", + "zerofrom", +] + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive 0.8.0", "zerofrom", ] @@ -1853,6 +2015,18 @@ dependencies = [ "synstructure", ] +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.17" @@ -1900,15 +2074,35 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", +] + [[package]] name = "zerovec" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" dependencies = [ - "yoke", + "yoke 0.7.5", "zerofrom", - "zerovec-derive", + "zerovec-derive 0.10.3", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke 0.8.0", + "zerofrom", + "zerovec-derive 0.11.1", ] [[package]] @@ -1921,3 +2115,14 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/changelog.d/18070.feature b/changelog.d/18070.feature new file mode 100644 index 0000000000..ecf468f0b6 --- /dev/null +++ b/changelog.d/18070.feature @@ -0,0 +1 @@ +Support for [MSC4235](https://github.com/matrix-org/matrix-spec-proposals/pull/4235): via query param for hierarchy endpoint. Contributed by Krishan (@kfiven). diff --git a/changelog.d/18196.feature b/changelog.d/18196.feature new file mode 100644 index 0000000000..6f4a67b519 --- /dev/null +++ b/changelog.d/18196.feature @@ -0,0 +1 @@ +Add `forget_forced_upon_leave` capability as per [MSC4267](https://github.com/matrix-org/matrix-spec-proposals/pull/4267). diff --git a/changelog.d/18418.removal b/changelog.d/18418.removal new file mode 100644 index 0000000000..ff1d081d8f --- /dev/null +++ b/changelog.d/18418.removal @@ -0,0 +1 @@ +Stop adding the "origin" field to newly-created events (PDUs). \ No newline at end of file diff --git a/changelog.d/18519.doc b/changelog.d/18519.doc new file mode 100644 index 0000000000..b6fde5f42b --- /dev/null +++ b/changelog.d/18519.doc @@ -0,0 +1 @@ +Fix documentation of the Delete Room Admin API's status field. \ No newline at end of file diff --git a/changelog.d/18553.misc b/changelog.d/18553.misc new file mode 100644 index 0000000000..bb143aacfc --- /dev/null +++ b/changelog.d/18553.misc @@ -0,0 +1 @@ +Replace `PyICU` crate with equivalent `icu_segmenter` Rust crate. \ No newline at end of file diff --git a/changelog.d/18573.misc b/changelog.d/18573.misc new file mode 100644 index 0000000000..373852638f --- /dev/null +++ b/changelog.d/18573.misc @@ -0,0 +1 @@ +Improve docstring on `simple_upsert_many`. \ No newline at end of file diff --git a/changelog.d/18575.misc b/changelog.d/18575.misc new file mode 100644 index 0000000000..0033f27df8 --- /dev/null +++ b/changelog.d/18575.misc @@ -0,0 +1 @@ +Raise poetry-core version cap to 2.1.3. diff --git a/changelog.d/18576.misc b/changelog.d/18576.misc new file mode 100644 index 0000000000..46cc78ec2d --- /dev/null +++ b/changelog.d/18576.misc @@ -0,0 +1 @@ +Raise setuptools_rust version cap to 1.11.1. diff --git a/changelog.d/18582.bugfix b/changelog.d/18582.bugfix new file mode 100644 index 0000000000..7091bcc997 --- /dev/null +++ b/changelog.d/18582.bugfix @@ -0,0 +1 @@ +Improve performance of device deletion by adding missing index. \ No newline at end of file diff --git a/changelog.d/18593.bugfix b/changelog.d/18593.bugfix new file mode 100644 index 0000000000..9079b9534a --- /dev/null +++ b/changelog.d/18593.bugfix @@ -0,0 +1 @@ +Fix `avatar_url` and `displayname` being sent on federation profile queries when they are not set. \ No newline at end of file diff --git a/changelog.d/18594.bugfix b/changelog.d/18594.bugfix new file mode 100644 index 0000000000..3293a731f2 --- /dev/null +++ b/changelog.d/18594.bugfix @@ -0,0 +1 @@ +Respond with 401 & `M_USER_LOCKED` when a locked user calls `POST /login`, as per the spec. diff --git a/changelog.d/18596.misc b/changelog.d/18596.misc new file mode 100644 index 0000000000..ce7bfdd129 --- /dev/null +++ b/changelog.d/18596.misc @@ -0,0 +1 @@ +Update to Rust 1.87.0 in CI, and bump the pinned commit of the `dtolnay/rust-toolchain` GitHub Action to `b3b07ba8b418998c39fb20f53e8b695cdcc8de1b`. \ No newline at end of file diff --git a/changelog.d/18602.misc b/changelog.d/18602.misc new file mode 100644 index 0000000000..637d842682 --- /dev/null +++ b/changelog.d/18602.misc @@ -0,0 +1 @@ +Speed up bulk device deletion. diff --git a/changelog.d/18605.bugfix b/changelog.d/18605.bugfix new file mode 100644 index 0000000000..516f09e502 --- /dev/null +++ b/changelog.d/18605.bugfix @@ -0,0 +1 @@ +Ensure policy servers are not asked to scan policy server change events, allowing rooms to disable the use of a policy server while the policy server is down. diff --git a/changelog.d/18625.misc b/changelog.d/18625.misc new file mode 100644 index 0000000000..f2672864c2 --- /dev/null +++ b/changelog.d/18625.misc @@ -0,0 +1 @@ +Log the room ID we're purging state for. diff --git a/changelog.d/18646.misc b/changelog.d/18646.misc new file mode 100644 index 0000000000..bb143aacfc --- /dev/null +++ b/changelog.d/18646.misc @@ -0,0 +1 @@ +Replace `PyICU` crate with equivalent `icu_segmenter` Rust crate. \ No newline at end of file diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index 1d74fee822..9d5f3c7f4f 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -45,6 +45,10 @@ def make_graph(pdus: List[dict], filename_prefix: str) -> None: colors = {"red", "green", "blue", "yellow", "purple"} for pdu in pdus: + # TODO: The "origin" field has since been removed from events generated + # by Synapse. We should consider removing it here as well but since this + # is part of `contrib/`, it is left for the community to revise and ensure things + # still work correctly. origins.add(pdu.get("origin")) color_map = {color: color for color in colors if color in origins} diff --git a/debian/changelog b/debian/changelog index 614901e3fd..32070a32d0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.133.0) stable; urgency=medium + + * New synapse release 1.133.0. + + -- Synapse Packaging team Tue, 01 Jul 2025 13:13:24 +0000 + matrix-synapse-py3 (1.133.0~rc1) stable; urgency=medium * New Synapse release 1.133.0rc1. diff --git a/docs/admin_api/event_reports.md b/docs/admin_api/event_reports.md index 9075e92882..225b431715 100644 --- a/docs/admin_api/event_reports.md +++ b/docs/admin_api/event_reports.md @@ -117,7 +117,6 @@ It returns a JSON body like the following: "hashes": { "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw" }, - "origin": "matrix.org", "origin_server_ts": 1592291711430, "prev_events": [ "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M" diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index bdda9b47ad..12af87148d 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -806,7 +806,7 @@ A response body like the following is returned: }, { "delete_id": "delete_id2", "room_id": "!roomid:example.com", - "status": "purging", + "status": "active", "shutdown_room": { "kicked_users": [ "@foobar:example.com" @@ -843,7 +843,7 @@ A response body like the following is returned: ```json { - "status": "purging", + "status": "active", "delete_id": "bHkCNQpHqOaFhPtK", "room_id": "!roomid:example.com", "shutdown_room": { @@ -876,8 +876,8 @@ The following fields are returned in the JSON response body: - `delete_id` - The ID for this purge - `room_id` - The ID of the room being deleted - `status` - The status will be one of: - - `shutting_down` - The process is removing users from the room. - - `purging` - The process is purging the room and event data from database. + - `scheduled` - The deletion is waiting to be started + - `active` - The process is purging the room and event data from database. - `complete` - The process has completed successfully. - `failed` - The process is aborted, an error has occurred. - `error` - A string that shows an error message if `status` is `failed`. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index d6efab96cf..eb6f04e301 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -29,8 +29,6 @@ easiest way of installing the latest version is to use [rustup](https://rustup.r Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`. -Synapse has an optional, improved user search with better Unicode support. For that you need the development package of `libicu`. On Debian or Ubuntu Linux, this can be installed with `sudo apt install libicu-dev`. - The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git). For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/). diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md index fa5ff4dcf7..e381b3d155 100644 --- a/docs/development/dependencies.md +++ b/docs/development/dependencies.md @@ -164,10 +164,7 @@ $ poetry cache clear --all . # including the wheel artifacts which is not covered by the above command # (see https://github.com/python-poetry/poetry/issues/10304) # -# This is necessary in order to rebuild or fetch new wheels. For example, if you update -# the `icu` library in on your system, you will need to rebuild the PyICU Python package -# in order to incorporate the correct dynamically linked library locations otherwise you -# will run into errors like: `ImportError: libicui18n.so.75: cannot open shared object file: No such file or directory` +# This is necessary in order to rebuild or fetch new wheels. $ rm -rf $(poetry config cache-dir) ``` diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 0853496ab7..0840f532b0 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -286,7 +286,7 @@ Installing prerequisites on Ubuntu or Debian: ```sh sudo apt install build-essential python3-dev libffi-dev \ python3-pip python3-setuptools sqlite3 \ - libssl-dev virtualenv libjpeg-dev libxslt1-dev libicu-dev + libssl-dev virtualenv libjpeg-dev libxslt1-dev ``` ##### ArchLinux @@ -295,7 +295,7 @@ Installing prerequisites on ArchLinux: ```sh sudo pacman -S base-devel python python-pip \ - python-setuptools python-virtualenv sqlite3 icu + python-setuptools python-virtualenv sqlite3 ``` ##### CentOS/Fedora @@ -305,8 +305,7 @@ Installing prerequisites on CentOS or Fedora Linux: ```sh sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \ libwebp-devel libxml2-devel libxslt-devel libpq-devel \ - python3-virtualenv libffi-devel openssl-devel python3-devel \ - libicu-devel + python3-virtualenv libffi-devel openssl-devel python3-devel sudo dnf group install "Development Tools" ``` @@ -333,7 +332,7 @@ dnf install python3.12 python3.12-devel ``` Finally, install common prerequisites ```bash -dnf install libicu libicu-devel libpq5 libpq5-devel lz4 pkgconf +dnf install libpq5 libpq5-devel lz4 pkgconf dnf group install "Development Tools" ``` ###### Using venv module instead of virtualenv command @@ -365,20 +364,6 @@ xcode-select --install Some extra dependencies may be needed. You can use Homebrew (https://brew.sh) for them. -You may need to install icu, and make the icu binaries and libraries accessible. -Please follow [the official instructions of PyICU](https://pypi.org/project/PyICU/) to do so. - -If you're struggling to get icu discovered, and see: -``` - RuntimeError: - Please install pkg-config on your system or set the ICU_VERSION environment - variable to the version of ICU you have installed. -``` -despite it being installed and having your `PATH` updated, you can omit this dependency by -not specifying `--extras all` to `poetry`. If using postgres, you can install Synapse via -`poetry install --extras saml2 --extras oidc --extras postgres --extras opentracing --extras redis --extras sentry`. -ICU is not a hard dependency on getting a working installation. - On ARM-based Macs you may also need to install libjpeg and libpq: ```sh brew install jpeg libpq @@ -400,8 +385,7 @@ Installing prerequisites on openSUSE: ```sh sudo zypper in -t pattern devel_basis sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \ - python-devel libffi-devel libopenssl-devel libjpeg62-devel \ - libicu-devel + python-devel libffi-devel libopenssl-devel libjpeg62-devel ``` ##### OpenBSD diff --git a/docs/upgrade.md b/docs/upgrade.md index d508e2231e..ca9ca121f2 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,13 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.134.0 + +## ICU bundled with Synapse + +Synapse now uses the Rust `icu` library for improved user search. Installing the +native ICU library on your system is no longer required. + # Upgrading to v1.130.0 ## Documented endpoint which can be delegated to a federation worker diff --git a/docs/user_directory.md b/docs/user_directory.md index be8664a016..75d32af44f 100644 --- a/docs/user_directory.md +++ b/docs/user_directory.md @@ -77,14 +77,11 @@ The user provided search term is lowercased and normalized using [NFKC](https:// this treats the string as case-insensitive, canonicalizes different forms of the same text, and maps some "roughly equivalent" characters together. -The search term is then split into words: - -* If [ICU](https://en.wikipedia.org/wiki/International_Components_for_Unicode) is - available, then the system's [default locale](https://unicode-org.github.io/icu/userguide/locale/#default-locales) - will be used to break the search term into words. (See the - [installation instructions](setup/installation.md) for how to install ICU.) -* If unavailable, then runs of ASCII characters, numbers, underscores, and hyphens - are considered words. +The search term is then split into segments using the [`icu_segmenter` +Rust crate](https://crates.io/crates/icu_segmenter). This crate ships with its +own dictionary and Long Short Term-Memory (LSTM) machine learning models +per-language to segment words. Read more [in the crate's +documentation](https://docs.rs/icu/latest/icu/segmenter/struct.WordSegmenter.html#method.new_auto). The queries for PostgreSQL and SQLite are detailed below, but their overall goal is to find matching users, preferring users who are "real" (e.g. not bots, diff --git a/flake.nix b/flake.nix index 749c10da1d..4ff6518aed 100644 --- a/flake.nix +++ b/flake.nix @@ -96,7 +96,6 @@ gnumake # Native dependencies for running Synapse. - icu libffi libjpeg libpqxx diff --git a/poetry.lock b/poetry.lock index 640ecb1d3d..dbc9bbfacc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1966,18 +1966,6 @@ files = [ [package.extras] plugins = ["importlib-metadata ; python_version < \"3.8\""] -[[package]] -name = "pyicu" -version = "2.14" -description = "Python extension wrapping the ICU C++ API" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"user-search\" or extra == \"all\"" -files = [ - {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"}, -] - [[package]] name = "pyjwt" version = "2.6.0" @@ -2433,30 +2421,30 @@ files = [ [[package]] name = "ruff" -version = "0.11.11" +version = "0.12.1" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.11.11-py3-none-linux_armv6l.whl", hash = "sha256:9924e5ae54125ed8958a4f7de320dab7380f6e9fa3195e3dc3b137c6842a0092"}, - {file = "ruff-0.11.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c8a93276393d91e952f790148eb226658dd275cddfde96c6ca304873f11d2ae4"}, - {file = "ruff-0.11.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d6e333dbe2e6ae84cdedefa943dfd6434753ad321764fd937eef9d6b62022bcd"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7885d9a5e4c77b24e8c88aba8c80be9255fa22ab326019dac2356cff42089fc6"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b5ab797fcc09121ed82e9b12b6f27e34859e4227080a42d090881be888755d4"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e231ff3132c1119ece836487a02785f099a43992b95c2f62847d29bace3c75ac"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a97c9babe1d4081037a90289986925726b802d180cca784ac8da2bbbc335f709"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8c4ddcbe8a19f59f57fd814b8b117d4fcea9bee7c0492e6cf5fdc22cfa563c8"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6224076c344a7694c6fbbb70d4f2a7b730f6d47d2a9dc1e7f9d9bb583faf390b"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:882821fcdf7ae8db7a951df1903d9cb032bbe838852e5fc3c2b6c3ab54e39875"}, - {file = "ruff-0.11.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:dcec2d50756463d9df075a26a85a6affbc1b0148873da3997286caf1ce03cae1"}, - {file = "ruff-0.11.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:99c28505ecbaeb6594701a74e395b187ee083ee26478c1a795d35084d53ebd81"}, - {file = "ruff-0.11.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9263f9e5aa4ff1dec765e99810f1cc53f0c868c5329b69f13845f699fe74f639"}, - {file = "ruff-0.11.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:64ac6f885e3ecb2fdbb71de2701d4e34526651f1e8503af8fb30d4915a3fe345"}, - {file = "ruff-0.11.11-py3-none-win32.whl", hash = "sha256:1adcb9a18802268aaa891ffb67b1c94cd70578f126637118e8099b8e4adcf112"}, - {file = "ruff-0.11.11-py3-none-win_amd64.whl", hash = "sha256:748b4bb245f11e91a04a4ff0f96e386711df0a30412b9fe0c74d5bdc0e4a531f"}, - {file = "ruff-0.11.11-py3-none-win_arm64.whl", hash = "sha256:6c51f136c0364ab1b774767aa8b86331bd8e9d414e2d107db7a2189f35ea1f7b"}, - {file = "ruff-0.11.11.tar.gz", hash = "sha256:7774173cc7c1980e6bf67569ebb7085989a78a103922fb83ef3dfe230cd0687d"}, + {file = "ruff-0.12.1-py3-none-linux_armv6l.whl", hash = "sha256:6013a46d865111e2edb71ad692fbb8262e6c172587a57c0669332a449384a36b"}, + {file = "ruff-0.12.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b3f75a19e03a4b0757d1412edb7f27cffb0c700365e9d6b60bc1b68d35bc89e0"}, + {file = "ruff-0.12.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9a256522893cb7e92bb1e1153283927f842dea2e48619c803243dccc8437b8be"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:069052605fe74c765a5b4272eb89880e0ff7a31e6c0dbf8767203c1fbd31c7ff"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a684f125a4fec2d5a6501a466be3841113ba6847827be4573fddf8308b83477d"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdecdef753bf1e95797593007569d8e1697a54fca843d78f6862f7dc279e23bd"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:70d52a058c0e7b88b602f575d23596e89bd7d8196437a4148381a3f73fcd5010"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84d0a69d1e8d716dfeab22d8d5e7c786b73f2106429a933cee51d7b09f861d4e"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cc32e863adcf9e71690248607ccdf25252eeeab5193768e6873b901fd441fed"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fd49a4619f90d5afc65cf42e07b6ae98bb454fd5029d03b306bd9e2273d44cc"}, + {file = "ruff-0.12.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ed5af6aaaea20710e77698e2055b9ff9b3494891e1b24d26c07055459bb717e9"}, + {file = "ruff-0.12.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:801d626de15e6bf988fbe7ce59b303a914ff9c616d5866f8c79eb5012720ae13"}, + {file = "ruff-0.12.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2be9d32a147f98a1972c1e4df9a6956d612ca5f5578536814372113d09a27a6c"}, + {file = "ruff-0.12.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:49b7ce354eed2a322fbaea80168c902de9504e6e174fd501e9447cad0232f9e6"}, + {file = "ruff-0.12.1-py3-none-win32.whl", hash = "sha256:d973fa626d4c8267848755bd0414211a456e99e125dcab147f24daa9e991a245"}, + {file = "ruff-0.12.1-py3-none-win_amd64.whl", hash = "sha256:9e1123b1c033f77bd2590e4c1fe7e8ea72ef990a85d2484351d408224d603013"}, + {file = "ruff-0.12.1-py3-none-win_arm64.whl", hash = "sha256:78ad09a022c64c13cc6077707f036bab0fac8cd7088772dcd1e5be21c5002efc"}, + {file = "ruff-0.12.1.tar.gz", hash = "sha256:806bbc17f1104fd57451a98a58df35388ee3ab422e029e8f5cf30aa4af2c138c"}, ] [[package]] @@ -3371,7 +3359,7 @@ docs = ["Sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"] [extras] -all = ["Pympler", "authlib", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pyicu", "pysaml2", "sentry-sdk", "txredisapi"] +all = ["Pympler", "authlib", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "sentry-sdk", "txredisapi"] cache-memory = ["Pympler"] jwt = ["authlib"] matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] @@ -3384,9 +3372,8 @@ sentry = ["sentry-sdk"] systemd = ["systemd-python"] test = ["idna", "parameterized"] url-preview = ["lxml"] -user-search = ["pyicu"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "ad95ef9da5a5f71fd0da41719e94b2920c8ba119f8bf942b25c2b50661ffa42b" +content-hash = "b463017b75a0783cc5f8f1f5951d1d7a475d5caf0adf19aaeef2debca8cec86c" diff --git a/pyproject.toml b/pyproject.toml index 162eae80ad..8d58e78fef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,7 +105,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.133.0rc1" +version = "1.133.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" @@ -256,7 +256,6 @@ hiredis = { version = "*", optional = true } Pympler = { version = "*", optional = true } parameterized = { version = ">=0.7.4", optional = true } idna = { version = ">=2.5", optional = true } -pyicu = { version = ">=2.10.2", optional = true } [tool.poetry.extras] # NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified @@ -279,10 +278,6 @@ redis = ["txredisapi", "hiredis"] # Required to use experimental `caches.track_memory_usage` config option. cache-memory = ["pympler"] test = ["parameterized", "idna"] -# Allows for better search for international characters in the user directory. This -# requires libicu's development headers installed on the system (e.g. libicu-dev on -# Debian-based distributions). -user-search = ["pyicu"] # The duplication here is awful. I hate hate hate hate hate it. However, for now I want # to ensure you can still `pip install matrix-synapse[all]` like today. Two motivations: @@ -318,8 +313,6 @@ all = [ "hiredis", # cache-memory "pympler", - # improved user search - "pyicu", # omitted: # - test: it's useful to have this separate from dev deps in the olddeps job # - systemd: this is a system-based requirement @@ -330,7 +323,7 @@ all = [ # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.11.11" +ruff = "0.12.1" # Type checking only works with the pydantic.v1 compat module from pydantic v2 pydantic = "^2" @@ -380,7 +373,7 @@ tomli = ">=1.2.3" # runtime errors caused by build system changes. # We are happy to raise these upper bounds upon request, # provided we check that it's safe to do so (i.e. that CI passes). -requires = ["poetry-core>=1.1.0,<=1.9.1", "setuptools_rust>=1.3,<=1.10.2"] +requires = ["poetry-core>=1.1.0,<=2.1.3", "setuptools_rust>=1.3,<=1.11.1"] build-backend = "poetry.core.masonry.api" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3a47f7ddf7..dab32c8952 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -43,6 +43,7 @@ sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive"] } serde_json = "1.0.85" ulid = "1.1.2" +icu_segmenter = "2.0.0" reqwest = { version = "0.12.15", default-features = false, features = [ "http2", "stream", diff --git a/rust/src/lib.rs b/rust/src/lib.rs index e33a8cc44c..6522148fa1 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -13,6 +13,7 @@ pub mod identifier; pub mod matrix_const; pub mod push; pub mod rendezvous; +pub mod segmenter; lazy_static! { static ref LOGGING_HANDLE: ResetHandle = pyo3_log::init(); @@ -53,6 +54,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { events::register_module(py, m)?; http_client::register_module(py, m)?; rendezvous::register_module(py, m)?; + segmenter::register_module(py, m)?; Ok(()) } diff --git a/rust/src/segmenter.rs b/rust/src/segmenter.rs new file mode 100644 index 0000000000..135b3c1779 --- /dev/null +++ b/rust/src/segmenter.rs @@ -0,0 +1,33 @@ +use icu_segmenter::options::WordBreakInvariantOptions; +use icu_segmenter::WordSegmenter; +use pyo3::prelude::*; + +#[pyfunction] +pub fn parse_words(text: &str) -> PyResult> { + let segmenter = WordSegmenter::new_auto(WordBreakInvariantOptions::default()); + let mut parts = Vec::new(); + let mut last = 0usize; + + // `segment_str` gives us word boundaries as a vector of indexes. Use that + // to build a vector of words, and return. + for boundary in segmenter.segment_str(text) { + if boundary > last { + parts.push(text[last..boundary].to_string()); + } + last = boundary; + } + Ok(parts) +} + +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "segmenter")?; + child_module.add_function(wrap_pyfunction!(parse_words, m)?)?; + + m.add_submodule(&child_module)?; + + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.segmenter", child_module)?; + + Ok(()) +} diff --git a/synapse/api/auth/internal.py b/synapse/api/auth/internal.py index 9fd4db68e1..afc7b5c4a9 100644 --- a/synapse/api/auth/internal.py +++ b/synapse/api/auth/internal.py @@ -29,6 +29,7 @@ from synapse.api.errors import ( InvalidClientTokenError, MissingClientTokenError, UnrecognizedRequestError, + UserLockedError, ) from synapse.http.site import SynapseRequest from synapse.logging.opentracing import active_span, force_tracing, start_active_span @@ -162,12 +163,7 @@ class InternalAuth(BaseAuth): if not allow_locked and await self.store.get_user_locked_status( requester.user.to_string() ): - raise AuthError( - 401, - "User account has been locked", - errcode=Codes.USER_LOCKED, - additional_fields={"soft_logout": True}, - ) + raise UserLockedError() # Deny the request if the user account has expired. # This check is only done for regular users, not appservice ones. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 601a09efe1..b832c2f6a1 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -306,6 +306,20 @@ class UserDeactivatedError(SynapseError): ) +class UserLockedError(SynapseError): + """The error returned to the client when the user attempted to access an + authenticated endpoint, but the account has been locked. + """ + + def __init__(self) -> None: + super().__init__( + code=HTTPStatus.UNAUTHORIZED, + msg="User account has been locked", + errcode=Codes.USER_LOCKED, + additional_fields={"soft_logout": True}, + ) + + class FederationDeniedError(SynapseError): """An error raised when the server tries to federate with a server which is not on its federation whitelist. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 259b2c70cb..1ae47d3c16 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -561,11 +561,17 @@ class ExperimentalConfig(Config): # MSC4076: Add `disable_badge_count`` to pusher configuration self.msc4076_enabled: bool = experimental.get("msc4076_enabled", False) + # MSC4235: Add `via` param to hierarchy endpoint + self.msc4235_enabled: bool = experimental.get("msc4235_enabled", False) + # MSC4263: Preventing MXID enumeration via key queries self.msc4263_limit_key_queries_to_users_who_share_rooms = experimental.get( "msc4263_limit_key_queries_to_users_who_share_rooms", False, ) + # MSC4267: Automatically forgetting rooms on leave + self.msc4267_enabled: bool = experimental.get("msc4267_enabled", False) + # MSC4155: Invite filtering self.msc4155_enabled: bool = experimental.get("msc4155_enabled", False) diff --git a/synapse/config/room.py b/synapse/config/room.py index bad387ecbf..e698c7bafd 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -85,4 +85,4 @@ class RoomConfig(Config): # When enabled, users will forget rooms when they leave them, either via a # leave, kick or ban. - self.forget_on_leave = config.get("forget_rooms_on_leave", False) + self.forget_on_leave: bool = config.get("forget_rooms_on_leave", False) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index a85e66d6bf..c77d569e2e 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -208,7 +208,6 @@ class EventBase(metaclass=abc.ABCMeta): depth: DictProperty[int] = DictProperty("depth") content: DictProperty[JsonDict] = DictProperty("content") hashes: DictProperty[Dict[str, str]] = DictProperty("hashes") - origin: DictProperty[str] = DictProperty("origin") origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts") room_id: DictProperty[str] = DictProperty("room_id") sender: DictProperty[str] = DictProperty("sender") diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 76df083d69..afb04881df 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -302,8 +302,8 @@ def create_local_event_from_event_dict( event_dict: JsonDict, internal_metadata_dict: Optional[JsonDict] = None, ) -> EventBase: - """Takes a fully formed event dict, ensuring that fields like `origin` - and `origin_server_ts` have correct values for a locally produced event, + """Takes a fully formed event dict, ensuring that fields like + `origin_server_ts` have correct values for a locally produced event, then signs and hashes it. """ @@ -319,7 +319,6 @@ def create_local_event_from_event_dict( if format_version == EventFormatVersions.ROOM_V1_V2: event_dict["event_id"] = _create_event_id(clock, hostname) - event_dict["origin"] = hostname event_dict.setdefault("origin_server_ts", time_now) event_dict.setdefault("unsigned", {}) diff --git a/synapse/events/validator.py b/synapse/events/validator.py index d1fb026cd6..15095cc4ef 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -67,7 +67,6 @@ class EventValidator: "auth_events", "content", "hashes", - "origin", "prev_events", "sender", "type", @@ -77,13 +76,6 @@ class EventValidator: if k not in event: raise SynapseError(400, "Event does not have key %s" % (k,)) - # Check that the following keys have string values - event_strings = ["origin"] - - for s in event_strings: - if not isinstance(getattr(event, s), str): - raise SynapseError(400, "'%s' not a string type" % (s,)) - # Depending on the room version, ensure the data is spec compliant JSON. if event.room_version.strict_canonicaljson: validate_canonicaljson(event.get_pdu_json()) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 45593430e8..8d1e156dab 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -322,8 +322,7 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB SynapseError: if the pdu is missing required fields or is otherwise not a valid matrix event """ - # we could probably enforce a bunch of other fields here (room_id, sender, - # origin, etc etc) + # we could probably enforce a bunch of other fields here (room_id, sender, etc.) assert_params_in_dict(pdu_json, ("type", "depth")) # Strip any unauthorized values from "unsigned" if they exist diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index de5b38caca..9e3e70ec1c 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -76,7 +76,7 @@ from synapse.storage.databases.main.registration import ( LoginTokenLookupResult, LoginTokenReused, ) -from synapse.types import JsonDict, Requester, UserID +from synapse.types import JsonDict, Requester, StrCollection, UserID from synapse.util import stringutils as stringutils from synapse.util.async_helpers import delay_cancellation, maybe_awaitable from synapse.util.msisdn import phone_number_to_msisdn @@ -1547,6 +1547,31 @@ class AuthHandler: user_id, (token_id for _, token_id, _ in tokens_and_devices) ) + async def delete_access_tokens_for_devices( + self, + user_id: str, + device_ids: StrCollection, + ) -> None: + """Invalidate access tokens for the devices + + Args: + user_id: ID of user the tokens belong to + device_ids: ID of device the tokens are associated with. + If None, tokens associated with any device (or no device) will + be deleted + """ + tokens_and_devices = await self.store.user_delete_access_tokens_for_devices( + user_id, + device_ids, + ) + + # see if any modules want to know about this + if self.password_auth_provider.on_logged_out_callbacks: + for token, _, device_id in tokens_and_devices: + await self.password_auth_provider.on_logged_out( + user_id=user_id, device_id=device_id, access_token=token + ) + async def add_threepid( self, user_id: str, medium: str, address: str, validated_at: int ) -> None: diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 8f9bf92fda..c6e44dae6a 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -671,12 +671,12 @@ class DeviceHandler(DeviceWorkerHandler): except_device_id: optional device id which should not be deleted """ device_map = await self.store.get_devices_by_user(user_id) - device_ids = list(device_map) if except_device_id is not None: - device_ids = [d for d in device_ids if d != except_device_id] - await self.delete_devices(user_id, device_ids) + device_map.pop(except_device_id, None) + user_device_ids = device_map.keys() + await self.delete_devices(user_id, user_device_ids) - async def delete_devices(self, user_id: str, device_ids: List[str]) -> None: + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: """Delete several devices Args: @@ -695,17 +695,10 @@ class DeviceHandler(DeviceWorkerHandler): else: raise - # Delete data specific to each device. Not optimised as it is not - # considered as part of a critical path. - for device_id in device_ids: - await self._auth_handler.delete_access_tokens_for_user( - user_id, device_id=device_id - ) - await self.store.delete_e2e_keys_by_device( - user_id=user_id, device_id=device_id - ) - - if self.hs.config.experimental.msc3890_enabled: + # Delete data specific to each device. Not optimised as its an + # experimental MSC. + if self.hs.config.experimental.msc3890_enabled: + for device_id in device_ids: # Remove any local notification settings for this device in accordance # with MSC3890. await self._account_data_handler.remove_account_data_for_user( @@ -713,6 +706,13 @@ class DeviceHandler(DeviceWorkerHandler): f"org.matrix.msc3890.local_notification_settings.{device_id}", ) + # If we're deleting a lot of devices, a bunch of them may not have any + # to-device messages queued up. We filter those out to avoid scheduling + # unnecessary tasks. + devices_with_messages = await self.store.get_devices_with_messages( + user_id, device_ids + ) + for device_id in devices_with_messages: # Delete device messages asynchronously and in batches using the task scheduler # We specify an upper stream id to avoid deleting non delivered messages # if an user re-uses a device ID. @@ -726,6 +726,10 @@ class DeviceHandler(DeviceWorkerHandler): }, ) + await self._auth_handler.delete_access_tokens_for_devices( + user_id, device_ids=device_ids + ) + # Pushers are deleted after `delete_access_tokens_for_user` is called so that # modules using `on_logged_out` hook can use them if needed. await self.hs.get_pusherpool().remove_pushers_by_devices(user_id, device_ids) @@ -819,10 +823,11 @@ class DeviceHandler(DeviceWorkerHandler): # This should only happen if there are no updates, so we bail. return - for device_id in device_ids: - logger.debug( - "Notifying about update %r/%r, ID: %r", user_id, device_id, position - ) + if logger.isEnabledFor(logging.DEBUG): + for device_id in device_ids: + logger.debug( + "Notifying about update %r/%r, ID: %r", user_id, device_id, position + ) # specify the user ID too since the user should always get their own device list # updates, even if they aren't in any rooms. @@ -922,9 +927,6 @@ class DeviceHandler(DeviceWorkerHandler): # can't call self.delete_device because that will clobber the # access token so call the storage layer directly await self.store.delete_devices(user_id, [old_device_id]) - await self.store.delete_e2e_keys_by_device( - user_id=user_id, device_id=old_device_id - ) # tell everyone that the old device is gone and that the dehydrated # device has a new display name @@ -946,7 +948,6 @@ class DeviceHandler(DeviceWorkerHandler): raise errors.NotFoundError() await self.delete_devices(user_id, [device_id]) - await self.store.delete_e2e_keys_by_device(user_id=user_id, device_id=device_id) @wrap_as_background_process("_handle_new_device_update_async") async def _handle_new_device_update_async(self) -> None: diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index cdc388b4ab..76aa90e11b 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -539,11 +539,17 @@ class ProfileHandler: response: JsonDict = {} try: if just_field is None or just_field == ProfileFields.DISPLAYNAME: - response["displayname"] = await self.store.get_profile_displayname(user) - + displayname = await self.store.get_profile_displayname(user) + # do not set the displayname field if it is None, + # since then we send a null in the JSON response + if displayname is not None: + response["displayname"] = displayname if just_field is None or just_field == ProfileFields.AVATAR_URL: - response["avatar_url"] = await self.store.get_profile_avatar_url(user) - + avatar_url = await self.store.get_profile_avatar_url(user) + # do not set the avatar_url field if it is None, + # since then we send a null in the JSON response + if avatar_url is not None: + response["avatar_url"] = avatar_url if self.hs.config.experimental.msc4133_enabled: if just_field is None: response.update(await self.store.get_profile_fields(user)) diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index dcfebb128c..170c477d6f 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -54,6 +54,9 @@ class RoomPolicyHandler: Returns: bool: True if the event is allowed in the room, False otherwise. """ + if event.type == "org.matrix.msc4284.policy" and event.state_key is not None: + return True # always allow policy server change events + policy_event = await self._storage_controllers.state.get_current_state_event( event.room_id, "org.matrix.msc4284.policy", "" ) diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 91b131d09b..1f322ac263 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -111,7 +111,15 @@ class RoomSummaryHandler: # If a user tries to fetch the same page multiple times in quick succession, # only process the first attempt and return its result to subsequent requests. self._pagination_response_cache: ResponseCache[ - Tuple[str, str, bool, Optional[int], Optional[int], Optional[str]] + Tuple[ + str, + str, + bool, + Optional[int], + Optional[int], + Optional[str], + Optional[Tuple[str, ...]], + ] ] = ResponseCache( hs.get_clock(), "get_room_hierarchy", @@ -126,6 +134,7 @@ class RoomSummaryHandler: max_depth: Optional[int] = None, limit: Optional[int] = None, from_token: Optional[str] = None, + remote_room_hosts: Optional[Tuple[str, ...]] = None, ) -> JsonDict: """ Implementation of the room hierarchy C-S API. @@ -143,6 +152,9 @@ class RoomSummaryHandler: limit: An optional limit on the number of rooms to return per page. Must be a positive integer. from_token: An optional pagination token. + remote_room_hosts: An optional list of remote homeserver server names. If defined, + each host will be used to try and fetch the room hierarchy. Must be a tuple so + that it can be hashed by the `RoomSummaryHandler._pagination_response_cache`. Returns: The JSON hierarchy dictionary. @@ -162,6 +174,7 @@ class RoomSummaryHandler: max_depth, limit, from_token, + remote_room_hosts, ), self._get_room_hierarchy, requester.user.to_string(), @@ -170,6 +183,7 @@ class RoomSummaryHandler: max_depth, limit, from_token, + remote_room_hosts, ) async def _get_room_hierarchy( @@ -180,6 +194,7 @@ class RoomSummaryHandler: max_depth: Optional[int] = None, limit: Optional[int] = None, from_token: Optional[str] = None, + remote_room_hosts: Optional[Tuple[str, ...]] = None, ) -> JsonDict: """See docstring for SpaceSummaryHandler.get_room_hierarchy.""" @@ -199,7 +214,7 @@ class RoomSummaryHandler: if not local_room: room_hierarchy = await self._summarize_remote_room_hierarchy( - _RoomQueueEntry(requested_room_id, ()), + _RoomQueueEntry(requested_room_id, remote_room_hosts or ()), False, ) root_room_entry = room_hierarchy[0] @@ -240,7 +255,7 @@ class RoomSummaryHandler: processed_rooms = set(pagination_session["processed_rooms"]) else: # The queue of rooms to process, the next room is last on the stack. - room_queue = [_RoomQueueEntry(requested_room_id, ())] + room_queue = [_RoomQueueEntry(requested_room_id, remote_room_hosts or ())] # Rooms we have already processed. processed_rooms = set() diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index ebd5a33ea5..8f3193fb47 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -109,6 +109,11 @@ class CapabilitiesRestServlet(RestServlet): "disallowed" ] = disallowed + if self.config.experimental.msc4267_enabled: + response["capabilities"]["org.matrix.msc4267.forget_forced_upon_leave"] = { + "enabled": self.config.room.forget_on_leave, + } + return HTTPStatus.OK, response diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 72b219447b..8a781f759c 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -42,6 +42,7 @@ from synapse.api.errors import ( NotApprovedError, SynapseError, UserDeactivatedError, + UserLockedError, ) from synapse.api.ratelimiting import Ratelimiter from synapse.api.urls import CLIENT_API_PREFIX @@ -313,7 +314,7 @@ class LoginRestServlet(RestServlet): should_issue_refresh_token=should_issue_refresh_token, # The user represented by an appservice's configured sender_localpart # is not actually created in Synapse. - should_check_deactivated=qualified_user_id != appservice.sender, + should_check_deactivated_or_locked=qualified_user_id != appservice.sender, request_info=request_info, ) @@ -367,7 +368,7 @@ class LoginRestServlet(RestServlet): auth_provider_id: Optional[str] = None, should_issue_refresh_token: bool = False, auth_provider_session_id: Optional[str] = None, - should_check_deactivated: bool = True, + should_check_deactivated_or_locked: bool = True, *, request_info: RequestInfo, ) -> LoginResponse: @@ -389,8 +390,8 @@ class LoginRestServlet(RestServlet): should_issue_refresh_token: True if this login should issue a refresh token alongside the access token. auth_provider_session_id: The session ID got during login from the SSO IdP. - should_check_deactivated: True if the user should be checked for - deactivation status before logging in. + should_check_deactivated_or_locked: True if the user should be checked for + deactivation or locked status before logging in. This exists purely for appservice's configured sender_localpart which doesn't have an associated user in the database. @@ -415,11 +416,14 @@ class LoginRestServlet(RestServlet): ) user_id = canonical_uid - # If the account has been deactivated, do not proceed with the login. - if should_check_deactivated: + # If the account has been deactivated or locked, do not proceed with the login. + if should_check_deactivated_or_locked: deactivated = await self._main_store.get_user_deactivated_status(user_id) if deactivated: raise UserDeactivatedError("This account has been deactivated") + locked = await self._main_store.get_user_locked_status(user_id) + if locked: + raise UserLockedError() device_id = login_submission.get("device_id") diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index c64ce1f9c7..4600a87778 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -1538,6 +1538,7 @@ class RoomHierarchyRestServlet(RestServlet): super().__init__() self._auth = hs.get_auth() self._room_summary_handler = hs.get_room_summary_handler() + self.msc4235_enabled = hs.config.experimental.msc4235_enabled async def on_GET( self, request: SynapseRequest, room_id: str @@ -1547,6 +1548,15 @@ class RoomHierarchyRestServlet(RestServlet): max_depth = parse_integer(request, "max_depth") limit = parse_integer(request, "limit") + # twisted.web.server.Request.args is incorrectly defined as Optional[Any] + remote_room_hosts = None + if self.msc4235_enabled: + args: Dict[bytes, List[bytes]] = request.args # type: ignore + via_param = parse_strings_from_args( + args, "org.matrix.msc4235.via", required=False + ) + remote_room_hosts = tuple(via_param or []) + return 200, await self._room_summary_handler.get_room_hierarchy( requester, room_id, @@ -1554,6 +1564,7 @@ class RoomHierarchyRestServlet(RestServlet): max_depth=max_depth, limit=limit, from_token=parse_string(request, "from"), + remote_room_hosts=remote_room_hosts, ) diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index c2d4bf8290..df3f264b06 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -34,6 +34,7 @@ from synapse.metrics.background_process_metrics import wrap_as_background_proces from synapse.storage.database import LoggingTransaction from synapse.storage.databases import Databases from synapse.types.storage import _BackgroundUpdates +from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer @@ -167,6 +168,12 @@ class PurgeEventsStorageController: break (room_id, groups_to_sequences) = next_to_delete + + logger.info( + "[purge] deleting state groups for room %s: %s", + room_id, + shortstr(groups_to_sequences.keys(), maxitems=10), + ) made_progress = await self._delete_state_groups( room_id, groups_to_sequences ) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index a4941e58f6..6188195614 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -1478,13 +1478,49 @@ class DatabasePool: """ Upsert, many times. + This executes a query equivalent to `INSERT INTO ... ON CONFLICT DO UPDATE`, + with multiple value rows. + The query may use emulated upserts if the database engine does not support upserts, + or if the table is currently unsafe to upsert. + + If there are no value columns, this instead generates a `ON CONFLICT DO NOTHING`. + Args: table: The table to upsert into - key_names: The key column names. - key_values: A list of each row's key column values. - value_names: The value column names - value_values: A list of each row's value column values. + key_names: The unique key column names. These are the columns used in the ON CONFLICT clause. + key_values: A list of each row's key column values, in the same order as `key_names`. + value_names: The non-unique value column names + value_values: A list of each row's value column values, in the same order as `value_names`. Ignored if value_names is empty. + + Example: + ```python + simple_upsert_many( + "mytable", + key_names=("room_id", "user_id"), + key_values=[ + ("!room1:example.org", "@user1:example.org"), + ("!room2:example.org", "@user2:example.org"), + ], + value_names=("wombat_count", "is_updated"), + value_values=[ + (42, True), + (7, False) + ], + ) + ``` + + gives something equivalent to: + + ```sql + INSERT INTO mytable (room_id, user_id, wombat_count, is_updated) + VALUES + ('!room1:example.org', '@user1:example.org', 42, True), + ('!room2:example.org', '@user2:example.org', 7, False) + ON CONFLICT DO UPDATE SET + wombat_count = EXCLUDED.wombat_count, + is_updated = EXCLUDED.is_updated + ``` """ # We can autocommit if it safe to upsert @@ -1513,6 +1549,8 @@ class DatabasePool: """ Upsert, many times. + See the documentation for `simple_upsert_many` for examples. + Args: table: The table to upsert into key_names: The key column names. diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 579d29ac15..a22eab2474 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -52,10 +52,11 @@ from synapse.storage.database import ( make_in_list_sql_clause, ) from synapse.storage.util.id_generators import MultiWriterIdGenerator -from synapse.types import JsonDict +from synapse.types import JsonDict, StrCollection from synapse.util import Duration, json_encoder from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.caches.stream_change_cache import StreamChangeCache +from synapse.util.iterutils import batch_iter from synapse.util.stringutils import parse_and_validate_server_name if TYPE_CHECKING: @@ -1027,6 +1028,40 @@ class DeviceInboxWorkerStore(SQLBaseStore): # loop first time we run this. self._clock.sleep(1) + async def get_devices_with_messages( + self, user_id: str, device_ids: StrCollection + ) -> StrCollection: + """Get the matching device IDs that have messages in the device inbox.""" + + def get_devices_with_messages_txn( + txn: LoggingTransaction, + batch_device_ids: StrCollection, + ) -> StrCollection: + clause, args = make_in_list_sql_clause( + self.database_engine, "device_id", batch_device_ids + ) + sql = f""" + SELECT DISTINCT device_id FROM device_inbox + WHERE {clause} AND user_id = ? + """ + args.append(user_id) + txn.execute(sql, args) + return {row[0] for row in txn} + + results: Set[str] = set() + for batch_device_ids in batch_iter(device_ids, 1000): + batch_results = await self.db_pool.runInteraction( + "get_devices_with_messages", + get_devices_with_messages_txn, + batch_device_ids, + # We don't need to run in a transaction as it's a single query + db_autocommit=True, + ) + + results.update(batch_results) + + return results + class DeviceInboxBackgroundUpdateStore(SQLBaseStore): DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop" diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 6191f22cd6..941d278e6c 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -282,7 +282,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): "count_devices_by_users", count_devices_by_users_txn, user_ids ) - @cached() + @cached(tree=True) async def get_device( self, user_id: str, device_id: str ) -> Optional[Mapping[str, Any]]: @@ -1861,7 +1861,7 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): ) raise StoreError(500, "Problem storing device.") - async def delete_devices(self, user_id: str, device_ids: List[str]) -> None: + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: """Deletes several devices. Args: @@ -1885,11 +1885,49 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): values=device_ids, keyvalues={"user_id": user_id}, ) - self._invalidate_cache_and_stream_bulk( - txn, self.get_device, [(user_id, device_id) for device_id in device_ids] + + # Also delete associated e2e keys. + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_device_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_one_time_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="dehydrated_devices", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_fallback_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, ) - for batch in batch_iter(device_ids, 100): + # We're bulk deleting potentially many devices at once, so + # let's not invalidate the cache for each device individually. + # Instead, we will invalidate the cache for the user as a whole. + self._invalidate_cache_and_stream(txn, self.get_device, (user_id,)) + self._invalidate_cache_and_stream( + txn, self.count_e2e_one_time_keys, (user_id,) + ) + self._invalidate_cache_and_stream( + txn, self.get_e2e_unused_fallback_key_types, (user_id,) + ) + + for batch in batch_iter(device_ids, 1000): await self.db_pool.runInteraction( "delete_devices", _delete_devices_txn, batch ) @@ -2061,32 +2099,36 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): context = get_active_span_text_map() def add_device_changes_txn( - txn: LoggingTransaction, stream_ids: List[int] + txn: LoggingTransaction, + batch_device_ids: StrCollection, + stream_ids: List[int], ) -> None: self._add_device_change_to_stream_txn( txn, user_id, - device_ids, + batch_device_ids, stream_ids, ) self._add_device_outbound_room_poke_txn( txn, user_id, - device_ids, + batch_device_ids, room_ids, stream_ids, context, ) - async with self._device_list_id_gen.get_next_mult( - len(device_ids) - ) as stream_ids: - await self.db_pool.runInteraction( - "add_device_change_to_stream", - add_device_changes_txn, - stream_ids, - ) + for batch_device_ids in batch_iter(device_ids, 1000): + async with self._device_list_id_gen.get_next_mult( + len(device_ids) + ) as stream_ids: + await self.db_pool.runInteraction( + "add_device_change_to_stream", + add_device_changes_txn, + batch_device_ids, + stream_ids, + ) return stream_ids[-1] diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 341e7014d6..0700b0087b 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -593,7 +593,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker txn, self.count_e2e_one_time_keys, (user_id, device_id) ) - @cached(max_entries=10000) + @cached(max_entries=10000, tree=True) async def count_e2e_one_time_keys( self, user_id: str, device_id: str ) -> Mapping[str, int]: @@ -808,7 +808,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker }, ) - @cached(max_entries=10000) + @cached(max_entries=10000, tree=True) async def get_e2e_unused_fallback_key_types( self, user_id: str, device_id: str ) -> Sequence[str]: @@ -1632,46 +1632,6 @@ class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): log_kv({"message": "Device keys stored."}) return True - async def delete_e2e_keys_by_device(self, user_id: str, device_id: str) -> None: - def delete_e2e_keys_by_device_txn(txn: LoggingTransaction) -> None: - log_kv( - { - "message": "Deleting keys for device", - "device_id": device_id, - "user_id": user_id, - } - ) - self.db_pool.simple_delete_txn( - txn, - table="e2e_device_keys_json", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self.db_pool.simple_delete_txn( - txn, - table="e2e_one_time_keys_json", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self._invalidate_cache_and_stream( - txn, self.count_e2e_one_time_keys, (user_id, device_id) - ) - self.db_pool.simple_delete_txn( - txn, - table="dehydrated_devices", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self.db_pool.simple_delete_txn( - txn, - table="e2e_fallback_keys_json", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self._invalidate_cache_and_stream( - txn, self.get_e2e_unused_fallback_key_types, (user_id, device_id) - ) - - await self.db_pool.runInteraction( - "delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn - ) - def _set_e2e_cross_signing_key_txn( self, txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 3db4460f57..9cc0493307 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -349,6 +349,19 @@ class EventsWorkerStore(SQLBaseStore): where_clause="type = 'm.room.member'", ) + # Added to support efficient reverse lookups on the foreign key + # (user_id, device_id) when deleting devices. + # We already had a UNIQUE index on these 4 columns but out-of-order + # so replace that one. + self.db_pool.updates.register_background_index_update( + update_name="event_txn_id_device_id_txn_id2", + index_name="event_txn_id_device_id_txn_id2", + table="event_txn_id_device_id", + columns=("user_id", "device_id", "room_id", "txn_id"), + unique=True, + replaces_index="event_txn_id_device_id_txn_id", + ) + def get_un_partial_stated_events_token(self, instance_name: str) -> int: return ( self._un_partial_stated_events_stream_id_gen.get_current_token_for_writer( diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 40c551bcb4..1e21996b12 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -40,14 +40,16 @@ from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, LoggingTransaction, + make_in_list_sql_clause, ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.stats import StatsStore from synapse.storage.types import Cursor from synapse.storage.util.id_generators import IdGenerator from synapse.storage.util.sequence import build_sequence_generator -from synapse.types import JsonDict, UserID, UserInfo +from synapse.types import JsonDict, StrCollection, UserID, UserInfo from synapse.util.caches.descriptors import cached +from synapse.util.iterutils import batch_iter if TYPE_CHECKING: from synapse.server import HomeServer @@ -2801,6 +2803,81 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): return await self.db_pool.runInteraction("user_delete_access_tokens", f) + async def user_delete_access_tokens_for_devices( + self, + user_id: str, + device_ids: StrCollection, + ) -> List[Tuple[str, int, Optional[str]]]: + """ + Invalidate access and refresh tokens belonging to a user + + Args: + user_id: ID of user the tokens belong to + device_ids: The devices to delete tokens for. + Returns: + A tuple of (token, token id, device id) for each of the deleted tokens + """ + + def user_delete_access_tokens_for_devices_txn( + txn: LoggingTransaction, batch_device_ids: StrCollection + ) -> List[Tuple[str, int, Optional[str]]]: + self.db_pool.simple_delete_many_txn( + txn, + table="refresh_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + clause, args = make_in_list_sql_clause( + txn.database_engine, "device_id", batch_device_ids + ) + args.append(user_id) + + if self.database_engine.supports_returning: + sql = f""" + DELETE FROM access_tokens + WHERE {clause} AND user_id = ? + RETURNING token, id, device_id + """ + txn.execute(sql, args) + tokens_and_devices = txn.fetchall() + else: + tokens_and_devices = self.db_pool.simple_select_many_txn( + txn, + table="access_tokens", + column="device_id", + iterable=batch_device_ids, + keyvalues={"user_id": user_id}, + retcols=("token", "id", "device_id"), + ) + + self.db_pool.simple_delete_many_txn( + txn, + table="access_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + self._invalidate_cache_and_stream_bulk( + txn, + self.get_user_by_access_token, + [(t[0],) for t in tokens_and_devices], + ) + return tokens_and_devices + + results = [] + for batch_device_ids in batch_iter(device_ids, 1000): + tokens_and_devices = await self.db_pool.runInteraction( + "user_delete_access_tokens_for_devices", + user_delete_access_tokens_for_devices_txn, + batch_device_ids, + ) + results.extend(tokens_and_devices) + + return results + async def delete_access_token(self, access_token: str) -> None: def f(txn: LoggingTransaction) -> None: self.db_pool.simple_delete_one_txn( diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 09671abcad..9deb9ab73c 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -37,16 +37,8 @@ from typing import ( import attr -try: - # Figure out if ICU support is available for searching users. - import icu - - USE_ICU = True -except ModuleNotFoundError: - # except ModuleNotFoundError: - USE_ICU = False - from synapse.api.errors import StoreError +from synapse.synapse_rust import segmenter as icu from synapse.util.stringutils import non_null_str_or_none if TYPE_CHECKING: @@ -1226,7 +1218,7 @@ def _filter_text_for_index(text: str) -> str: def _parse_query_sqlite(search_term: str) -> str: """Takes a plain unicode string from the user and converts it into a form - that can be passed to database. + that can be passed to the database. We use this so that we can add prefix matching, which isn't something that is supported by default. @@ -1242,7 +1234,7 @@ def _parse_query_sqlite(search_term: str) -> str: def _parse_query_postgres(search_term: str) -> Tuple[str, str, str]: """Takes a plain unicode string from the user and converts it into a form - that can be passed to database. + that can be passed to the database. We use this so that we can add prefix matching, which isn't something that is supported by default. """ @@ -1272,12 +1264,7 @@ def _parse_query_postgres(search_term: str) -> Tuple[str, str, str]: def _parse_words(search_term: str) -> List[str]: - """Split the provided search string into a list of its words. - - If support for ICU (International Components for Unicode) is available, use it. - Otherwise, fall back to using a regex to detect word boundaries. This latter - solution works well enough for most latin-based languages, but doesn't work as well - with other languages. + """Split the provided search string into a list of its words using ICU. Args: search_term: The search string. @@ -1285,18 +1272,7 @@ def _parse_words(search_term: str) -> List[str]: Returns: A list of the words in the search string. """ - if USE_ICU: - return _parse_words_with_icu(search_term) - - return _parse_words_with_regex(search_term) - - -def _parse_words_with_regex(search_term: str) -> List[str]: - """ - Break down search term into words, when we don't have ICU available. - See: `_parse_words` - """ - return re.findall(r"([\w-]+)", search_term, re.UNICODE) + return _parse_words_with_icu(search_term) def _parse_words_with_icu(search_term: str) -> List[str]: @@ -1310,22 +1286,13 @@ def _parse_words_with_icu(search_term: str) -> List[str]: A list of the words in the search string. """ results = [] - breaker = icu.BreakIterator.createWordInstance(icu.Locale.getDefault()) - breaker.setText(search_term) - i = 0 - while True: - j = breaker.nextBoundary() - if j < 0: - break - + for part in icu.parse_words(search_term): # We want to make sure that we split on `@` and `:` specifically, as # they occur in user IDs. - for result in re.split(r"[@:]+", search_term[i:j]): + for result in re.split(r"[@:]+", part): results.append(result.strip()) - i = j - - # libicu will break up words that have punctuation in them, but to handle + # icu will break up words that have punctuation in them, but to handle # cases where user IDs have '-', '.' and '_' in them we want to *not* break # those into words and instead allow the DB to tokenise them how it wants. # diff --git a/synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql b/synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql new file mode 100644 index 0000000000..b568fc04f2 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9207, 'event_txn_id_device_id_txn_id2', '{}'); diff --git a/synapse/synapse_rust/segmenter.pyi b/synapse/synapse_rust/segmenter.pyi new file mode 100644 index 0000000000..5f36765947 --- /dev/null +++ b/synapse/synapse_rust/segmenter.pyi @@ -0,0 +1,3 @@ +from typing import List + +def parse_words(text: str) -> List[str]: ... diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index df8829baeb..76e6e139c6 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -225,7 +225,7 @@ KNOWN_KEYS = { "depth", "event_id", "hashes", - "origin", + "origin", # old events were created with an origin field. "origin_server_ts", "prev_events", "room_id", diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index d7b9fb8bc6..9cdc1604da 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -48,7 +48,6 @@ class EventSigningTestCase(unittest.TestCase): def test_sign_minimal(self) -> None: event_dict = { "event_id": "$0:domain", - "origin": "domain", "origin_server_ts": 1000000, "signatures": {}, "type": "X", @@ -64,7 +63,7 @@ class EventSigningTestCase(unittest.TestCase): self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) self.assertEqual( - event.hashes["sha256"], "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI" + event.hashes["sha256"], "A6Nco6sqoy18PPfPDVdYvoowfc0PVBk9g9OiyT3ncRM" ) self.assertTrue(hasattr(event, "signatures")) @@ -72,15 +71,14 @@ class EventSigningTestCase(unittest.TestCase): self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEqual( event.signatures[HOSTNAME][KEY_NAME], - "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+" - "aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA", + "PBc48yDVszWB9TRaB/+CZC1B+pDAC10F8zll006j+NN" + "fe4PEMWcVuLaG63LFTK9e4rwJE8iLZMPtCKhDTXhpAQ", ) def test_sign_message(self) -> None: event_dict = { "content": {"body": "Here is the message content"}, "event_id": "$0:domain", - "origin": "domain", "origin_server_ts": 1000000, "type": "m.room.message", "room_id": "!r:domain", @@ -98,7 +96,7 @@ class EventSigningTestCase(unittest.TestCase): self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) self.assertEqual( - event.hashes["sha256"], "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g" + event.hashes["sha256"], "rDCeYBepPlI891h/RkI2/Lkf9bt7u0TxFku4tMs7WKk" ) self.assertTrue(hasattr(event, "signatures")) @@ -106,6 +104,6 @@ class EventSigningTestCase(unittest.TestCase): self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEqual( event.signatures[HOSTNAME][KEY_NAME], - "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUw" - "u6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA", + "Ay4aj2b5oJ1k8INYZ9n3KnszCflM0emwcmQQ7vxpbdc" + "Sv9bkJxIZdWX1IJllcZLq89+D3sSabE+vqPtZs9akDw", ) diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 654e6521a2..521251a97e 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -122,7 +122,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase): "prev_events": "prev_events", "prev_state": "prev_state", "auth_events": "auth_events", - "origin": "domain", + "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", # Also include a key that should be removed. @@ -139,7 +139,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase): "prev_events": "prev_events", "prev_state": "prev_state", "auth_events": "auth_events", - "origin": "domain", + "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", "content": {}, @@ -148,13 +148,12 @@ class PruneEventTestCase(stdlib_unittest.TestCase): }, ) - # As of room versions we now redact the membership, prev_states, and origin keys. + # As of room versions we now redact the membership and prev_states keys. self.run_test( { "type": "A", "prev_state": "prev_state", "membership": "join", - "origin": "example.com", }, {"type": "A", "content": {}, "signatures": {}, "unsigned": {}}, room_version=RoomVersions.V11, @@ -238,7 +237,6 @@ class PruneEventTestCase(stdlib_unittest.TestCase): { "type": "m.room.create", "content": {"not_a_real_key": True}, - "origin": "some_homeserver", "nonsense_field": "some_random_garbage", }, { diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index a7b702fbd1..58ead90909 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -535,7 +535,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): "depth": 1000, "origin_server_ts": 1, "type": "m.room.member", - "origin": "test.servx", "content": {"membership": "join"}, "auth_events": [], "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"}, @@ -552,7 +551,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): "depth": 1000, "origin_server_ts": 1, "type": "m.room.member", - "origin": "test.servx", "auth_events": [], "content": {"membership": "join"}, "unsigned": { @@ -579,7 +577,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): "depth": 1000, "origin_server_ts": 1, "type": "m.room.power_levels", - "origin": "test.servx", "content": {}, "auth_events": [], "unsigned": { diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index b55fa1a8fd..bf18c1e72a 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -1080,6 +1080,62 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): self.assertEqual(federation_requests, 2) self._assert_hierarchy(result, expected) + def test_fed_remote_room_hosts(self) -> None: + """ + Test if requested room is available over federation using via's. + """ + fed_hostname = self.hs.hostname + "2" + fed_space = "#fed_space:" + fed_hostname + fed_subroom = "#fed_sub_room:" + fed_hostname + + remote_room_hosts = tuple(fed_hostname) + + requested_room_entry = _RoomEntry( + fed_space, + { + "room_id": fed_space, + "world_readable": True, + "join_rule": "public", + "room_type": RoomTypes.SPACE, + }, + [ + { + "type": EventTypes.SpaceChild, + "room_id": fed_space, + "state_key": fed_subroom, + "content": {"via": [fed_hostname]}, + } + ], + ) + child_room = { + "room_id": fed_subroom, + "world_readable": True, + "join_rule": "public", + } + + async def summarize_remote_room_hierarchy( + _self: Any, room: Any, suggested_only: bool + ) -> Tuple[Optional[_RoomEntry], Dict[str, JsonDict], Set[str]]: + return requested_room_entry, {fed_subroom: child_room}, set() + + expected = [ + (fed_space, [fed_subroom]), + (fed_subroom, ()), + ] + + with mock.patch( + "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room_hierarchy", + new=summarize_remote_room_hierarchy, + ): + result = self.get_success( + self.handler.get_room_hierarchy( + create_requester(self.user), + fed_space, + remote_room_hosts=remote_room_hosts, + ) + ) + self._assert_hierarchy(result, expected) + class RoomSummaryTestCase(unittest.HomeserverTestCase): servlets = [ diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 5f73dbdc4a..e9c631e2aa 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -2846,6 +2846,16 @@ class UserRestTestCase(unittest.HomeserverTestCase): self.assertEqual(Codes.USER_LOCKED, channel.json_body["errcode"]) self.assertTrue(channel.json_body["soft_logout"]) + # User is not authorized to log in anymore + channel = self.make_request( + "POST", + "/_matrix/client/r0/login", + {"type": "m.login.password", "user": "user", "password": "pass"}, + ) + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.USER_LOCKED, channel.json_body["errcode"]) + self.assertTrue(channel.json_body["soft_logout"]) + @override_config({"user_directory": {"enabled": True, "search_all_users": True}}) def test_locked_user_not_in_user_dir(self) -> None: # User is available in the user dir diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index 8af00221c2..13831462e8 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -264,3 +264,43 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertTrue(capabilities["m.get_login_token"]["enabled"]) + + @override_config( + { + "experimental_features": {"msc4267_enabled": True}, + "forget_rooms_on_leave": True, + } + ) + def test_get_forget_forced_upon_leave_with_auto_forget(self) -> None: + # Server auto-forgets on /leave, expect enabled client capability + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertTrue( + capabilities["org.matrix.msc4267.forget_forced_upon_leave"]["enabled"] + ) + + @override_config( + { + "experimental_features": {"msc4267_enabled": True}, + "forget_rooms_on_leave": False, + } + ) + def test_get_forget_forced_upon_leave_without_auto_forget(self) -> None: + # Server doesn't auto-forget on /leave, expect disabled client capability + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse( + capabilities["org.matrix.msc4267.forget_forced_upon_leave"]["enabled"] + ) diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py index c26932069f..f97ca12d84 100644 --- a/tests/storage/test_user_directory.py +++ b/tests/storage/test_user_directory.py @@ -32,10 +32,8 @@ from synapse.rest.client import login, register, room from synapse.server import HomeServer from synapse.storage import DataStore from synapse.storage.background_updates import _BackgroundUpdateHandler -from synapse.storage.databases.main import user_directory from synapse.storage.databases.main.user_directory import ( _parse_words_with_icu, - _parse_words_with_regex, ) from synapse.storage.roommember import ProfileInfo from synapse.util import Clock @@ -44,12 +42,6 @@ from tests.server import ThreadedMemoryReactorClock from tests.test_utils.event_injection import inject_member_event from tests.unittest import HomeserverTestCase, override_config -try: - import icu -except ImportError: - icu = None # type: ignore - - ALICE = "@alice:a" BOB = "@bob:b" BOBBY = "@bobby:a" @@ -438,8 +430,6 @@ class UserDirectoryInitialPopulationTestcase(HomeserverTestCase): class UserDirectoryStoreTestCase(HomeserverTestCase): - use_icu = False - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastores().main @@ -451,12 +441,6 @@ class UserDirectoryStoreTestCase(HomeserverTestCase): self.get_success(self.store.update_profile_in_user_dir(BELA, "Bela", None)) self.get_success(self.store.add_users_in_public_rooms("!room:id", (ALICE, BOB))) - self._restore_use_icu = user_directory.USE_ICU - user_directory.USE_ICU = self.use_icu - - def tearDown(self) -> None: - user_directory.USE_ICU = self._restore_use_icu - def test_search_user_dir(self) -> None: # normally when alice searches the directory she should just find # bob because bobby doesn't share a room with her. @@ -648,24 +632,14 @@ class UserDirectoryStoreTestCase(HomeserverTestCase): test_search_user_dir_accent_insensitivity.skip = "not supported yet" # type: ignore -class UserDirectoryStoreTestCaseWithIcu(UserDirectoryStoreTestCase): - use_icu = True - - if not icu: - skip = "Requires PyICU" - - class UserDirectoryICUTestCase(HomeserverTestCase): - if not icu: - skip = "Requires PyICU" - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastores().main self.user_dir_helper = GetUserDirectoryTables(self.store) def test_icu_word_boundary(self) -> None: - """Tests that we correctly detect word boundaries when ICU (International - Components for Unicode) support is available. + """Tests that we correctly detect word boundaries with ICU + (International Components for Unicode). """ display_name = "Gáo" @@ -714,12 +688,3 @@ class UserDirectoryICUTestCase(HomeserverTestCase): self.assertEqual(_parse_words_with_icu("user-1"), ["user-1"]) self.assertEqual(_parse_words_with_icu("user-ab"), ["user-ab"]) self.assertEqual(_parse_words_with_icu("user.--1"), ["user", "-1"]) - - def test_regex_word_boundary_punctuation(self) -> None: - """ - Tests the behaviour of punctuation with the non-ICU tokeniser - """ - self.assertEqual( - _parse_words_with_regex("lazy'fox jumped:over the.dog"), - ["lazy", "fox", "jumped", "over", "the", "dog"], - )