diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c451e49301..801d369c48 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -67,7 +67,7 @@ jobs: - name: Get team registry token id: import-secrets - uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 + uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0 with: url: https://vault.infra.ci.i.element.dev role: ${{ steps.vault-jwt-role.outputs.role_name }} @@ -164,7 +164,7 @@ jobs: - name: Get team registry token id: import-secrets - uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 + uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0 with: url: https://vault.infra.ci.i.element.dev role: ${{ steps.vault-jwt-role.outputs.role_name }} @@ -186,7 +186,7 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Install Cosign - uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1 + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Calculate docker image tag uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 diff --git a/CHANGES.md b/CHANGES.md index 95ab39d0e3..c3ed13ecb5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,40 @@ +# Synapse 1.154.0rc1 (2026-05-27) + +## Features + +- Add support for [MSC4452: Preview URL capabilities API](https://github.com/matrix-org/matrix-spec-proposals/pull/4452) which exposes a `io.element.msc4452.preview_url` capability. + If `experimental_features.msc4452_enabled` is `true`, the `/_matrix/(client/v1/media|media/v3)/preview_url` endpoint + now responds with a 403 status code when the capability is disabled. ([\#19715](https://github.com/element-hq/synapse/issues/19715)) + +## Bugfixes + +- Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. ([\#19468](https://github.com/element-hq/synapse/issues/19468), [\#19791](https://github.com/element-hq/synapse/issues/19791)) +- Fix access token cache not being invalidated for sessions using refresh tokens. Contributed by @FrenchGithubUser @ Famedly. ([\#19483](https://github.com/element-hq/synapse/issues/19483)) +- Fix bug where Synapse would return 400 (`M_BAD_JSON`) when sending a message with a `mentions` field and Synapse module `check_event_allowed` callback registered (frozen event). Contributed by @gaetan-sbt. ([\#19634](https://github.com/element-hq/synapse/issues/19634)) +- Fix long-standing but niche bug with `/sync` where it could attempt to fetch data with flawed invalid future tokens. ([\#19644](https://github.com/element-hq/synapse/issues/19644)) +- Fix `/sync` failing when [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) are enabled and the sync request filters out Ephemeral Data Units (EDUs). ([\#19787](https://github.com/element-hq/synapse/issues/19787)) +- Fix packaging for Fedora and EPEL caused by unnecessary bumping `attrs` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. ([\#19789](https://github.com/element-hq/synapse/issues/19789)) +- Fix merging signatures when a policy server is running under the same server name as Synapse. The bug was re-introduced in v1.153.0rc1 after being fixed earlier in v1.151.0rc1. Contributed by @tulir @ Beeper. ([\#19797](https://github.com/element-hq/synapse/issues/19797)) + +## Improved Documentation + +- Added details about how Synapse syncs the picture claim when `update_profile_information` setting is true. ([\#19508](https://github.com/element-hq/synapse/issues/19508)) + +## Internal Changes + +- Port `Event.content` field to Rust. ([\#19725](https://github.com/element-hq/synapse/issues/19725)) +- Prefer close backfill points (absolute distance). ([\#19748](https://github.com/element-hq/synapse/issues/19748)) +- Replace unique `quarantined_media` waiting patterns with standard `wait_for_stream_token(...)`. ([\#19764](https://github.com/element-hq/synapse/issues/19764)) +- Improve Synapse logging around when someone encounters `We can't get valid state history.` so you can correlate everything by `event_id`. ([\#19765](https://github.com/element-hq/synapse/issues/19765)) +- Tidy up Rust `RoomVersion` structs. ([\#19766](https://github.com/element-hq/synapse/issues/19766)) +- Update `WorkerLock` tests to better stress the `WORKER_LOCK_MAX_RETRY_INTERVAL`. ([\#19772](https://github.com/element-hq/synapse/issues/19772)) +- Refactor [MSC4242: State DAG](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) checks behind a single `TypeIs` helper to avoid scattered `isinstance` casts. ([\#19774](https://github.com/element-hq/synapse/issues/19774)) +- Use `StrCollection` for `prev_state_events`. ([\#19777](https://github.com/element-hq/synapse/issues/19777)) +- Fix up the construction of events in tests, ahead of the Rust event port. ([\#19781](https://github.com/element-hq/synapse/issues/19781)) + + + + # Synapse 1.153.0 (2026-05-19) No significant changes since 1.153.0rc3. diff --git a/Cargo.lock b/Cargo.lock index 8285c7cf38..761e99b3e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -952,9 +952,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", "rand_core", diff --git a/changelog.d/19468.bugfix b/changelog.d/19468.bugfix deleted file mode 100644 index 003716d296..0000000000 --- a/changelog.d/19468.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. diff --git a/changelog.d/19483.bugfix b/changelog.d/19483.bugfix deleted file mode 100644 index 9e4fb20996..0000000000 --- a/changelog.d/19483.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix access token cache not being invalidated for sessions using refresh tokens. Contributed by @FrenchGithubUser @ Famedly. diff --git a/changelog.d/19508.doc b/changelog.d/19508.doc deleted file mode 100644 index 2550116341..0000000000 --- a/changelog.d/19508.doc +++ /dev/null @@ -1 +0,0 @@ -Added details about how Synapse syncs the picture claim when `update_profile_information` setting is true. diff --git a/changelog.d/19634.bugfix b/changelog.d/19634.bugfix deleted file mode 100644 index e8fcb43570..0000000000 --- a/changelog.d/19634.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where Synapse would return 400 (`M_BAD_JSON`) when sending a message with `mentions` field and Synapse module `check_event_allowed` callback registered (frozen event). Contributed by @gaetan-sbt. \ No newline at end of file diff --git a/changelog.d/19644.bugfix b/changelog.d/19644.bugfix deleted file mode 100644 index 73ab4bc63e..0000000000 --- a/changelog.d/19644.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix long-standing but niche bug with sync where it could attempt to fetch data with flawed invalid future tokens. diff --git a/changelog.d/19715.feature b/changelog.d/19715.feature deleted file mode 100644 index 973fe66e7d..0000000000 --- a/changelog.d/19715.feature +++ /dev/null @@ -1,3 +0,0 @@ -Add support for "MSC4452 Preview URL capabilities API" which exposes a `io.element.msc4452.preview_url` capability. -If `experimental_features.msc4452_enabled` is `true`, the `/_matrix/(client/v1/media|media/v3)/preview_url` endpoint -now responds with a 403 status code when the capability is disabled. diff --git a/changelog.d/19725.misc b/changelog.d/19725.misc deleted file mode 100644 index b320f42b9c..0000000000 --- a/changelog.d/19725.misc +++ /dev/null @@ -1 +0,0 @@ -Port `Event.content` field to Rust. diff --git a/changelog.d/19730.bugfix b/changelog.d/19730.bugfix new file mode 100644 index 0000000000..074405d944 --- /dev/null +++ b/changelog.d/19730.bugfix @@ -0,0 +1 @@ +Work around bug that sometimes breaks joining restricted rooms that require a remote join. Contributed by @tulir @ Beeper. diff --git a/changelog.d/19748.misc b/changelog.d/19748.misc deleted file mode 100644 index eedd4e92a2..0000000000 --- a/changelog.d/19748.misc +++ /dev/null @@ -1 +0,0 @@ -Prefer close backfill points (absolute distance). diff --git a/changelog.d/19764.misc b/changelog.d/19764.misc deleted file mode 100644 index 8704e3eed6..0000000000 --- a/changelog.d/19764.misc +++ /dev/null @@ -1 +0,0 @@ -Replace unique `quarantined_media` waiting patterns with standard `wait_for_stream_token(...)`. diff --git a/changelog.d/19765.misc b/changelog.d/19765.misc deleted file mode 100644 index 7ac5375c7e..0000000000 --- a/changelog.d/19765.misc +++ /dev/null @@ -1 +0,0 @@ -Improve Synapse logging around when someone encounters `We can't get valid state history.` so you can correlate everything by `event_id`. diff --git a/changelog.d/19766.misc b/changelog.d/19766.misc deleted file mode 100644 index 699852daa2..0000000000 --- a/changelog.d/19766.misc +++ /dev/null @@ -1 +0,0 @@ -Tidy up Rust `RoomVersion` structs. diff --git a/changelog.d/19772.misc b/changelog.d/19772.misc deleted file mode 100644 index 939507f5c3..0000000000 --- a/changelog.d/19772.misc +++ /dev/null @@ -1 +0,0 @@ -Update `WorkerLock` tests to better stress the `WORKER_LOCK_MAX_RETRY_INTERVAL`. diff --git a/changelog.d/19774.misc b/changelog.d/19774.misc deleted file mode 100644 index 5a2cb4d800..0000000000 --- a/changelog.d/19774.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor MSC4242 state DAG checks behind a single `TypeIs` helper to avoid scattered `isinstance` casts. diff --git a/changelog.d/19775.misc b/changelog.d/19775.misc new file mode 100644 index 0000000000..3ff0fd34b9 --- /dev/null +++ b/changelog.d/19775.misc @@ -0,0 +1 @@ +Add `GcpJsonFormatter` logging formatter for use with Google Cloud Logging and GKE deployments. \ No newline at end of file diff --git a/changelog.d/19777.misc b/changelog.d/19777.misc deleted file mode 100644 index cd049f8153..0000000000 --- a/changelog.d/19777.misc +++ /dev/null @@ -1 +0,0 @@ -Use `StrCollection` for `prev_state_events`. diff --git a/changelog.d/19781.misc b/changelog.d/19781.misc deleted file mode 100644 index 63c2482ded..0000000000 --- a/changelog.d/19781.misc +++ /dev/null @@ -1 +0,0 @@ -Fix up event-construction in tests ahead of the Rust event port. diff --git a/changelog.d/19784.bugfix b/changelog.d/19784.bugfix deleted file mode 100644 index c68524d57a..0000000000 --- a/changelog.d/19784.bugfix +++ /dev/null @@ -1 +0,0 @@ -Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' due to performance problems. diff --git a/changelog.d/19787.bugfix b/changelog.d/19787.bugfix deleted file mode 100644 index 26bd3e2252..0000000000 --- a/changelog.d/19787.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `/sync` failing when [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) are enabled and the sync request filters out Ephemeral Data Units (EDUs). \ No newline at end of file diff --git a/changelog.d/19789.bugfix b/changelog.d/19789.bugfix deleted file mode 100644 index f6c325ec82..0000000000 --- a/changelog.d/19789.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix packaging for Fedora and EPEL caused by unnecessary bumping `attrs` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. diff --git a/changelog.d/19791.bugfix b/changelog.d/19791.bugfix deleted file mode 100644 index 003716d296..0000000000 --- a/changelog.d/19791.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. diff --git a/changelog.d/19801.misc b/changelog.d/19801.misc new file mode 100644 index 0000000000..56ec0f7ddc --- /dev/null +++ b/changelog.d/19801.misc @@ -0,0 +1 @@ +Add more logging to the to-device message replication stream. diff --git a/complement/go.mod b/complement/go.mod index d1ccf6b496..5ebf19bbf6 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -1,8 +1,6 @@ module github.com/element-hq/synapse -go 1.24.1 - -toolchain go1.24.4 +go 1.25.0 require ( github.com/matrix-org/complement v0.0.0-20251120181401-44111a2a8a9d @@ -16,7 +14,7 @@ require ( github.com/hashicorp/go-set/v3 v3.0.0 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/oleiade/lane/v2 v2.0.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect gotest.tools/v3 v3.4.0 // indirect ) @@ -47,13 +45,15 @@ require ( github.com/tidwall/sjson v1.2.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect - go.opentelemetry.io/proto/otlp v1.7.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/sys v0.38.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/time v0.11.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/tools v0.42.0 // indirect ) diff --git a/complement/go.sum b/complement/go.sum index f82487882b..25c6922669 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -2,8 +2,8 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -38,8 +38,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/go-set/v3 v3.0.0 h1:CaJBQvQCOWoftrBcDt7Nwgo0kdpmrKxar/x2o6pV9JA= @@ -101,55 +101,55 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 h1:BEj3SPM81McUZHYjRS5pEgNgnmzGJ5tRpU5krWnV8Bs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0/go.mod h1:9cKLGBDzI/F3NoHLQGm4ZrYdIHsvGt6ej6hUowxY0J4= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= -go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= -go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -157,20 +157,20 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a h1:SGktgSolFCo75dnHJF2yMvnns6jCmHFJ0vE4Vn2JKvQ= -google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a/go.mod h1:a77HrdMjoeKbnd2jmgcWdaS++ZLZAEq3orIOAEIKiVw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= -google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= diff --git a/debian/changelog b/debian/changelog index b1a4e04bc3..4c3999413b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.154.0~rc1) stable; urgency=medium + + * New Synapse release 1.154.0rc1. + + -- Synapse Packaging team Wed, 27 May 2026 12:23:54 +0100 + matrix-synapse-py3 (1.153.0) stable; urgency=medium * New Synapse release 1.153.0. diff --git a/docs/structured_logging.md b/docs/structured_logging.md index 761d6466dd..c6916670aa 100644 --- a/docs/structured_logging.md +++ b/docs/structured_logging.md @@ -78,3 +78,44 @@ loggers: The above logging config will set Synapse as 'INFO' logging level by default, with the SQL layer at 'WARNING', and will log JSON formatted messages to a remote endpoint at 10.1.2.3:9999. + +## Google Cloud Logging (GKE) + +When running Synapse on GKE, use `synapse.logging.GcpJsonFormatter`. It outputs +JSON to stdout with a `severity` field that Google Cloud Logging maps to the +correct per-entry severity. Without this, GKE assigns `ERROR` to everything +written to stderr regardless of the actual Python log level. + +Example output: + +```json +{"severity":"INFO","message":"Processed request: 3.481sec 200 GET /sync","logger":"synapse.access.http.8008","time":"2026-05-12T13:40:37.829Z"} +``` + +Configuration: + +```yaml +version: 1 +disable_existing_loggers: false + +formatters: + gcp_json: + class: synapse.logging.GcpJsonFormatter + +handlers: + console: + class: logging.StreamHandler + formatter: gcp_json + stream: ext://sys.stdout + +loggers: + synapse.storage.SQL: + level: WARNING + twisted: + handlers: [console] + propagate: false + +root: + level: INFO + handlers: [console] +``` diff --git a/poetry.lock b/poetry.lock index 6093b13c02..4afbaad2c1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -752,18 +752,18 @@ test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"] [[package]] name = "idna" -version = "3.11" +version = "3.15" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, - {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, + {file = "idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8"}, + {file = "idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc"}, ] [package.extras] -all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] [[package]] name = "ijson" diff --git a/pyproject.toml b/pyproject.toml index e92e7b5c21..76ccbbe946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.153.0" +version = "1.154.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 2eb4dd1c14..b09e9694c5 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.153/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.154/synapse-config.schema.json type: object properties: modules: diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 236c8ca03c..8a19dba5ee 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1355,7 +1355,15 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): current_state = { state_key: event_map[event_id] for state_key, event_id in state_before_join.items() + # TODO figure out why events present in state_before_join are sometimes not found in event_map + # See https://github.com/element-hq/synapse/issues/19465 + if event_id in event_map } + if len(current_state) < len(state_before_join): + logger.warning( + "Some events from state_before_join were not found in event_map: %s", + set(state_before_join.values()) - set(event_map.keys()), + ) servers_that_can_issue_invite = get_servers_from_users( get_users_which_can_issue_invite(current_state) ) diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index e46e6dc2ef..d19815d6e2 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -251,8 +251,8 @@ class RoomPolicyHandler: # Note: if the policy server and event sender are the same server, the sender # might not have added policy server signatures to the event for whatever reason. # When this happens, we don't want to obliterate the event's existing signatures - # because the event will fail authorization. This is why we add defaults rather - # than simply `update` the signatures on the event. + # because the event will fail authorization. This is why we add items individually + # rather than simply `update` the signatures on the event. # # This situation can happen if the homeserver and policy server parts are # logically the same server, but run by different software. For example, Synapse @@ -261,7 +261,9 @@ class RoomPolicyHandler: # servers need to manually fetch signatures for. This is the code that allows # those events to continue working (because they're legally sent, even if missing # the policy server signature). - event.signatures.update(signature) + signatures = signature.get(policy_server.server_name, {}) + for key_id, sig in signatures.items(): + event.signatures.add_signature(policy_server.server_name, key_id, sig) except HttpResponseException as ex: # re-wrap HTTP errors as `SynapseError` so they can be proxied to clients directly raise ex.to_synapse_error() from ex diff --git a/synapse/logging/__init__.py b/synapse/logging/__init__.py index 15b92d7ef3..4058561e5e 100644 --- a/synapse/logging/__init__.py +++ b/synapse/logging/__init__.py @@ -22,10 +22,14 @@ import logging from synapse.logging._remote import RemoteHandler -from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter +from synapse.logging._terse_json import ( + GcpJsonFormatter, + JsonFormatter, + TerseJsonFormatter, +) # These are imported to allow for nicer logging configuration files. -__all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter"] +__all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter", "GcpJsonFormatter"] # Debug logger for https://github.com/matrix-org/synapse/issues/9533 etc issue9533_logger = logging.getLogger("synapse.9533_debug") diff --git a/synapse/logging/_terse_json.py b/synapse/logging/_terse_json.py index d9ff70b252..afa3288809 100644 --- a/synapse/logging/_terse_json.py +++ b/synapse/logging/_terse_json.py @@ -25,6 +25,7 @@ Log formatters that output terse JSON. import json import logging +from datetime import datetime, timezone _encoder = json.JSONEncoder(ensure_ascii=False, separators=(",", ":")) @@ -93,3 +94,31 @@ class TerseJsonFormatter(JsonFormatter): } return self._format(record, event) + + +class GcpJsonFormatter(logging.Formatter): + """JSON formatter compatible with Google Cloud Logging structured logging. + + Outputs `severity` (not `level`) so GCL correctly maps each log record to + the right severity instead of inheriting ERROR from stderr. + """ + + def format(self, record: logging.LogRecord) -> str: + msg = record.getMessage() + if record.exc_info: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + msg = f"{msg}\n{record.exc_text}" + + event = { + "severity": record.levelname, + "message": msg, + "logger": record.name, + "time": datetime.fromtimestamp(record.created, tz=timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + )[:-3] + + "Z", + } + + return _encoder.encode(event) diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 36dd39ed67..95364778cc 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -182,13 +182,10 @@ class ReplicationStreamer: self.command_handler.will_announce_positions() for stream in all_streams: - self.command_handler.send_command( - PositionCommand( - stream.NAME, - self._instance_name, - stream.last_token, - stream.last_token, - ) + self._send_position_command( + stream_name=stream.NAME, + prev_token=stream.last_token, + new_token=stream.last_token, ) for stream in all_streams: @@ -205,9 +202,9 @@ class ReplicationStreamer: last_token = stream.last_token logger.debug( - "Getting stream: %s: %s -> %s", + "Getting stream updates for %s: %s -> %s", stream.NAME, - stream.last_token, + last_token, stream.current_token(self._instance_name), ) try: @@ -217,26 +214,7 @@ class ReplicationStreamer: logger.info("Failed to handle stream %s", stream.NAME) raise - logger.debug( - "Sending %d updates", - len(updates), - ) - - if updates: - logger.info( - "Streaming: %s -> %s (limited: %s, updates: %s, max token: %s)", - stream.NAME, - updates[-1][0], - limited, - len(updates), - current_token, - ) - stream_updates_counter.labels( - stream_name=stream.NAME, - **{SERVER_NAME_LABEL: self.server_name}, - ).inc(len(updates)) - - else: + if not updates: # The token has advanced but there is no data to # send, so we send a `POSITION` to inform other # workers of the updated position. @@ -266,21 +244,28 @@ class ReplicationStreamer: # POSITION with last token of X+1, which will # cause them to check if there were any missing # updates between X and X+1. - logger.info( - "Sending position: %s -> %s", - stream.NAME, - current_token, - ) - self.command_handler.send_command( - PositionCommand( - stream.NAME, - self._instance_name, - last_token, - current_token, - ) + self._send_position_command( + stream_name=stream.NAME, + prev_token=last_token, + new_token=current_token, ) continue + logger.info( + "Sending update for %s: %s -> %s (limited: %s, updates: %s, max token: %s)", + stream.NAME, + last_token, + updates[-1][0], + limited, + len(updates), + current_token, + ) + + stream_updates_counter.labels( + stream_name=stream.NAME, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(len(updates)) + # Some streams return multiple rows with the same stream IDs, # we need to make sure they get sent out in batches. We do # this by setting the current token to all but the last of @@ -300,18 +285,10 @@ class ReplicationStreamer: # token, in which case we want to send out a `POSITION` # to tell other workers the actual current position. if updates[-1][0] < current_token: - logger.info( - "Sending position: %s -> %s", - stream.NAME, - current_token, - ) - self.command_handler.send_command( - PositionCommand( - stream.NAME, - self._instance_name, - updates[-1][0], - current_token, - ) + self._send_position_command( + stream_name=stream.NAME, + prev_token=updates[-1][0], + new_token=current_token, ) logger.debug("No more pending updates, breaking poke loop") @@ -319,6 +296,25 @@ class ReplicationStreamer: self.pending_updates = False self.is_looping = False + def _send_position_command( + self, *, stream_name: str, prev_token: int, new_token: int + ) -> None: + """Send a POSITION command over replication""" + logger.info( + "Sending position for %s: %s -> %s", + stream_name, + prev_token, + new_token, + ) + self.command_handler.send_command( + PositionCommand( + stream_name, + self._instance_name, + prev_token, + new_token, + ) + ) + def _batch_updates( updates: list[tuple[Token, StreamRow]], diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index fc61f46c1c..a3806fd112 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -897,10 +897,20 @@ class DeviceInboxWorkerStore(SQLBaseStore): ) -> None: assert self._can_write_to_device - local_by_user_then_device = {} + # A map from user id, to device id, to a pair of (serialized message, msgid). + local_by_user_then_device: dict[str, dict[str, tuple[str, str]]] = {} + for user_id, messages_by_device in messages_by_user_then_device.items(): - messages_json_for_user = {} + # Mesages to send to this specific user. A map + # from device id, to a pair of (serialized message, msgid). + messages_json_for_user: dict[str, tuple[str, str]] = {} + devices = list(messages_by_device.keys()) + if not devices: + # No to-device messages for this user. (For example, someone has + # hit `/sendToDevice` with an empty {device: message} dict.) + continue + if len(devices) == 1 and devices[0] == "*": # Handle wildcard device_ids. # We exclude hidden devices (such as cross-signing keys) here as they are @@ -912,15 +922,28 @@ class DeviceInboxWorkerStore(SQLBaseStore): retcol="device_id", ) - message_json = json_encoder.encode(messages_by_device["*"]) + # Don't bother to serialize if there are no devices for this user + if not devices: + if issue9533_logger.isEnabledFor(logging.DEBUG): + msgid = _get_msgid_for_message(messages_by_device["*"]) + issue9533_logger.debug( + "Dropping wildcard to-device message for user %s with no devices (msgid %s)", + user_id, + msgid, + ) + continue + + message_json, msgid = _serialize_to_device_message( + user_id=user_id, device_id="*", msg=messages_by_device["*"] + ) for device_id in devices: # Add the message for all devices for this user on this # server. - messages_json_for_user[device_id] = message_json + messages_json_for_user[device_id] = (message_json, msgid) else: - if not devices: - continue - + # Query the database to determine which of the target devices actually + # exist. + # # We exclude hidden devices (such as cross-signing keys) here as they are # not expected to receive to-device messages. rows = cast( @@ -938,19 +961,25 @@ class DeviceInboxWorkerStore(SQLBaseStore): for (device_id,) in rows: # Only insert into the local inbox if the device exists on # this server - with start_active_span("serialise_to_device_message"): - msg = messages_by_device[device_id] - set_tag(SynapseTags.TO_DEVICE_TYPE, msg["type"]) - set_tag(SynapseTags.TO_DEVICE_SENDER, msg["sender"]) - set_tag(SynapseTags.TO_DEVICE_RECIPIENT, user_id) - set_tag(SynapseTags.TO_DEVICE_RECIPIENT_DEVICE, device_id) - set_tag( - SynapseTags.TO_DEVICE_MSGID, - msg["content"].get(EventContentFields.TO_DEVICE_MSGID), - ) - message_json = json_encoder.encode(msg) + msg = messages_by_device[device_id] + message_json, msgid = _serialize_to_device_message( + user_id=user_id, device_id=device_id, msg=msg + ) + messages_json_for_user[device_id] = (message_json, msgid) - messages_json_for_user[device_id] = message_json + if issue9533_logger.isEnabledFor(logging.DEBUG): + # Log any messages we are dropping + unmapped_devices = ( + messages_by_device.keys() - messages_json_for_user.keys() + ) + if unmapped_devices: + issue9533_logger.debug( + "Dropping to-device messages for unknown devices: %s", + [ + f"{user_id}/{device_id} (msgid {_get_msgid_for_message(messages_by_device[device_id])})" + for device_id in unmapped_devices + ], + ) if messages_json_for_user: local_by_user_then_device[user_id] = messages_json_for_user @@ -965,22 +994,21 @@ class DeviceInboxWorkerStore(SQLBaseStore): values=[ (user_id, device_id, stream_id, message_json, self._instance_name) for user_id, messages_by_device in local_by_user_then_device.items() - for device_id, message_json in messages_by_device.items() + for device_id, (message_json, _msgid) in messages_by_device.items() ], ) if issue9533_logger.isEnabledFor(logging.DEBUG): issue9533_logger.debug( - "Stored to-device messages with stream_id %i: %s", + "Storing to-device messages with stream_id %i: %s", stream_id, [ - f"{user_id}/{device_id} (msgid " - f"{msg['content'].get(EventContentFields.TO_DEVICE_MSGID)})" + f"{user_id}/{device_id} (msgid {msgid})" for ( user_id, messages_by_device, - ) in messages_by_user_then_device.items() - for (device_id, msg) in messages_by_device.items() + ) in local_by_user_then_device.items() + for (device_id, (_msg, msgid)) in messages_by_device.items() ], ) @@ -1066,6 +1094,29 @@ class DeviceInboxWorkerStore(SQLBaseStore): return results +def _serialize_to_device_message( + *, user_id: str, device_id: str, msg: JsonDict +) -> tuple[str, str]: + """Serialiize a to-device message, ready to add to the device_inbox table. + + Returns a tuple (message_json, msgid). + """ + with start_active_span("serialise_to_device_message"): + msgid = _get_msgid_for_message(msg) + set_tag(SynapseTags.TO_DEVICE_TYPE, msg["type"]) + set_tag(SynapseTags.TO_DEVICE_SENDER, msg["sender"]) + set_tag(SynapseTags.TO_DEVICE_RECIPIENT, user_id) + set_tag(SynapseTags.TO_DEVICE_RECIPIENT_DEVICE, device_id) + set_tag(SynapseTags.TO_DEVICE_MSGID, msgid) + message_json = json_encoder.encode(msg) + return message_json, msgid + + +def _get_msgid_for_message(msg: JsonDict) -> str: + """Extract the message ID from a to-device message.""" + return str(msg["content"].get(EventContentFields.TO_DEVICE_MSGID, "")) + + class DeviceInboxBackgroundUpdateStore(SQLBaseStore): DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop" REMOVE_DEAD_DEVICES_FROM_INBOX = "remove_dead_devices_from_device_inbox" diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index a1afe6d2ae..1e053be6af 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -38,6 +38,7 @@ from typing import ( import attr from sortedcontainers import SortedList, SortedSet +from synapse.logging import issue9533_logger from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.database import ( DatabasePool, @@ -774,6 +775,14 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): # We move the current min position up if the minimum current positions # of all instances is higher (since by definition all positions less # that that have been persisted). + # + # If we are one of several writers, then we don't need to factor our own + # `_current_position` into `_persisted_upto_position` unless we have unfinished + # writes (since we know that any future write that happens locally will have + # a higher stream ID than any of the other writers' current positions). In other + # words, when we have no outstanding writes, then the new `_persisted_upto_position` + # can be the minimum of all *other* writers' current positions, + # our_current_position = self._current_positions.get(self._instance_name, 0) min_curr = min( ( @@ -783,7 +792,6 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): ), default=our_current_position, ) - if our_current_position and (self._unfinished_ids or self._in_flight_fetches): min_curr = min(min_curr, our_current_position) @@ -820,6 +828,22 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): # do. break + # Hacky debug logging to attempt to trace https://github.com/element-hq/synapse/issues/19795 + if ( + issue9533_logger.isEnabledFor(logging.DEBUG) + and self._stream_name == "to_device" + ): + issue9533_logger.debug( + "stream_id=%i now persisted for stream=%s; _current_positions=%s _unfinished_ids=%s, _known_persisted_positions=%s _persisted_upto_position=%i min_curr=%i", + new_id, + self._stream_name, + self._current_positions, + self._unfinished_ids, + self._known_persisted_positions, + self._persisted_upto_position, + min_curr, + ) + def _update_stream_positions_table_txn(self, txn: Cursor) -> None: """Update the `stream_positions` table with newly persisted position.""" diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index 2d3e917aef..4f2188b8e7 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -27,6 +27,7 @@ from synapse.handlers.room_policy import POLICY_SERVER_KEY_ID from synapse.rest import admin from synapse.rest.client import filter, login, room, sync from synapse.server import HomeServer +from synapse.synapse_rust.events import Signatures from synapse.types import JsonDict, UserID from synapse.util.clock import Clock @@ -113,7 +114,15 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): self.OTHER_SERVER_NAME, self.signing_key, ) - return sigs + # Only return the new signature like the policy server spec says, + # not any others that were already in the event + return { + self.OTHER_SERVER_NAME: { + POLICY_SERVER_KEY_ID: sigs[self.OTHER_SERVER_NAME][ + POLICY_SERVER_KEY_ID + ] + } + } async def policy_server_signs_event_with_wrong_key( destination: str, pdu: EventBase, timeout: int | None = None @@ -169,6 +178,19 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): state_key="", ) + def _sign_with_random_key(self, server_name: str, event: EventBase) -> None: + non_policyserver_key = signedjson.key.generate_signing_key( + "non_policyserver_key" + ) + event.signatures = Signatures( + compute_event_signature( + event.room_version, + event.get_dict(), + server_name, + non_policyserver_key, + ) + ) + def test_no_policy_event_set(self) -> None: # We don't need to modify the room state at all - we're testing the default # case where a room doesn't use a policy server. @@ -316,11 +338,69 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): }, }, ) + # Sign the event as the origin server first, since that's what events passed to + # ask_policy_server_to_sign_event will generally look like. The exact key used + # here isn't important. + self._sign_with_random_key("example.org", event) self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event self.get_success( self.handler.ask_policy_server_to_sign_event(event, verify=True) ) - self.assertEqual(len(event.signatures), 1) + # Standard success case: event has signatures from the origin and the policy server + self.assertEqual( + { + server: len(signatures) + for server, signatures in event.signatures.as_dict().items() + }, + {"example.org": 1, self.OTHER_SERVER_NAME: 1}, + f"Expected signatures for the origin homeserver (example.org) and policy server ({self.OTHER_SERVER_NAME})", + ) + + def test_ask_origin_server_to_sign_event_doesnt_replace_signatures(self) -> None: + """ + ``ask_policy_server_to_sign_event`` has had bugs where it accidentally overwrote + the origin server's signature in the case where the origin server has the same + server name as the policy server (each have their own signing key). This test is + otherwise equivalent to the success case test above, but the server name for + origin event sending server and the policy server are the same and we want to + ensure both signatures are preserved. + """ + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + event = make_test_event( + room_version=self.room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:" + self.OTHER_SERVER_NAME, + "content": { + "msgtype": "m.text", + "body": "This is another signed event.", + }, + }, + ) + # Sign the event as the origin server that sent the event, which in this case + # has the same server name as the policy server. We're using a different key + # than `self.signing_key` (for the policy server), as the ed25519:policy_server + # key is only used for policy server signatures, not any other federation traffic + # even when the origin server and policy are logically the same server. + self._sign_with_random_key(self.OTHER_SERVER_NAME, event) + self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event + self.get_success( + self.handler.ask_policy_server_to_sign_event(event, verify=True) + ) + # Less common success case: the event origin server is logically the same as + # the policy server, so there will be two signatures from one server name. + # It's important to make sure both signatures are preserved. + self.assertEqual( + { + server: len(signatures) + for server, signatures in event.signatures.as_dict().items() + }, + {self.OTHER_SERVER_NAME: 2}, + f"Expected 2 signatures for the origin server and policy server under the same server name ({self.OTHER_SERVER_NAME}) but with different keys", + ) def test_ask_policy_server_to_sign_event_refuses(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) diff --git a/tests/logging/test_terse_json.py b/tests/logging/test_terse_json.py index a857737ddf..433408fee6 100644 --- a/tests/logging/test_terse_json.py +++ b/tests/logging/test_terse_json.py @@ -28,7 +28,11 @@ from twisted.web.http import HTTPChannel from twisted.web.server import Request from synapse.http.site import SynapseRequest -from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter +from synapse.logging._terse_json import ( + GcpJsonFormatter, + JsonFormatter, + TerseJsonFormatter, +) from synapse.logging.context import LoggingContext, LoggingContextFilter from synapse.types import JsonDict @@ -251,3 +255,77 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): self.assertEqual(log["log"], "Hello there, wally!") self.assertEqual(log["exc_type"], "ValueError") self.assertEqual(log["exc_value"], "That's wrong, you wally!") + + +class GcpJsonFormatterTestCase(LoggerCleanupMixin, TestCase): + def setUp(self) -> None: + self.output = StringIO() + + def get_log_line(self) -> JsonDict: + data = self.output.getvalue() + logs = data.splitlines() + self.assertEqual(len(logs), 1) + self.assertEqual(data.count("\n"), 1) + return json.loads(logs[0]) + + def test_gcp_json_output(self) -> None: + """ + GcpJsonFormatter produces exactly the four fields GCL expects. + """ + handler = logging.StreamHandler(self.output) + handler.setFormatter(GcpJsonFormatter()) + logger = self.get_logger(handler) + + logger.info("Hello there, %s!", "wally") + + log = self.get_log_line() + + self.assertIncludes( + log.keys(), {"severity", "message", "logger", "time"}, exact=True + ) + self.assertEqual(log["message"], "Hello there, wally!") + self.assertEqual(log["severity"], "INFO") + self.assertTrue(log["time"].endswith("Z")) + + def test_severity_levels(self) -> None: + """ + Python log levels are mapped to their GCL severity equivalents. + """ + cases = [ + (logging.DEBUG, "DEBUG"), + (logging.INFO, "INFO"), + (logging.WARNING, "WARNING"), + (logging.ERROR, "ERROR"), + (logging.CRITICAL, "CRITICAL"), + ] + for level, expected_severity in cases: + self.output = StringIO() + handler = logging.StreamHandler(self.output) + handler.setFormatter(GcpJsonFormatter()) + logger = self.get_logger(handler) + logger.setLevel(level) + logger.log(level, "test") + log = self.get_log_line() + self.assertEqual(log["severity"], expected_severity, f"level={level}") + + def test_gcp_json_with_exception(self) -> None: + """ + Exception info is appended to the message field, not separate keys. + """ + handler = logging.StreamHandler(self.output) + handler.setFormatter(GcpJsonFormatter()) + logger = self.get_logger(handler) + + try: + raise ValueError("That's wrong, you wally!") + except ValueError: + logger.exception("Hello there, %s!", "wally") + + log = self.get_log_line() + + self.assertIncludes( + log.keys(), {"severity", "message", "logger", "time"}, exact=True + ) + self.assertIn("Hello there, wally!", log["message"]) + self.assertIn("ValueError", log["message"]) + self.assertIn("That's wrong, you wally!", log["message"])