From c0c2b37d5e21dfce3c07fc1f8259518b549ace34 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 10 Jul 2026 21:23:33 -0400 Subject: [PATCH 001/158] MSC4140: update error responses (#19539) - Impose limit of scheduled delayed events - Update error codes to match latest draft of MSC4140 --------- Co-authored-by: Eric Eastwood --- changelog.d/19539.bugfix | 1 + changelog.d/19539.feature | 1 + rust/src/config/mod.rs | 2 +- rust/src/handlers/versions.rs | 5 +- synapse/config/server.py | 33 ++- synapse/handlers/delayed_events.py | 27 ++- synapse/rest/client/capabilities.py | 5 + synapse/rest/client/room.py | 42 +--- .../storage/databases/main/delayed_events.py | 73 +++++- .../main/delta/88/01_add_delayed_events.sql | 2 + tests/config/test_server.py | 52 +++++ tests/rest/client/test_capabilities.py | 38 +++ tests/rest/client/test_rooms.py | 218 +++++++++++++++++- 13 files changed, 434 insertions(+), 65 deletions(-) create mode 100644 changelog.d/19539.bugfix create mode 100644 changelog.d/19539.feature diff --git a/changelog.d/19539.bugfix b/changelog.d/19539.bugfix new file mode 100644 index 0000000000..41beb3a179 --- /dev/null +++ b/changelog.d/19539.bugfix @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Update error responses to match their format in the current draft of the MSC. diff --git a/changelog.d/19539.feature b/changelog.d/19539.feature new file mode 100644 index 0000000000..93eb4cc1b9 --- /dev/null +++ b/changelog.d/19539.feature @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Limit how many delayed events a user may have scheduled at once. diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index ef9ecb067c..b5f641613b 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -47,7 +47,7 @@ pub struct AuthConfig { } #[derive(FromPyObject, Clone)] pub struct ServerConfig { - pub max_event_delay_ms: Option, + pub msc4140_enabled: bool, } #[derive(FromPyObject, Clone)] diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index db8bca82d6..bfe064eb5e 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -304,10 +304,7 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4028: config.experimental.msc4028_push_encrypted_events, msc4108: config.experimental.msc4108_enabled || (config.experimental.msc4108_delegation_endpoint.is_some()), - msc4140: config - .server - .max_event_delay_ms - .is_some_and(|max_event_delay_ms| max_event_delay_ms > 0), + msc4140: config.server.msc4140_enabled, msc3575: config.experimental.msc3575_enabled, msc4133: config.experimental.msc4133_enabled, msc4133_stable: true, diff --git a/synapse/config/server.py b/synapse/config/server.py index 282679aabe..695a6cb9d4 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -37,6 +37,7 @@ from twisted.conch.ssh.keys import Key from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.types import JsonDict, StrSequence +from synapse.util.duration import Duration from synapse.util.module_loader import load_module from synapse.util.stringutils import parse_and_validate_server_name @@ -949,13 +950,33 @@ class ServerConfig(Config): # The maximum allowed delay duration for delayed events (MSC4140). max_event_delay_duration = config.get("max_event_delay_duration") if max_event_delay_duration is not None: - self.max_event_delay_ms: int | None = self.parse_duration( - max_event_delay_duration - ) - if self.max_event_delay_ms <= 0: - raise ConfigError("max_event_delay_duration must be a positive value") + max_event_delay_ms = self.parse_duration(max_event_delay_duration) + if max_event_delay_ms <= 0: + raise ConfigError( + "'max_event_delay_duration' must be a positive value if set", + ("max_event_delay_duration",), + ) + self.max_event_delay_duration = Duration(milliseconds=max_event_delay_ms) else: - self.max_event_delay_ms = None + self.max_event_delay_duration = Duration() + + # The maximum number of delayed events a user may have scheduled at a time. + # (Defined here despite being experimental to be near the other MSC4140 config) + self.max_delayed_events_per_user: int = config.get( + "experimental_features", {} + ).get("msc4140_max_delayed_events_per_user", 100) + if ( + not isinstance(self.max_delayed_events_per_user, int) + or self.max_delayed_events_per_user < 0 + ): + raise ConfigError( + "'msc4140_max_delayed_events_per_user' must be a non-negative integer", + ("experimental", "msc4140_max_delayed_events_per_user"), + ) + + self.msc4140_enabled = bool( + self.max_delayed_events_per_user and self.max_event_delay_duration + ) def has_tls_listener(self) -> bool: return any(listener.is_tls() for listener in self.listeners) diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index f016d95e31..13d6a54de2 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -13,12 +13,13 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING, Optional from twisted.internet.interfaces import IDelayedCall from synapse.api.constants import EventTypes, StickyEvent, StickyEventField -from synapse.api.errors import ShadowBanError, SynapseError +from synapse.api.errors import Codes, ShadowBanError, SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.config.workers import MAIN_PROCESS_INSTANCE_NAME from synapse.http.site import SynapseRequest @@ -330,7 +331,7 @@ class DelayedEventsHandler: state_key: str | None, origin_server_ts: int | None, content: JsonDict, - delay: int, + delay: Duration, sticky_duration_ms: int | None, ) -> str: """ @@ -344,20 +345,37 @@ class DelayedEventsHandler: origin_server_ts: The custom timestamp to send the event with. If None, the timestamp will be the actual time when the event is sent. content: The content of the event to be sent. - delay: How long (in milliseconds) to wait before automatically sending the event. + delay: How long to wait before automatically sending the event. sticky_duration_ms: If an MSC4354 sticky event: the sticky duration (in milliseconds). The event will be attempted to be reliably delivered to clients and remote servers during its sticky period. Returns: The ID of the added delayed event. Raises: - SynapseError: if the delayed event fails validation checks. + SynapseError: if the delayed event fails validation checks, or + if the requested delay is longer than allowed, or + if sending delayed events has been disallowed entirely. """ # Use standard request limiter for scheduling new delayed events. # TODO: Instead apply ratelimiting based on the scheduled send time. # See https://github.com/element-hq/synapse/issues/18021 await self._request_ratelimiter.ratelimit(requester) + if not self._config.server.msc4140_enabled: + raise SynapseError( + HTTPStatus.FORBIDDEN, + "Sending delayed events has been disallowed", + Codes.FORBIDDEN, + ) + if delay > self._config.server.max_event_delay_duration: + requested_delay = delay.as_millis() + max_delay = self._config.server.max_event_delay_duration.as_millis() + raise SynapseError( + HTTPStatus.FORBIDDEN, + f"The requested delay ({requested_delay}ms) exceeds the allowed maximum ({max_delay}ms)", + Codes.FORBIDDEN, + ) + self._event_creation_handler.validator.validate_builder( self._event_creation_handler.event_builder_factory.for_room_version( await self._store.get_room_version(room_id), @@ -384,6 +402,7 @@ class DelayedEventsHandler: content=content, delay=delay, sticky_duration_ms=sticky_duration_ms, + limit=self._config.server.max_delayed_events_per_user, ) if self._repl_client is not None: diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 2be5f5849d..4ddaaeda74 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -109,6 +109,11 @@ class CapabilitiesRestServlet(RestServlet): "capabilities" ]["m.profile_fields"] + response["capabilities"]["org.matrix.msc4140.delayed_events"] = { + "max_delay_ms": self.config.server.max_event_delay_duration.as_millis(), + "max_scheduled": self.config.server.max_delayed_events_per_user, + } + if self.config.experimental.msc4267_enabled: response["capabilities"]["org.matrix.msc4267.forget_forced_upon_leave"] = { "enabled": self.config.room.forget_on_leave, diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 36f638e236..951b4ff08b 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -84,6 +84,7 @@ from synapse.types import JsonDict, Requester, StreamToken, ThirdPartyInstanceID from synapse.types.state import StateFilter from synapse.util.cancellation import cancellable from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.events import generate_fake_event_id from synapse.util.stringutils import parse_and_validate_server_name @@ -215,7 +216,6 @@ class RoomStateEventRestServlet(RestServlet): self.auth = hs.get_auth() self.clock = hs.get_clock() self._event_serializer = hs.get_event_client_serializer() - self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self._msc4354_enabled = hs.config.experimental.msc4354_enabled @@ -343,7 +343,7 @@ class RoomStateEventRestServlet(RestServlet): if self._msc4354_enabled: sticky_duration_ms = parse_integer(request, StickyEvent.QUERY_PARAM_NAME) - delay = _parse_request_delay(request, self._max_event_delay_ms) + delay = _parse_request_for_delayed_event_delay(request) if delay is not None: delay_id = await self.delayed_events_handler.add( requester, @@ -416,7 +416,6 @@ class RoomSendEventRestServlet(TransactionRestServlet): self.event_creation_handler = hs.get_event_creation_handler() self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() - self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._msc4354_enabled = hs.config.experimental.msc4354_enabled def register(self, http_server: HttpServer) -> None: @@ -442,7 +441,7 @@ class RoomSendEventRestServlet(TransactionRestServlet): if self._msc4354_enabled: sticky_duration_ms = parse_integer(request, StickyEvent.QUERY_PARAM_NAME) - delay = _parse_request_delay(request, self._max_event_delay_ms) + delay = _parse_request_for_delayed_event_delay(request) if delay is not None: delay_id = await self.delayed_events_handler.add( requester, @@ -515,47 +514,20 @@ class RoomSendEventRestServlet(TransactionRestServlet): ) -def _parse_request_delay( - request: SynapseRequest, - max_delay: int | None, -) -> int | None: +def _parse_request_for_delayed_event_delay(request: SynapseRequest) -> Duration | None: """Parses from the request string the delay parameter for delayed event requests, and checks it for correctness. Args: request: the twisted HTTP request. - max_delay: the maximum allowed value of the delay parameter, - or None if no delay parameter is allowed. Returns: The value of the requested delay, or None if it was absent. Raises: - SynapseError: if the delay parameter is present and forbidden, - or if it exceeds the maximum allowed value. + SynapseError: if the delay parameter is present and invalid. """ - delay = parse_integer(request, "org.matrix.msc4140.delay") - if delay is None: - return None - if max_delay is None: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Delayed events are not supported on this server", - Codes.UNKNOWN, - { - "org.matrix.msc4140.errcode": "M_MAX_DELAY_UNSUPPORTED", - }, - ) - if delay > max_delay: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "The requested delay exceeds the allowed maximum.", - Codes.UNKNOWN, - { - "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", - "org.matrix.msc4140.max_delay": max_delay, - }, - ) - return delay + delay_ms = parse_integer(request, "org.matrix.msc4140.delay") + return Duration(milliseconds=delay_ms) if delay_ms is not None else None # TODO: Needs unit testing for room ID + alias joins diff --git a/synapse/storage/databases/main/delayed_events.py b/synapse/storage/databases/main/delayed_events.py index 1727f589e2..bb512611e4 100644 --- a/synapse/storage/databases/main/delayed_events.py +++ b/synapse/storage/databases/main/delayed_events.py @@ -17,7 +17,7 @@ from typing import TYPE_CHECKING, NewType import attr -from synapse.api.errors import NotFoundError +from synapse.api.errors import LimitExceededError, NotFoundError from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage.database import ( DatabasePool, @@ -28,6 +28,7 @@ from synapse.storage.database import ( from synapse.storage.engines import PostgresEngine from synapse.types import JsonDict, RoomID from synapse.util import stringutils +from synapse.util.duration import Duration from synapse.util.json import json_encoder if TYPE_CHECKING: @@ -122,20 +123,84 @@ class DelayedEventsStore(SQLBaseStore): state_key: str | None, origin_server_ts: int | None, content: JsonDict, - delay: int, + delay: Duration, sticky_duration_ms: int | None, + limit: int, ) -> tuple[DelayID, Timestamp]: """ Inserts a new delayed event in the DB. + Args: + user_localpart: The localpart of the requester of the delayed event, who will be its owner. + device_id: The device ID of the requester. + creation_ts: The timestamp of when the request to add the delayed event was made. + room_id: The ID of the room where the event should be sent to. + event_type: The type of event to be sent. + state_key: The state key of the event to be sent, or None if it is not a state event. + origin_server_ts: The custom timestamp to send the event with. + If None, the timestamp will be the actual time when the event is sent. + content: The content of the event to be sent. + delay: How long to wait before automatically sending the event. + sticky_duration_ms: If an MSC4354 sticky event: the sticky duration (in milliseconds). + The event will be attempted to be reliably delivered to clients and remote servers + during its sticky period. + limit: The maximum number of delayed events the DB may store for the given requester. + Must be greater than 0. Returns: The generated ID assigned to the added delayed event, and the send time of the next delayed event to be sent, which is either the event just added or one added earlier. + + Raises: + LimitExceededError: if the DB has reached the limit of + how many delayed events it may store for the given requester. + AssertionError: if the limit is not greater than 0. """ + assert limit > 0, "limit must be greater than 0" + delay_id = _generate_delay_id() - send_ts = Timestamp(creation_ts + delay) + delay_ms = delay.as_millis() + send_ts = creation_ts + delay_ms def add_delayed_event_txn(txn: LoggingTransaction) -> Timestamp: + num_existing: int = self.db_pool.simple_select_one_onecol_txn( + txn, + table="delayed_events", + keyvalues={"user_localpart": user_localpart}, + retcol="COUNT(*)", + ) + if num_existing >= limit: + # Find the send_ts threshold that will bring the queue back under the limit. + # When the amount of existing delayed events has reached the limit, + # this will be the send time of the next delayed event to be sent. + # When the amount has exceeded the limit (e.g., due to config changes), + # this will be the send time of the delayed event that will be sent + # once all earlier events that exceed the limit have been sent. + # + # FIXME: Remove "AS subquery" after dropping support for PostgreSQL <16 + txn.execute( + """ + SELECT MAX(send_ts) FROM ( + SELECT * FROM delayed_events + WHERE user_localpart = ? + ORDER BY send_ts ASC + LIMIT ? + ) AS subquery + """, + ( + user_localpart, + num_existing - limit + 1, + ), + ) + row = txn.fetchone() + assert row + retry_after_ms = row[0] - self.clock.time_msec() + err = LimitExceededError( + limiter_name="add_delayed_event", + retry_after_ms=retry_after_ms if retry_after_ms > 0 else None, + ) + err.msg = "The maximum number of delayed events has been reached." + raise err + self.db_pool.simple_insert_txn( txn, table="delayed_events", @@ -143,7 +208,7 @@ class DelayedEventsStore(SQLBaseStore): "delay_id": delay_id, "user_localpart": user_localpart, "device_id": device_id, - "delay": delay, + "delay": delay_ms, "send_ts": send_ts, "room_id": room_id, "event_type": event_type, diff --git a/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql b/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql index 78ba5129af..4abe0ccaf4 100644 --- a/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql +++ b/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql @@ -22,6 +22,8 @@ CREATE TABLE delayed_events ( state_key TEXT, origin_server_ts BIGINT, content bytea NOT NULL, + -- is_processed = TRUE means that the work of sending the delayed event has begun. + -- Once the send is complete, the delayed event is removed from this table. is_processed BOOLEAN NOT NULL DEFAULT FALSE, PRIMARY KEY (user_localpart, delay_id) ); diff --git a/tests/config/test_server.py b/tests/config/test_server.py index d3c59ae14c..41ea8fb5b1 100644 --- a/tests/config/test_server.py +++ b/tests/config/test_server.py @@ -18,11 +18,15 @@ # # + +from typing import Any + import yaml from synapse.config._base import ConfigError, RootConfig from synapse.config.homeserver import HomeServerConfig from synapse.config.server import ServerConfig, generate_ip_set, is_threepid_reserved +from synapse.types import JsonDict from tests import unittest @@ -189,6 +193,54 @@ class ServerConfigTestCase(unittest.TestCase): self.assertEqual(conf["listeners"], expected_listeners) + def test_max_delayed_events_enforces_positive(self) -> None: + """ + Test that the configured maximum allowed delay must be a positive value if set, + as per documentation + """ + + def generate_config(value: int) -> JsonDict: + return {"max_event_delay_duration": value} + + _read_config(generate_config(1)) + + with self.assertRaises(ConfigError): + _read_config(generate_config(0)) + + with self.assertRaises(ConfigError): + _read_config(generate_config(-1)) + + def test_max_delayed_events_per_user_enforces_non_negative_int(self) -> None: + """ + Test that the configured maximum number of delayed events must be a non-negative value if set, + as a negative limit can never be satisfied + """ + + def generate_config(value: Any) -> JsonDict: + return { + "experimental_features": {"msc4140_max_delayed_events_per_user": value} + } + + for allowed_value in (0, 1): + _read_config(generate_config(allowed_value)) + + for disallowed_value in (-1, 0.5): + with self.assertRaises(ConfigError): + _read_config(generate_config(disallowed_value)) + + +def _read_config(config_values: JsonDict) -> None: + ServerConfig(RootConfig()).read_config( + yaml.safe_load( + HomeServerConfig().generate_config( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + ) + ) + | config_values + ) + class GenerateIpSetTestCase(unittest.TestCase): def test_empty(self) -> None: diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index c28e0605b5..42926c4359 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -26,6 +26,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.rest.client import capabilities, login from synapse.server import HomeServer from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.unittest import override_config, skip_unless @@ -203,6 +204,43 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): ["avatar_url"], ) + def test_get_delayed_events_capabilities_default_config_msc4140(self) -> None: + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_delay_ms"], 0 + ) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_scheduled"], 100 + ) + + @override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 50, + }, + } + ) + def test_get_delayed_events_capabilities_custom_config_msc4140(self) -> None: + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_delay_ms"], + Duration(days=1).as_millis(), + ) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_scheduled"], 50 + ) + @override_config({"enable_3pid_changes": False}) def test_get_change_3pid_capabilities_3pid_disabled(self) -> None: """Test if change 3pid is disabled that the server responds it.""" diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 8d4892ae91..90063ccb06 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -61,6 +61,7 @@ from synapse.rest.client import ( from synapse.server import HomeServer from synapse.types import JsonDict, JsonMapping, RoomAlias, UserID, create_requester from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.stringutils import random_string from tests import unittest @@ -2503,7 +2504,12 @@ class RoomDelayedEventTestCase(RoomBase): {}, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) - self.assertNotIn("org.matrix.msc4140.errcode", channel.json_body) + # Assert that the standard error response uses a valid errcode. + # The specific errcode is irrelevant for the purpose of this test. + self.assertIsInstance( + channel.json_body.get("errcode"), + str, + ) def test_delayed_event_unsupported_by_default(self) -> None: """Test that sending a delayed event is unsupported with the default config.""" @@ -2515,10 +2521,35 @@ class RoomDelayedEventTestCase(RoomBase): ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) self.assertEqual( - "M_MAX_DELAY_UNSUPPORTED", - channel.json_body.get("org.matrix.msc4140.errcode"), + Codes.FORBIDDEN, + channel.json_body.get("errcode"), + channel.json_body, + ) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 0, + }, + } + ) + def test_delayed_event_disabled_by_limit(self) -> None: + """Test that delayed events are disabled by configuring the per-user limit to 0.""" + channel = self.make_request( + "PUT", + ( + "rooms/%s/send/m.room.message/mid1?org.matrix.msc4140.delay=2000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) + self.assertEqual( + Codes.FORBIDDEN, + channel.json_body.get("errcode"), channel.json_body, ) @@ -2533,13 +2564,178 @@ class RoomDelayedEventTestCase(RoomBase): ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) self.assertEqual( - "M_MAX_DELAY_EXCEEDED", - channel.json_body.get("org.matrix.msc4140.errcode"), + Codes.FORBIDDEN, + channel.json_body.get("errcode"), channel.json_body, ) + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 1, + }, + } + ) + def test_delayed_event_user_limit_reached(self) -> None: + """Test that users cannot have more delayed events scheduled at once than allowed.""" + # Disable rate-limits for this user. We want to specifically test the storage-based limit, not the request limits + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=15000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + # Send a delayed event to eat up the limit + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Try to send another delayed event (we expect to hit the limit on the max number of delayed events that can be scheduled at once) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + # Confirm that the response includes the time remaining until the next of the user's + # delayed events to be sent, at which point another delayed event may be scheduled + # without exceeding the limit + retry_after_headers = channel.headers.getRawHeaders("Retry-After") + assert retry_after_headers + retry_after_sec = int(retry_after_headers[0]) + self.assertGreater(retry_after_sec, 0) + # Confirm that there is only a single value to the Retry-After header, as per RFC9110 + self.assertEqual(1, len(retry_after_headers)) + + # Wait until we're able to retry again (the retry time from the error response) + self.reactor.advance(retry_after_sec) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 1, + }, + } + ) + def test_delayed_event_processed_user_limit_reached(self) -> None: + """ + Test that delayed events in the midst of being sent still count towards the limit of + how many delayed events a user may have scheduled at once. + """ + send_after = Duration(seconds=1) + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + f"rooms/%s/send/m.room.message?org.matrix.msc4140.delay={send_after.as_millis()}" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Simulate the server taking a long time to persist delayed events + simulated_send_lag = Duration(seconds=5) + event_creation_handler = self.hs.get_event_creation_handler() + orig_send_fn = event_creation_handler.create_and_send_nonmember_event + + async def slow_send_fn(*args: Any, **kwargs: Any) -> Any: + await self.clock.sleep(simulated_send_lag) + return await orig_send_fn(*args, **kwargs) + + with patch.object(event_creation_handler, orig_send_fn.__name__, slow_send_fn): + self.reactor.advance(send_after.as_secs()) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + # Confirm that the response lacks a Retry-After header, because the reason for this limit + # is the server taking an indeterminitely long time to process a delayed event, and the + # server doesn't know how much longer the client should wait before sending more requests + retry_after_headers = channel.headers.getRawHeaders("Retry-After") + assert not retry_after_headers + + # Wait until the delayed event gets persisted + self.reactor.advance(simulated_send_lag.as_secs()) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 5, + }, + } + ) + def test_delayed_event_user_limit_exceeded(self) -> None: + """ + Test that delayed event limits work properly when + the number of already scheduled events exceeds the configured limit. + + This can be invoked by the server admin lowering the configured limit & restarting the server + while a user has fewer scheduled delayed events than the old limit, but more than the new limit. + """ + send_after: Duration + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + f"rooms/%s/send/m.room.message?org.matrix.msc4140.delay={send_after.as_millis()}" + % self.room_id + ).encode("ascii"), + {"body": f"test (send after {send_after.as_secs()}s)", "msgtype": "m.text"}, + ) + + for i in range(4): + send_after = Duration(seconds=i) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Simulate restarting the server after having reconfigured the limit + # to be lower than the number of delayed events we just scheduled. + # + # Set the limit > 1 to test not having to wait for _all_ delayed events + # to be sent before being able to schedule a new one. + self.hs.config.server.max_delayed_events_per_user = 2 + + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + retry_after_header = channel.headers.getRawHeaders("Retry-After") + assert retry_after_header + retry_after_sec = int(retry_after_header[0]) + assert retry_after_sec > 0 + + # Wait until we're able to retry again (the retry time from the error response) + self.reactor.advance(retry_after_sec) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + @unittest.override_config({"max_event_delay_duration": "24h"}) def test_delayed_event_with_negative_delay(self) -> None: """Test that sending a delayed event fails if its delay is negative.""" @@ -2595,7 +2791,7 @@ class RoomDelayedEventTestCase(RoomBase): """ # Test that new delayed events are correctly ratelimited. - args = ( + make_delayed_event_request = lambda: self.make_request( "POST", ( "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=2000" @@ -2603,9 +2799,9 @@ class RoomDelayedEventTestCase(RoomBase): ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.OK, channel.code, channel.result) - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) # Add the current user to the ratelimit overrides, allowing them no ratelimiting. @@ -2614,7 +2810,7 @@ class RoomDelayedEventTestCase(RoomBase): ) # Test that the new delayed events aren't ratelimited anymore. - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.OK, channel.code, channel.result) From 0bd28389b19e165cc5e11782cb3c55abca94652a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Jul 2026 09:34:07 +0100 Subject: [PATCH 002/158] Improve caching for presence (#19939) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This does two things, first it adds a config flag to ignore rooms for the purposes of presence routing. Secondly, it changes the caching behaviour to try and improve the cache hit ratio. Previously, the size of the `do_users_share_a_room` cache (which stores pairs of users) needs to `O(n²)` for the number of online users, which is infeasible for large servers. Instead, we call `get_users_in_room` for both the syncing and updated users. This sounds more expensive, but a) we will already have cached the syncing user's rooms, and b) we will only calculate the updated user's rooms once (rather than once per syncing user). --------- Co-authored-by: Claude Fable 5 --- changelog.d/19935.feature | 1 + changelog.d/19939.misc | 1 + .../configuration/config_documentation.md | 10 + schema/synapse-config.schema.yaml | 12 + synapse/config/server.py | 4 + synapse/handlers/presence.py | 86 ++++++- synapse/storage/databases/main/roommember.py | 14 +- tests/handlers/test_presence.py | 213 ++++++++++++++++++ 8 files changed, 330 insertions(+), 11 deletions(-) create mode 100644 changelog.d/19935.feature create mode 100644 changelog.d/19939.misc diff --git a/changelog.d/19935.feature b/changelog.d/19935.feature new file mode 100644 index 0000000000..cd5fa804b8 --- /dev/null +++ b/changelog.d/19935.feature @@ -0,0 +1 @@ +Add an `exclude_rooms_from_presence` configuration option to stop presence being routed between users solely because they share one of the listed rooms. diff --git a/changelog.d/19939.misc b/changelog.d/19939.misc new file mode 100644 index 0000000000..59d480c98d --- /dev/null +++ b/changelog.d/19939.misc @@ -0,0 +1 @@ +Minor presence performance improvements for large servers. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index ec4098fdf6..62a4676adb 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4310,6 +4310,16 @@ exclude_rooms_from_sync: - '!foo:example.com' ``` --- +### `exclude_rooms_from_presence` + +*(array)* A list of rooms to exclude from presence updates. Presence will not be routed between two users solely because they share one of these rooms. Users who also share a non-excluded room continue to exchange presence as normal. Defaults to `[]`. + +Example configuration: +```yaml +exclude_rooms_from_presence: +- '!foo:example.com' +``` +--- ## Opentracing Configuration options related to Opentracing support. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 0d1aff071b..62e09377d4 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -5354,6 +5354,18 @@ properties: default: [] examples: - - "!foo:example.com" + exclude_rooms_from_presence: + type: array + description: >- + A list of rooms to exclude from presence updates. Presence will not be + routed between two users solely because they share one of these rooms. + Users who also share a non-excluded room continue to exchange presence as + normal. + items: + type: string + default: [] + examples: + - - "!foo:example.com" opentracing: type: object description: >- diff --git a/synapse/config/server.py b/synapse/config/server.py index 695a6cb9d4..832183f32b 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -936,6 +936,10 @@ class ServerConfig(Config): config.get("exclude_rooms_from_sync") or [] ) + self.rooms_to_exclude_from_presence: list[str] = ( + config.get("exclude_rooms_from_presence") or [] + ) + delete_stale_devices_after: str | None = ( config.get("delete_stale_devices_after") or None ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 28fbbfc1e1..6ee517786b 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -222,6 +222,12 @@ class BasePresenceHandler(abc.ABC): self._presence_enabled = hs.config.server.presence_enabled self._track_presence = hs.config.server.track_presence + # Rooms which, on their own, should not cause presence to be routed + # between their members. See `exclude_rooms_from_presence` in the config. + self._rooms_to_exclude_from_presence = frozenset( + hs.config.server.rooms_to_exclude_from_presence + ) + # The (configurable) presence state machine timers. self._last_active_granularity = ( hs.config.server.presence_last_active_granularity @@ -435,6 +441,7 @@ class BasePresenceHandler(abc.ABC): self.store, self.presence_router, states, + self._rooms_to_exclude_from_presence, ) for destinations, host_states in hosts_to_states: @@ -688,7 +695,12 @@ class WorkerPresenceHandler(BasePresenceHandler): async def notify_from_replication( self, states: list[UserPresenceState], stream_id: int ) -> None: - parties = await get_interested_parties(self.store, self.presence_router, states) + parties = await get_interested_parties( + self.store, + self.presence_router, + states, + self._rooms_to_exclude_from_presence, + ) room_ids_to_states, users_to_states = parties self.notifier.on_new_event( @@ -1141,6 +1153,7 @@ class PresenceHandler(BasePresenceHandler): self.store, self.presence_router, list(to_federation_ping.values()), + self._rooms_to_exclude_from_presence, ) for destinations, states in hosts_to_states: @@ -1414,7 +1427,12 @@ class PresenceHandler(BasePresenceHandler): """ stream_id, max_token = await self.store.update_presence(states) - parties = await get_interested_parties(self.store, self.presence_router, states) + parties = await get_interested_parties( + self.store, + self.presence_router, + states, + self._rooms_to_exclude_from_presence, + ) room_ids_to_states, users_to_states = parties self.notifier.on_new_event( @@ -1561,7 +1579,10 @@ class PresenceHandler(BasePresenceHandler): observed_user.to_string() ) - if observer_room_ids & observed_room_ids: + shared_room_ids = ( + observer_room_ids & observed_room_ids + ) - self._rooms_to_exclude_from_presence + if shared_room_ids: return True return False @@ -1672,6 +1693,12 @@ class PresenceHandler(BasePresenceHandler): to be handled. """ + # Excluded rooms should not, on their own, share presence between their + # members. This method is entirely per-room presence fan-out, so skip + # excluded rooms wholesale. + if room_id in self._rooms_to_exclude_from_presence: + return + # Sets of newly joined users. Note that if the local server is # joining a remote room for the first time we'll see both the joining # user and all remote users as newly joined. @@ -1929,6 +1956,9 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): self.server_name = hs.hostname self.clock = hs.get_clock() self.store = hs.get_datastores().main + self._rooms_to_exclude_from_presence = frozenset( + hs.config.server.rooms_to_exclude_from_presence + ) async def get_new_events( self, @@ -2043,9 +2073,31 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): **{SERVER_NAME_LABEL: self.server_name}, ).inc() - sharing_users = await self.store.do_users_share_a_room( - user_id, updated_users - ) + # An updated user is interesting if they share a + # (non-excluded) room with the syncing user. We check by + # intersecting the cached per-user room sets rather than via + # `do_users_share_a_room`: its per-pair cache has a + # quadratic working set and is cleared wholesale on every + # membership change, so on busy servers every check missed + # into SQL. + # + # For every presence update we need to run this code for + # every user that is currently syncing. The + # `get_rooms_for_user` will therefore be computed only once + # for each updated user regardless of the number of syncing + # users. + # + # The syncing user's rooms will also be cached as its needed + # during sync processing anyway. + my_rooms = await self.store.get_rooms_for_user(user_id) + if self._rooms_to_exclude_from_presence: + my_rooms = my_rooms - self._rooms_to_exclude_from_presence + rooms_by_user = await self.store.get_rooms_for_users(updated_users) + sharing_users = { + updated_user + for updated_user, rooms in rooms_by_user.items() + if not my_rooms.isdisjoint(rooms) + } interested_and_updated_users = ( sharing_users.union(additional_users_interested_in) @@ -2060,7 +2112,9 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): ).inc() users_interested_in = ( - await self.store.get_users_who_share_room_with_user(user_id) + await self.store.get_users_who_share_room_with_user( + user_id, self._rooms_to_exclude_from_presence + ) ) users_interested_in.update(additional_users_interested_in) @@ -2073,7 +2127,9 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): # No from_key has been specified. Return the presence for all users # this user is interested in interested_and_updated_users = ( - await self.store.get_users_who_share_room_with_user(user_id) + await self.store.get_users_who_share_room_with_user( + user_id, self._rooms_to_exclude_from_presence + ) ) interested_and_updated_users.update(additional_users_interested_in) @@ -2473,7 +2529,10 @@ def _combine_device_states( async def get_interested_parties( - store: DataStore, presence_router: PresenceRouter, states: list[UserPresenceState] + store: DataStore, + presence_router: PresenceRouter, + states: list[UserPresenceState], + excluded_rooms: AbstractSet[str] = frozenset(), ) -> tuple[dict[str, list[UserPresenceState]], dict[str, list[UserPresenceState]]]: """Given a list of states return which entities (rooms, users) are interested in the given states. @@ -2482,6 +2541,8 @@ async def get_interested_parties( store: The homeserver's data store. presence_router: A module for augmenting the destinations for presence updates. states: A list of incoming user presence updates. + excluded_rooms: Rooms which should not, on their own, cause presence to + be routed between their members. Returns: A 2-tuple of `(room_ids_to_states, users_to_states)`, @@ -2492,6 +2553,8 @@ async def get_interested_parties( for state in states: room_ids = await store.get_rooms_for_user(state.user_id) for room_id in room_ids: + if room_id in excluded_rooms: + continue room_ids_to_states.setdefault(room_id, []).append(state) # Always notify self @@ -2512,6 +2575,7 @@ async def get_interested_remotes( store: DataStore, presence_router: PresenceRouter, states: list[UserPresenceState], + excluded_rooms: AbstractSet[str] = frozenset(), ) -> list[tuple[StrCollection, Collection[UserPresenceState]]]: """Given a list of presence states figure out which remote servers should be sent which. @@ -2522,6 +2586,8 @@ async def get_interested_remotes( store: The homeserver's data store. presence_router: A module for augmenting the destinations for presence updates. states: A list of incoming user presence updates. + excluded_rooms: Rooms which should not, on their own, cause presence to + be routed to their remote members. Returns: A map from destinations to presence states to send to that destination. @@ -2535,6 +2601,8 @@ async def get_interested_remotes( room_ids = await store.get_rooms_for_user(state.user_id) hosts: set[str] = set() for room_id in room_ids: + if room_id in excluded_rooms: + continue room_hosts = await store.get_current_hosts_in_room(room_id) hosts.update(room_hosts) hosts_and_states.append((hosts, [state])) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 736f3e4c78..667ad1ace8 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -971,12 +971,22 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): return {u for u, share_room in user_dict.items() if share_room} - async def get_users_who_share_room_with_user(self, user_id: str) -> set[str]: - """Returns the set of users who share a room with `user_id`""" + async def get_users_who_share_room_with_user( + self, user_id: str, excluded_rooms: AbstractSet[str] = frozenset() + ) -> set[str]: + """Returns the set of users who share a room with `user_id`. + + Args: + user_id: The user to find the co-occupants of. + excluded_rooms: Rooms which should not, on their own, count as a + shared room. + """ room_ids = await self.get_rooms_for_user(user_id) user_who_share_room: set[str] = set() for room_id in room_ids: + if room_id in excluded_rooms: + continue user_ids = await self.get_users_in_room(room_id) user_who_share_room.update(user_ids) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 88d9d2aebb..0b638b2115 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -51,6 +51,8 @@ from synapse.handlers.presence import ( FEDERATION_TIMEOUT, PresenceHandler, WorkerPresenceHandler, + get_interested_parties, + get_interested_remotes, handle_timeout, handle_update, ) @@ -2323,6 +2325,217 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): return event +class PresenceExcludeRoomsTestCase(unittest.HomeserverTestCase): + """Tests that `exclude_rooms_from_presence` stops presence being routed + between users solely because they share an excluded room.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.hs = hs + self.store = hs.get_datastores().main + self.presence_router = hs.get_presence_router() + self.presence_handler = hs.get_presence_handler() + + self.user1 = self.register_user("user1", "pass") + self.token1 = self.login("user1", "pass") + self.user2 = self.register_user("user2", "pass") + self.token2 = self.login("user2", "pass") + + def test_excluded_rooms_not_routed(self) -> None: + # Two rooms that user1 is joined to. + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + + state = UserPresenceState.default(self.user1) + + # Without any exclusions both rooms are interested in user1's presence. + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties(self.store, self.presence_router, [state]) + ) + self.assertIn(excluded_room, room_ids_to_states) + self.assertIn(shared_room, room_ids_to_states) + + # Excluding one room drops it as an interested party, but the other + # (non-excluded) room still routes presence... + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties( + self.store, + self.presence_router, + [state], + frozenset({excluded_room}), + ) + ) + self.assertNotIn(excluded_room, room_ids_to_states) + self.assertIn(shared_room, room_ids_to_states) + + # ...and the user always receives their own presence, even when all of + # their rooms are excluded. + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties( + self.store, + self.presence_router, + [state], + frozenset({excluded_room, shared_room}), + ) + ) + self.assertNotIn(excluded_room, room_ids_to_states) + self.assertNotIn(shared_room, room_ids_to_states) + self.assertIn(self.user1, users_to_states) + + @override_config({"exclude_rooms_from_presence": ["!excluded:test"]}) + def test_config_populates_handler(self) -> None: + """The config option should be plumbed through to the presence handler + and the presence event source as a frozenset.""" + self.assertEqual( + self.presence_handler._rooms_to_exclude_from_presence, + frozenset({"!excluded:test"}), + ) + + event_source = self.hs.get_event_sources().sources.presence + self.assertEqual( + event_source._rooms_to_exclude_from_presence, + frozenset({"!excluded:test"}), + ) + + def test_is_visible_respects_excluded_rooms(self) -> None: + """`is_visible` (which drives the read side of /sync) should not + consider two users to share presence solely via an excluded room.""" + user1 = UserID.from_string(self.user1) + user2 = UserID.from_string(self.user2) + + # A single shared room: the two users can see each other's presence. + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(excluded_room, self.user2, tok=self.token2) + + self.assertTrue( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + # Excluding the only shared room hides presence between them. + self.presence_handler._rooms_to_exclude_from_presence = frozenset( + {excluded_room} + ) + self.assertFalse( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + # But a second, non-excluded shared room restores visibility. + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + self.assertTrue( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + def test_get_interested_remotes_respects_excluded_rooms(self) -> None: + """The federation fan-out side (`get_interested_remotes`) must not route + presence to servers reached solely via an excluded room.""" + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + state = UserPresenceState.default(self.user1) + + def hosts_for(excluded: frozenset) -> set: + result = self.get_success( + get_interested_remotes( + self.store, self.presence_router, [state], excluded + ) + ) + hosts: set[str] = set() + for room_hosts, _ in result: + hosts.update(room_hosts) + return hosts + + # The local server is a host in the room (all members are local here), + # so presence would be routed there... + self.assertIn("test", hosts_for(frozenset())) + # ...but excluding the only room removes it as a source of destinations. + self.assertNotIn("test", hosts_for(frozenset({excluded_room}))) + + +class PresenceGetNewEventsStreamTestCase(unittest.HomeserverTestCase): + """Tests the incremental (`from_key`) branch of + `PresenceEventSource.get_new_events`, which decides which updated users are + interesting to the syncing user by intersecting their cached room sets. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.presence_handler = hs.get_presence_handler() + self.event_source = hs.get_event_sources().sources.presence + + self.user1 = self.register_user("user1", "pass") + self.token1 = self.login("user1", "pass") + self.user2 = self.register_user("user2", "pass") + self.token2 = self.login("user2", "pass") + self.user3 = self.register_user("user3", "pass") + self.token3 = self.login("user3", "pass") + + def _set_presence(self, user_id: str, state: str = "online") -> None: + self.get_success( + self.presence_handler.set_state( + UserID.from_string(user_id), "dev", {"presence": state} + ) + ) + + def _updated_users_seen_by(self, user_id: str, from_key: int) -> set[str]: + states, _ = self.get_success( + self.event_source.get_new_events( + user=UserID.from_string(user_id), from_key=from_key + ) + ) + return {state.user_id for state in states} + + def test_incremental_interest(self) -> None: + """A syncing user sees updates from users they share a room with (and + themselves), but not from strangers.""" + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + # user3 is in an unrelated room. + self.helper.create_room_as(self.user3, tok=self.token3) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1) + self._set_presence(self.user2) + self._set_presence(self.user3) + + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertIn(self.user1, seen) + self.assertIn(self.user2, seen) # always sees own updates + self.assertNotIn(self.user3, seen) + + def test_incremental_interest_excluded_room(self) -> None: + """Sharing only an excluded room does not make an updated user + interesting; sharing an additional normal room does.""" + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(excluded_room, self.user2, tok=self.token2) + + self.event_source._rooms_to_exclude_from_presence = frozenset({excluded_room}) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1, "online") + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertNotIn(self.user1, seen) + + # A second, non-excluded shared room restores interest. (Use a + # different presence state, as repeating the same one would not + # generate a new update.) + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1, "unavailable") + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertIn(self.user1, seen) + + class WorkerPresenceThrottleTestCase(BaseMultiWorkerStreamTestCase): """Tests that sync workers suppress the per-sync-request presence updates that the presence writer would discard anyway, while relaying genuine From be511b22a2d0c07bfa2a6672d00e8c1ba1be60fc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Jul 2026 10:08:57 +0100 Subject: [PATCH 003/158] Send offline presence for stale states when presence is disabled (#19948) If presence is disabled after having been enabled, the presence states in the database (and hence on clients) are frozen at whatever they were when presence was last enabled: nothing writes to the presence stream any more and /sync omits the presence section entirely, so clients show the old presence states forever. Fix this in two parts: 1. At startup, if presence is disabled but the database still contains non-offline presence states, the presence writer sends out one final round of updates marking those users as offline. 2. /sync no longer unconditionally omits presence when presence is disabled: incremental syncs whose since token is behind the presence stream still get the straggling updates. As the stream doesn't advance while presence is disabled, clients catch up once and the check then short-circuits to a token comparison. Remote servers already handle this themselves by timing out our users ([`FEDERATION_TIMEOUT`](https://github.com/element-hq/synapse/blob/4d8905a15a417ed0054ec2533d243932d890bbbd/synapse/handlers/presence.py#L194-L198)), so no federation changes are needed. Note that this only fixes the issue if presence is fully disabled. If set to `untracked` we still have the same issue, however since modules would still write to presence we can't just clobber everything like we do in this patch. --- changelog.d/19948.bugfix | 1 + synapse/handlers/presence.py | 39 +++++++++++ synapse/handlers/sync.py | 16 ++++- tests/handlers/test_presence.py | 119 ++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19948.bugfix diff --git a/changelog.d/19948.bugfix b/changelog.d/19948.bugfix new file mode 100644 index 0000000000..5cd34d1abf --- /dev/null +++ b/changelog.d/19948.bugfix @@ -0,0 +1 @@ +Fix presence states being shown to clients forever after presence is disabled, by marking any previously only users as offline. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6ee517786b..55dd5ffb59 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -122,6 +122,7 @@ from synapse.types import ( ) from synapse.util.async_helpers import Linearizer from synapse.util.duration import Duration +from synapse.util.iterutils import batch_iter from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer @@ -984,6 +985,14 @@ class PresenceHandler(BasePresenceHandler): Duration(minutes=1), ) + if not self._presence_enabled and self.user_to_current_state: + # Presence is disabled but the database still contains non-offline + # presence states, i.e. presence used to be enabled. Nothing writes + # to the presence stream while presence is disabled, so without + # intervention clients would show the stale states forever. Send + # out one final round of updates marking everyone as offline. + self.clock.call_when_running(self._mark_stale_presence_as_offline) + presence_wheel_timer_size_gauge.register_hook( homeserver_instance_id=hs.get_instance_id(), hook=lambda: {(self.server_name,): len(self.wheel_timer)}, @@ -1041,6 +1050,36 @@ class PresenceHandler(BasePresenceHandler): [self.user_to_current_state[user_id] for user_id in unpersisted] ) + @wrap_as_background_process("PresenceHandler._mark_stale_presence_as_offline") + async def _mark_stale_presence_as_offline(self) -> None: + """One-off job, run at startup when presence is disabled, that marks + any non-offline presence states left over from when presence was + enabled as offline, and streams the changes out to clients. + """ + states = [ + state.copy_and_replace( + state=PresenceState.OFFLINE, + status_msg=None, + currently_active=False, + ) + for state in self.user_to_current_state.values() + if state.state != PresenceState.OFFLINE + ] + if not states: + return + + logger.info( + "Presence is disabled: marking %d stale presence states as offline", + len(states), + ) + + self.user_to_current_state.update({state.user_id: state for state in states}) + + # There may be a lot of stale states (e.g. everyone that was online + # when presence was disabled), so persist them in batches. + for batch in batch_iter(states, 500): + await self._persist_and_notify(list(batch)) + async def _update_states( self, new_states: Iterable[UserPresenceState], diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9ecfe0da0f..a05d6c6e59 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1759,9 +1759,19 @@ class SyncHandler: await self._generate_sync_entry_for_account_data(sync_result_builder) # Presence data is included if the server has it enabled and not filtered out. - include_presence_data = bool( - self.hs_config.server.presence_enabled - and not sync_config.filter_collection.blocks_all_presence() + presence_enabled = bool(self.hs_config.server.presence_enabled) + if not presence_enabled and since_token is not None: + # Even with presence disabled we send down any presence updates the + # client hasn't yet seen, so that the "mark everyone as offline" + # updates written when presence was disabled reach clients that + # would otherwise show the old presence states forever. The stream + # doesn't advance while presence is disabled, so once clients have + # caught up this check stops any further presence work. + presence_enabled = ( + since_token.presence_key < sync_result_builder.now_token.presence_key + ) + include_presence_data = ( + presence_enabled and not sync_config.filter_collection.blocks_all_presence() ) # Device list updates are sent if a since token is provided. include_device_list_updates = bool(since_token and since_token.device_list_key) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 0b638b2115..a562050842 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -1002,6 +1002,125 @@ class PresenceHandlerInitTestCase(unittest.HomeserverTestCase): ) self.assertEqual(state.state, sync_state) + @unittest.override_config({"presence": {"enabled": False}}) + def test_restored_presence_flushed_offline_when_presence_disabled(self) -> None: + """If presence is disabled, any non-offline presence states left in the + database from when presence was enabled should be marked as offline at + startup, and the updates streamed out to clients. + """ + main_store = self.hs.get_datastores().main + before_token = main_store.get_current_presence_token() + + # Get the handler, which schedules the startup flush. + presence_handler = self.hs.get_presence_handler() + + # Fire pending `call_when_running` hooks and let the flush complete. + self.reactor.run() + self.reactor.advance(0) + + # The user should now be offline, both in memory and in the database. + state = self.get_success( + presence_handler.get_state(UserID.from_string(self.user_id)) + ) + self.assertEqual(state.state, PresenceState.OFFLINE) + + db_state = self.get_success(main_store.get_presence_for_users([self.user_id]))[ + self.user_id + ] + self.assertEqual(db_state.state, PresenceState.OFFLINE) + + # The flush must advance the presence stream so that syncing clients + # are sent the offline updates. + self.assertGreater(main_store.get_current_presence_token(), before_token) + + +class PresenceDisabledSyncTestCase(unittest.HomeserverTestCase): + """Tests that stale presence states left over from when presence was + enabled reach clients over /sync, and that the startup flush marks them as + offline and sends the offline updates down /sync too. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + ] + + @unittest.override_config({"presence": {"enabled": False}}) + def test_stale_presence_flushed_offline_and_sent_on_sync(self) -> None: + user1 = self.register_user("alice", "pass") + user1_tok = self.login(user1, "pass") + user2 = self.register_user("bob", "pass") + user2_tok = self.login(user2, "pass") + + room_id = self.helper.create_room_as(user1, tok=user1_tok) + self.helper.join(room_id, user2, tok=user2_tok) + + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch = channel.json_body["next_batch"] + + # Seed a stale online presence state for user1, left over from when + # presence was enabled: in the database, and in the presence handler's + # in-memory state (which at startup is preloaded from the database). + now = self.clock.time_msec() + stale_state = UserPresenceState( + user_id=user1, + state=PresenceState.ONLINE, + last_active_ts=now, + last_federation_update_ts=now, + last_user_sync_ts=now, + status_msg=None, + currently_active=True, + ) + main_store = self.hs.get_datastores().main + self.get_success(main_store.update_presence([stale_state])) + + presence_handler = self.hs.get_presence_handler() + assert isinstance(presence_handler, PresenceHandler) + presence_handler.user_to_current_state[user1] = stale_state + + # The stale state comes down user2's incremental sync, even though + # presence is disabled. + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + presence_events = channel.json_body["presence"]["events"] + self.assertEqual( + [(e["sender"], e["content"]["presence"]) for e in presence_events], + [(user1, PresenceState.ONLINE)], + ) + next_batch = channel.json_body["next_batch"] + + # Run the startup flush, as scheduled when the presence writer starts + # up with presence disabled. + self.get_success(presence_handler._mark_stale_presence_as_offline()) + + # The stale state should have been marked offline in the database... + db_state = self.get_success(main_store.get_presence_for_users([user1]))[user1] + self.assertEqual(db_state.state, PresenceState.OFFLINE) + + # ... and the offline update also comes down user2's sync. + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + presence_events = channel.json_body["presence"]["events"] + self.assertEqual( + [(e["sender"], e["content"]["presence"]) for e in presence_events], + [(user1, PresenceState.OFFLINE)], + ) + + # Once caught up, further syncs include no presence. + next_batch = channel.json_body["next_batch"] + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body.get("presence", {}).get("events", []), []) + # Timer values used by `PresenceConfigurableTimersTestCase`, all larger than # the corresponding defaults. From 733620acd75d7e022c8af69bc30e4992da43f8ff Mon Sep 17 00:00:00 2001 From: Sandro Date: Mon, 13 Jul 2026 11:39:56 +0200 Subject: [PATCH 004/158] Fix SYNAPSE_ASYNC_IO_REACTOR=1 on Python 3.14 (#19949) --- changelog.d/19949.bugfix | 1 + synapse/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19949.bugfix diff --git a/changelog.d/19949.bugfix b/changelog.d/19949.bugfix new file mode 100644 index 0000000000..24ddd4dc4f --- /dev/null +++ b/changelog.d/19949.bugfix @@ -0,0 +1 @@ +Fix `SYNAPSE_ASYNC_IO_REACTOR=1` on Python 3.14. diff --git a/synapse/__init__.py b/synapse/__init__.py index 3acfc1a0d7..a223066f04 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -49,7 +49,9 @@ if strtobool(os.environ.get("SYNAPSE_ASYNC_IO_REACTOR", "0")): from twisted.internet import asyncioreactor - asyncioreactor.install(asyncio.get_event_loop()) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + asyncioreactor.install(loop) # Twisted and canonicaljson will fail to import when this file is executed to # get the __version__ during a fresh install. That's OK and subsequent calls to From 292bbb8f30b95ced2679c9a60e8af3e14cbbbe6d Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 13 Jul 2026 13:32:09 +0100 Subject: [PATCH 005/158] Fix a bug causing device list pruning to skip some rows when the transaction gets retried. (#19947) Introduced in: #19473 Noticed in: https://github.com/element-hq/synapse/pull/19556#discussion_r3505783541 I have not experienced the bug in the real world, it's just something I noticed by reading. -- Fix bug in `_prune_device_lists_changes_in_room` when transaction is retried The `nonlocal` variable is a footgun as it increments the counter even though the transaction did not commit yet and may still be retried. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19947.bugfix | 1 + synapse/storage/databases/main/devices.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 changelog.d/19947.bugfix diff --git a/changelog.d/19947.bugfix b/changelog.d/19947.bugfix new file mode 100644 index 0000000000..c16d6deea0 --- /dev/null +++ b/changelog.d/19947.bugfix @@ -0,0 +1 @@ +Fix a bug causing device list pruning to skip some rows when the transaction gets retried. \ No newline at end of file diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index b293b47fea..0e4c8ac491 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -2561,9 +2561,14 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): # We default to 0 here as that is less than all possible stream IDs. min_stream_id = 0 - def prune_device_lists_changes_in_room_txn(txn: LoggingTransaction) -> int: - nonlocal min_stream_id - + def prune_device_lists_changes_in_room_txn( + txn: LoggingTransaction, min_stream_id: int + ) -> tuple[int, int]: + """ + Returns tuple of: + - number of rows deleted + - new `min_stream_id` for the next iteration + """ delete_sql = """ DELETE FROM device_lists_changes_in_room WHERE stream_id IN ( @@ -2596,13 +2601,14 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): updatevalues={"stream_id": min_stream_id}, ) - return num_deleted + return num_deleted, min_stream_id progress_num_rows_deleted = 0 while True: - batch_deleted = await self.db_pool.runInteraction( + batch_deleted, min_stream_id = await self.db_pool.runInteraction( "prune_device_lists_changes_in_room", prune_device_lists_changes_in_room_txn, + min_stream_id, ) finished = batch_deleted < PRUNE_DEVICE_LISTS_BATCH_SIZE From c93d5230c00d3b1c4d17fc5c33c3e05a080fab72 Mon Sep 17 00:00:00 2001 From: Wessel Date: Mon, 13 Jul 2026 18:06:17 +0200 Subject: [PATCH 006/158] Honor `de.sorunome.msc2409.push_ephemeral` flag in appservice handler (#19928) --- changelog.d/19928.bugfix | 1 + synapse/handlers/appservice.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19928.bugfix diff --git a/changelog.d/19928.bugfix b/changelog.d/19928.bugfix new file mode 100644 index 0000000000..d69e4febbe --- /dev/null +++ b/changelog.d/19928.bugfix @@ -0,0 +1 @@ +Fix a regression where application services that opted into ephemeral events using the legacy `de.sorunome.msc2409.push_ephemeral` registration flag stopped receiving ephemeral events (including to-device messages used for encryption). Introduced in v1.156.0. diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 36b2f63e41..68b8aa71f1 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -313,7 +313,10 @@ class ApplicationServicesHandler: StreamKeyType.PRESENCE, StreamKeyType.TO_DEVICE, ) - and service.supports_ephemeral + # Honour both the stable `receive_ephemeral` registration flag and the + # legacy `de.sorunome.msc2409.push_ephemeral` one, matching the + # transaction body built in `ApplicationServiceApi.push_bulk`. + and (service.supports_ephemeral or service.supports_unstable_ephemeral) ) or ( stream_key == StreamKeyType.DEVICE_LIST From bf0f4f6a9dcc14f92f1a1256ad64755b373bfded Mon Sep 17 00:00:00 2001 From: SpiritCroc Date: Mon, 13 Jul 2026 18:12:26 +0200 Subject: [PATCH 007/158] Support MSC4446: allow moving fully read markers backwards (#19663) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19663.feature | 1 + rust/src/config/mod.rs | 1 + rust/src/handlers/versions.rs | 4 + synapse/config/experimental.py | 4 + synapse/handlers/read_marker.py | 8 +- synapse/rest/client/read_marker.py | 17 ++++ synapse/rest/client/receipts.py | 22 +++++ tests/rest/client/test_read_marker.py | 130 +++++++++++++++++++++++++ tests/rest/client/test_receipts.py | 135 ++++++++++++++++++++++++++ tests/rest/client/test_versions.py | 11 +++ 10 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19663.feature diff --git a/changelog.d/19663.feature b/changelog.d/19663.feature new file mode 100644 index 0000000000..9f61182a90 --- /dev/null +++ b/changelog.d/19663.feature @@ -0,0 +1 @@ +Support [MSC4446](https://github.com/matrix-org/matrix-spec-proposals/pull/4446) for moving fully read markers backwards. Contributed by @SpiritCroc @ Beeper. diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index b5f641613b..d79d12a83a 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -72,4 +72,5 @@ pub struct ExperimentalConfig { pub msc4222_enabled: bool, pub msc4491_enabled: bool, pub msc4143_enabled: bool, + pub msc4446_enabled: bool, } diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index bfe064eb5e..25da9d23fc 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -266,6 +266,9 @@ pub struct UnstableFeatureMap { /// MSC4143: Matrix RTC transports (LiveKit backend) #[serde(rename = "org.matrix.msc4143")] msc4143_enabled: bool, + /// MSC4446: Allow moving the fully read marker backwards. + #[serde(rename = "com.beeper.msc4446")] + msc4446_enabled: bool, // Whether new rooms will be set to encrypted or not (based on presets). #[serde(rename = "io.element.e2ee_forced.public")] @@ -316,6 +319,7 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4445_initial_sync_timeline_topological_ordering: true, msc4491_enabled: config.experimental.msc4491_enabled, msc4143_enabled: config.experimental.msc4143_enabled, + msc4446_enabled: config.experimental.msc4446_enabled, e2ee_forced_public: config .room .encryption_enabled_by_default_for_room_presets diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index f99f7b139e..6ad9f53517 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -287,6 +287,10 @@ class ExperimentalConfig(Config): # (and MSC4308: Thread Subscriptions extension to Sliding Sync) self.msc4306_enabled: bool = experimental.get("msc4306_enabled", False) + # MSC4446: Allow moving the fully read marker backwards. + # Tracked in: https://github.com/element-hq/synapse/issues/19940 + self.msc4446_enabled: bool = experimental.get("msc4446_enabled", False) + # MSC4354: Sticky Events # Tracked in: https://github.com/element-hq/synapse/issues/19409 # Note that sticky events persisted before this feature is enabled will not be diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index 85d2dd62bb..3f3b9e6d8b 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -41,7 +41,11 @@ class ReadMarkerHandler: ) async def received_client_read_marker( - self, room_id: str, user_id: str, event_id: str + self, + room_id: str, + user_id: str, + event_id: str, + allow_backward: bool = False, ) -> None: """Updates the read marker for a given user in a given room if the event ID given is ahead in the stream relative to the current read marker. @@ -59,7 +63,7 @@ class ReadMarkerHandler: # Get event ordering, this also ensures we know about the event event_ordering = await self.store.get_event_ordering(event_id, room_id) - if existing_read_marker: + if existing_read_marker and not allow_backward: try: old_event_ordering = await self.store.get_event_ordering( existing_read_marker["event_id"], room_id diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index 874e7487bf..8e0f2a2e7a 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -23,6 +23,7 @@ import logging from typing import TYPE_CHECKING from synapse.api.constants import ReceiptTypes +from synapse.api.errors import Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -66,6 +67,21 @@ class ReadMarkerRestServlet(RestServlet): body = parse_json_object_from_request(request) unrecognized_types = set(body.keys()) - self._known_receipt_types + + if self.config.experimental.msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + 400, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + + # Prevent considering the `allow_backward` field as a receipt type. + unrecognized_types -= {"com.beeper.allow_backward"} + else: + allow_backward = False + if unrecognized_types: # It's fine if there are unrecognized receipt types, but let's log # it to help debug clients that have typoed the receipt type. @@ -86,6 +102,7 @@ class ReadMarkerRestServlet(RestServlet): room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, ) else: await self.receipts_handler.received_client_receipt( diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index d3a43537bb..949a1e64ad 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -20,6 +20,7 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING from synapse.api.constants import MAIN_TIMELINE, ReceiptTypes @@ -50,6 +51,7 @@ class ReceiptRestServlet(RestServlet): self.read_marker_handler = hs.get_read_marker_handler() self.presence_handler = hs.get_presence_handler() self._main_store = hs.get_datastores().main + self._msc4446_enabled = hs.config.experimental.msc4446_enabled self._known_receipt_types = { ReceiptTypes.READ, @@ -73,6 +75,25 @@ class ReceiptRestServlet(RestServlet): body = parse_json_object_from_request(request) + if self._msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + + if allow_backward and receipt_type != ReceiptTypes.FULLY_READ: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward is only allowed to be true for " + f"{ReceiptTypes.FULLY_READ}.", + Codes.INVALID_PARAM, + ) + else: + allow_backward = False + # Pull the thread ID, if one exists. thread_id = None if "thread_id" in body: @@ -108,6 +129,7 @@ class ReceiptRestServlet(RestServlet): room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, ) else: await self.receipts_handler.received_client_receipt( diff --git a/tests/rest/client/test_read_marker.py b/tests/rest/client/test_read_marker.py index c8bb0da5e6..ad13d3607e 100644 --- a/tests/rest/client/test_read_marker.py +++ b/tests/rest/client/test_read_marker.py @@ -66,6 +66,19 @@ class ReadMarkerTestCase(unittest.HomeserverTestCase): self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() + def _get_fully_read_marker(self, room_id: str) -> str | None: + content = self.get_success( + self.store.get_account_data_for_room_and_type( + self.owner, + room_id, + "m.fully_read", + ) + ) + if content is None: + return None + + return content.get("event_id") + def test_send_read_marker(self) -> None: room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) @@ -98,6 +111,123 @@ class ReadMarkerTestCase(unittest.HomeserverTestCase): ) self.assertEqual(channel.code, 200, channel.result) + def test_send_read_marker_does_not_move_backwards_by_default(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_send_read_marker_can_move_backwards_with_opt_in(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id, "com.beeper.allow_backward": True}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), older_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_send_read_marker_does_not_move_backwards_with_explicit_opt_out( + self, + ) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={ + "m.fully_read": older_event_id, + "com.beeper.allow_backward": False, + }, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + def test_send_read_marker_ignores_opt_in_when_feature_disabled(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id, "com.beeper.allow_backward": True}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + def test_send_read_marker_missing_previous_event(self) -> None: """ Test moving a read marker from an event that previously existed but was diff --git a/tests/rest/client/test_receipts.py b/tests/rest/client/test_receipts.py index 3a6a869c54..0835eec6de 100644 --- a/tests/rest/client/test_receipts.py +++ b/tests/rest/client/test_receipts.py @@ -24,6 +24,7 @@ from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EduTypes, EventTypes, HistoryVisibility, ReceiptTypes +from synapse.api.errors import Codes from synapse.rest.client import login, receipts, room, sync from synapse.server import HomeServer from synapse.types import JsonDict @@ -44,6 +45,7 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.url = "/sync?since=%s" self.next_batch = "s0" + self.store = hs.get_datastores().main # Register the first user self.user_id = self.register_user("kermit", "monkey") @@ -59,6 +61,19 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): # Join the second user self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + def _get_fully_read_marker(self) -> str | None: + content = self.get_success( + self.store.get_account_data_for_room_and_type( + self.user2, + self.room_id, + ReceiptTypes.FULLY_READ, + ) + ) + if content is None: + return None + + return content.get("event_id") + def test_send_receipt(self) -> None: # Send a message. res = self.helper.send(self.room_id, body="hello", tok=self.tok) @@ -258,6 +273,126 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) self.assertEqual(channel.json_body["errcode"], "M_NOT_JSON", channel.json_body) + def test_fully_read_receipt_does_not_move_backwards_by_default(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_fully_read_receipt_can_move_backwards_with_opt_in(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), older_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_fully_read_receipt_does_not_move_backwards_with_explicit_opt_out( + self, + ) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": False}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_allow_backward_is_rejected_for_read_receipts(self) -> None: + event_id = self.helper.send(self.room_id, body="1", tok=self.tok)["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ}/{event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + def test_allow_backward_is_ignored_when_feature_disabled(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + def _get_read_receipt(self) -> JsonDict | None: """Syncs and returns the read receipt.""" diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index d656098469..bbdbe38e07 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -142,6 +142,17 @@ class VersionsTestCase(unittest.HomeserverTestCase): channel.json_body, ) + def test_msc4446_false_by_default(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertFalse(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_msc4446_true_if_enabled(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertTrue(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + def _sanity_check_versions_response(self, versions_response: JsonDict) -> None: """ Make sure this looks like a `/_matrix/client/versions` response From aafb1fabcf398d00f247bc3c877e7a1b81f09fd1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:04:15 +0000 Subject: [PATCH 008/158] Bump golang.org/x/crypto from 0.51.0 to 0.52.0 in /complement (#19925) Signed-off-by: dependabot[bot] --- complement/go.mod | 2 +- complement/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/complement/go.mod b/complement/go.mod index aa2333a5a0..cea013573b 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -51,7 +51,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect - golang.org/x/crypto v0.51.0 // indirect + golang.org/x/crypto v0.52.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.45.0 // indirect diff --git a/complement/go.sum b/complement/go.sum index 49c3724e83..4858bc78c8 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -122,8 +122,8 @@ go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXd golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= From 1b181ee81351f221625b206783e8598ad067145f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:24:11 +0000 Subject: [PATCH 009/158] Bump log from 0.4.32 to 0.4.33 in the patches group (#19917) Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87f50ea5df..aaf8fa4512 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -718,9 +718,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru-slab" From 10f6370daf7538c4079a35d4881ee3be22006c0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:24:39 +0000 Subject: [PATCH 010/158] Bump bytes from 1.11.1 to 1.12.0 (#19918) Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aaf8fa4512..7ca2265aca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,9 +78,9 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" [[package]] name = "cc" From 60aa4466c866d52789556fad83b1ae34c83a6511 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:25:07 +0000 Subject: [PATCH 011/158] Bump the minor-and-patches group with 2 updates (#19919) Signed-off-by: dependabot[bot] --- .github/workflows/complement_tests.yml | 2 +- .github/workflows/docs-pr.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/latest_deps.yml | 2 +- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/release-artifacts.yml | 8 ++++---- .github/workflows/schema.yaml | 4 ++-- .github/workflows/tests.yml | 14 +++++++------- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index a891802ac8..9d9d67d23b 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -65,7 +65,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index e1a5b7be89..56d5b7a87a 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -24,7 +24,7 @@ jobs: mdbook-version: '0.5.2' - name: Setup python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 7236bf99d9..9ded866c95 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -64,7 +64,7 @@ jobs: run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - name: Setup python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 815593ffcd..d520c817f0 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -93,7 +93,7 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: pip install .[all,test] diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 06545bd18a..b18468a95b 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - run: pip install tomli diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index c6b9f60baf..b13b5956cb 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - id: set-distros @@ -78,7 +78,7 @@ jobs: ${{ runner.os }}-buildx- - name: Set up python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -131,7 +131,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: # setup-python@v4 doesn't impose a default python version. Need to use 3.x # here, because `python` on osx points to Python 2.7. @@ -168,7 +168,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index e36114d354..388e7475d6 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - name: Install check-jsonschema @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - name: Install PyYAML diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 45fa2b8cad..7b674de224 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -127,7 +127,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20' 'sqlglot>=28.0.0'" @@ -137,7 +137,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: .ci/scripts/check_lockfile.py @@ -220,7 +220,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: "pip install 'towncrier>=18.6.0rc1'" @@ -328,7 +328,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod @@ -345,7 +345,7 @@ jobs: if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: "pip install rstcheck" @@ -394,7 +394,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - id: get-matrix @@ -485,7 +485,7 @@ jobs: sudo apt-get -qq install build-essential libffi-dev python3-dev \ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.10" From ff7c3b94180eac634a6fd983ac1527976cb812fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:25:28 +0000 Subject: [PATCH 012/158] Bump actions/cache from 5.0.5 to 6.1.0 (#19920) Signed-off-by: dependabot[bot] --- .github/workflows/release-artifacts.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index b13b5956cb..9df5055e60 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -70,7 +70,7 @@ jobs: uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Set up docker layer caching - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7b674de224..1887cc0afd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -194,7 +194,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | .mypy_cache From 6bfec47c477cb14fd937904dbf76a7e50fe62804 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:46:28 +0100 Subject: [PATCH 013/158] Revert "Fix flaky 3PID inhibit error unit tests" (#19916) --- changelog.d/19913.misc | 1 - changelog.d/19916.misc | 1 + tests/rest/client/test_account.py | 16 +++++++--------- tests/rest/client/test_register.py | 7 ++++--- tests/server.py | 5 +---- tests/unittest.py | 4 ---- 6 files changed, 13 insertions(+), 21 deletions(-) delete mode 100644 changelog.d/19913.misc create mode 100644 changelog.d/19916.misc diff --git a/changelog.d/19913.misc b/changelog.d/19913.misc deleted file mode 100644 index 31b5a4d37b..0000000000 --- a/changelog.d/19913.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a flake in 3PID inhibit error unit tests, causing occasional failures in CI. \ No newline at end of file diff --git a/changelog.d/19916.misc b/changelog.d/19916.misc new file mode 100644 index 0000000000..aeec5ae412 --- /dev/null +++ b/changelog.d/19916.misc @@ -0,0 +1 @@ +Add note to 3PID email token request unit tests that the endpoint being tested can have an expected, artificial delay of up to 1s. \ No newline at end of file diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 158d7f33f0..c0494c606f 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -325,13 +325,8 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): email = "test@example.com" client_secret = "foobar" - session_id = self._request_token( - email, - client_secret, - # The endpoint intentionally adds up to 1000ms of jitter to avoid - # leaking whether the email address is bound to an account. - timeout_ms=3000, - ) + + session_id = self._request_token(email, client_secret) self.assertIsNotNone(session_id) @@ -370,18 +365,21 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): client_secret: str, ip: str = "127.0.0.1", next_link: str | None = None, - timeout_ms: int = 1000, ) -> str: body = {"client_secret": client_secret, "email": email, "send_attempt": 1} if next_link is not None: body["next_link"] = next_link + channel = self.make_request( "POST", b"account/password/email/requestToken", body, client_ip=ip, - timeout_ms=timeout_ms, + await_result=False, ) + # Note: The endpoint intentionally adds up to 1000ms of jitter to avoid + # leaking whether the email address is bound to an account. + channel.await_result(timeout_ms=1000) if channel.code != 200: raise HttpResponseException( diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index a9f3ac2462..1b04ea9d5c 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -753,10 +753,11 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): "POST", b"register/email/requestToken", {"client_secret": "foobar", "email": email, "send_attempt": 1}, - # The endpoint intentionally adds up to 1000ms of jitter to avoid - # leaking whether the email address is already bound to an account. - timeout_ms=3000, + await_result=False, ) + # Note: The endpoint intentionally adds up to 1000ms of jitter to avoid + # leaking whether the email address is bound to an account. + channel.await_result(timeout_ms=1000) self.assertEqual(200, channel.code, channel.result) self.assertIsNotNone(channel.json_body.get("sid")) diff --git a/tests/server.py b/tests/server.py index 15a7661c34..052f76d755 100644 --- a/tests/server.py +++ b/tests/server.py @@ -459,7 +459,6 @@ def make_request( await_result: bool = True, custom_headers: Iterable[CustomHeaderType] | None = None, client_ip: str = "127.0.0.1", - timeout_ms: int = 1000, ) -> FakeChannel: """ Make a web request using the given method, path and content, and render it @@ -488,8 +487,6 @@ def make_request( custom_headers: (name, value) pairs to add as request headers client_ip: The IP to use as the requesting IP. Useful for testing ratelimiting. - timeout_ms: if `await_result` is `True`, the amount of time to wait on - the request before timing out. Ignored otherwise. Returns: channel @@ -574,7 +571,7 @@ def make_request( req.requestReceived(method, path, b"1.1") if await_result: - channel.await_result(timeout_ms=timeout_ms) + channel.await_result() return channel diff --git a/tests/unittest.py b/tests/unittest.py index 202f7120ef..3d130c1d77 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -571,7 +571,6 @@ class HomeserverTestCase(TestCase): await_result: bool = True, custom_headers: Iterable[CustomHeaderType] | None = None, client_ip: str = "127.0.0.1", - timeout_ms: int = 1000, ) -> FakeChannel: """ Create a SynapseRequest at the path using the method and containing the @@ -600,8 +599,6 @@ class HomeserverTestCase(TestCase): client_ip: The IP to use as the requesting IP. Useful for testing ratelimiting. - timeout_ms: if `await_result` is `True`, the amount of time to wait on - the request before timing out. Ignored otherwise. Returns: The FakeChannel object which stores the result of the request. @@ -621,7 +618,6 @@ class HomeserverTestCase(TestCase): await_result, custom_headers, client_ip, - timeout_ms, ) def setup_test_homeserver( From 1979fcca52e4fec8eb89bf804145b86d86ab9b89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:19:15 +0000 Subject: [PATCH 014/158] Bump actions/checkout from 6.0.3 to 7.0.0 (#19921) Signed-off-by: dependabot[bot] --- .github/workflows/complement_tests.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/docs-pr.yaml | 4 +- .github/workflows/docs.yaml | 2 +- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 8 ++-- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/push_complement_image.yml | 6 +-- .github/workflows/release-artifacts.yml | 8 ++-- .github/workflows/schema.yaml | 4 +- .github/workflows/tests.yml | 44 ++++++++++----------- .github/workflows/triage_labelled.yml | 2 +- .github/workflows/twisted_trunk.yml | 8 ++-- 13 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index 9d9d67d23b..629a48dcc7 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout synapse codebase - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: path: synapse diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 39ddb61918..8ecf8fb723 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,7 +31,7 @@ jobs: uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 56d5b7a87a..a1b355e74f 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -13,7 +13,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -50,7 +50,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 9ded866c95..f8479a8c51 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,7 @@ jobs: needs: - pre steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index e0817698f4..88decb33d9 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index d520c817f0..0b39c5c372 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -42,7 +42,7 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -77,7 +77,7 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -151,7 +151,7 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -201,7 +201,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index b18468a95b..856e51e7ab 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -16,7 +16,7 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 6f4c966cdc..34f86520b0 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,17 +33,17 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 if: github.event_name == 'push' with: ref: master diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 9df5055e60..a54722e95b 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,7 +27,7 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -61,7 +61,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: path: src @@ -129,7 +129,7 @@ jobs: os: "ubuntu-24.04-arm" steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: @@ -167,7 +167,7 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 388e7475d6..c4d46a1058 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -14,7 +14,7 @@ jobs: name: Ensure Synapse config schema is valid runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -40,7 +40,7 @@ jobs: name: Ensure generated documentation is up-to-date runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1887cc0afd..58974ad7db 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -106,7 +106,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -126,7 +126,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -136,7 +136,7 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -149,7 +149,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -171,7 +171,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -207,7 +207,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -216,7 +216,7 @@ jobs: if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 @@ -234,7 +234,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -253,7 +253,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -271,7 +271,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -307,7 +307,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -326,7 +326,7 @@ jobs: if: ${{ needs.changes.outputs.golangci == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: @@ -344,7 +344,7 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -393,7 +393,7 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -414,7 +414,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -470,7 +470,7 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -533,7 +533,7 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -583,7 +583,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers @@ -630,7 +630,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - run: sudo apt-get -qq install xmlsec1 postgresql-client - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -673,7 +673,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -726,7 +726,7 @@ jobs: - changes steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -747,7 +747,7 @@ jobs: - changes steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index 85d7be7b34..f6880fef1f 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -22,7 +22,7 @@ jobs: # This field is case-sensitive. TARGET_STATUS: Needs info steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # Only clone the script file we care about, instead of the whole repo. sparse-checkout: .ci/scripts/triage_labelled_issue.sh diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 1b906f7f44..80984ceeb3 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust @@ -115,7 +115,7 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -172,7 +172,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 651b44e0fccb5aed95d79997f90e35c69e3f7af7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 14 Jul 2026 16:14:35 -0500 Subject: [PATCH 015/158] Update release script to be compatible with macOS BSD vs GNU `sed` problems: ``` sed -i '0,/^\$id: .*/s||$id: https://element-hq.github.io/synapse/schema/synapse/v1.157/synapse-config.schema.json|' schema/synapse-config.schema.yaml sed: 1: "schema/synapse-config.s ...": bad flag in substitute command: 'h' ``` --- scripts-dev/release.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index ea4fb0f142..af4a199277 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -260,7 +260,18 @@ def _prepare() -> None: schema_file = "schema/synapse-config.schema.yaml" major_minor_version = ".".join(new_version.split(".")[:2]) url = f"https://element-hq.github.io/synapse/schema/synapse/v{major_minor_version}/synapse-config.schema.json" - subprocess.check_output(["sed", "-i", f"0,/^\\$id: .*/s||$id: {url}|", schema_file]) + # Find/replace the `$id: ...` line in `schema/synapse-config.schema.yaml` with a new + # unique identifier for this release + # + # We use two `open(...)` blocks as it's easier to read/write then figure out the + # seek/truncate dance with one. + with open(schema_file) as f: + contents = f.read() + contents = re.sub( + r"^\$id: .*", f"$id: {url}", contents, count=1, flags=re.MULTILINE + ) + with open(schema_file, "w") as f: + f.write(contents) # Generate changelogs. generate_and_write_changelog(synapse_repo, current_version, new_version) From 72be9b5e1766bb93478f3b73354f19f49ea091e7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 14 Jul 2026 16:20:56 -0500 Subject: [PATCH 016/158] 1.157.0rc1 --- CHANGES.md | 49 +++++++++++++++++++++++++++++++ changelog.d/19539.bugfix | 1 - changelog.d/19539.feature | 1 - changelog.d/19663.feature | 1 - changelog.d/19802.feature | 1 - changelog.d/19808.feature | 1 - changelog.d/19826.bugfix | 1 - changelog.d/19837.misc | 1 - changelog.d/19871.misc | 1 - changelog.d/19878.misc | 1 - changelog.d/19879.misc | 1 - changelog.d/19888.misc | 1 - changelog.d/19890.misc | 1 - changelog.d/19895.removal | 1 - changelog.d/19896.misc | 1 - changelog.d/19897.misc | 1 - changelog.d/19901.bugfix | 1 - changelog.d/19902.bugfix | 2 -- changelog.d/19911.misc | 1 - changelog.d/19912.misc | 1 - changelog.d/19916.misc | 1 - changelog.d/19922.misc | 1 - changelog.d/19923.misc | 1 - changelog.d/19928.bugfix | 1 - changelog.d/19929.misc | 1 - changelog.d/19935.feature | 1 - changelog.d/19936.misc | 1 - changelog.d/19938.misc | 1 - changelog.d/19939.misc | 1 - changelog.d/19941.misc | 1 - changelog.d/19942.misc | 1 - changelog.d/19947.bugfix | 1 - changelog.d/19948.bugfix | 1 - changelog.d/19949.bugfix | 1 - debian/changelog | 6 ++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 37 files changed, 57 insertions(+), 36 deletions(-) delete mode 100644 changelog.d/19539.bugfix delete mode 100644 changelog.d/19539.feature delete mode 100644 changelog.d/19663.feature delete mode 100644 changelog.d/19802.feature delete mode 100644 changelog.d/19808.feature delete mode 100644 changelog.d/19826.bugfix delete mode 100644 changelog.d/19837.misc delete mode 100644 changelog.d/19871.misc delete mode 100644 changelog.d/19878.misc delete mode 100644 changelog.d/19879.misc delete mode 100644 changelog.d/19888.misc delete mode 100644 changelog.d/19890.misc delete mode 100644 changelog.d/19895.removal delete mode 100644 changelog.d/19896.misc delete mode 100644 changelog.d/19897.misc delete mode 100644 changelog.d/19901.bugfix delete mode 100644 changelog.d/19902.bugfix delete mode 100644 changelog.d/19911.misc delete mode 100644 changelog.d/19912.misc delete mode 100644 changelog.d/19916.misc delete mode 100644 changelog.d/19922.misc delete mode 100644 changelog.d/19923.misc delete mode 100644 changelog.d/19928.bugfix delete mode 100644 changelog.d/19929.misc delete mode 100644 changelog.d/19935.feature delete mode 100644 changelog.d/19936.misc delete mode 100644 changelog.d/19938.misc delete mode 100644 changelog.d/19939.misc delete mode 100644 changelog.d/19941.misc delete mode 100644 changelog.d/19942.misc delete mode 100644 changelog.d/19947.bugfix delete mode 100644 changelog.d/19948.bugfix delete mode 100644 changelog.d/19949.bugfix diff --git a/CHANGES.md b/CHANGES.md index 38213bd2ff..c92f9d80d9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,52 @@ +# synapse 1.157.0rc1 (2026-07-14) + +## Features + +- [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Limit how many delayed events a user may have scheduled at once. ([\#19539](https://github.com/element-hq/synapse/issues/19539)) +- Support [MSC4446](https://github.com/matrix-org/matrix-spec-proposals/pull/4446) for moving fully read markers backwards. Contributed by @SpiritCroc @ Beeper. ([\#19663](https://github.com/element-hq/synapse/issues/19663)) +- Add before and after time filters to the 'Redact events of a user' Admin API. ([\#19802](https://github.com/element-hq/synapse/issues/19802)) +- Updated experimental support for [MSC4388: Secure out-of-band channel for sign in with QR](https://github.com/matrix-org/matrix-spec-proposals/pull/4388). ([\#19808](https://github.com/element-hq/synapse/issues/19808)) +- Add an `exclude_rooms_from_presence` configuration option to stop presence being routed between users solely because they share one of the listed rooms. ([\#19935](https://github.com/element-hq/synapse/issues/19935)) + +## Bugfixes + +- [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Update error responses to match their format in the current draft of the MSC. ([\#19539](https://github.com/element-hq/synapse/issues/19539)) +- Lock Sliding Sync connections when inserting lazy members, to prevent repeated deadlocks. ([\#19826](https://github.com/element-hq/synapse/issues/19826)) +- Fix the `flag_existing_quarantined_media` background update skipping some quarantined remote media. Introduced in v1.152.0. ([\#19901](https://github.com/element-hq/synapse/issues/19901)) +- Fix a bug introduced in Synapse v1.150.0 where reactivating a deactivated and erased user did not restore their profile, breaking login, name changes, and invitations. + Contributed by @m4us1ne. ([\#19902](https://github.com/element-hq/synapse/issues/19902)) +- Fix a regression where application services that opted into ephemeral events using the legacy `de.sorunome.msc2409.push_ephemeral` registration flag stopped receiving ephemeral events (including to-device messages used for encryption). Introduced in v1.156.0. ([\#19928](https://github.com/element-hq/synapse/issues/19928)) +- Fix a bug causing device list pruning to skip some rows when the transaction gets retried. ([\#19947](https://github.com/element-hq/synapse/issues/19947)) +- Fix presence states being shown to clients forever after presence is disabled, by marking any previously only users as offline. ([\#19948](https://github.com/element-hq/synapse/issues/19948)) +- Fix `SYNAPSE_ASYNC_IO_REACTOR=1` on Python 3.14. ([\#19949](https://github.com/element-hq/synapse/issues/19949)) + +## Deprecations and Removals + +- Remove support for experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) auth delegation, in favour of the stable Matrix Authentication Service integration support. See [the upgrade notes](https://element-hq.github.io/synapse/v1.157/upgrade.html#upgrading-to-v11570). ([\#19895](https://github.com/element-hq/synapse/issues/19895)) + +## Internal Changes + +- Port the synchronous core of client event serialization to Rust. ([\#19837](https://github.com/element-hq/synapse/issues/19837), [\#19922](https://github.com/element-hq/synapse/issues/19922)) +- Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool). ([\#19871](https://github.com/element-hq/synapse/issues/19871), [\#19879](https://github.com/element-hq/synapse/issues/19879)) +- Allow Rust code to have database access via Python database connection pool. ([\#19878](https://github.com/element-hq/synapse/issues/19878)) +- Add `golangci-lint` to CI. ([\#19888](https://github.com/element-hq/synapse/issues/19888)) +- Remove wall-clock dependency of `test_redact_messages_all_rooms` test, as this caused flakiness. ([\#19890](https://github.com/element-hq/synapse/issues/19890)) +- Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint from `POST` to `GET`. ([\#19896](https://github.com/element-hq/synapse/issues/19896)) +- Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint paging to match spec conventions. ([\#19897](https://github.com/element-hq/synapse/issues/19897)) +- Fix storage type mismatches where values were bound with a type that didn't match their database column. ([\#19911](https://github.com/element-hq/synapse/issues/19911)) +- Speed up deletion of old sliding sync connections by adding an index. ([\#19912](https://github.com/element-hq/synapse/issues/19912)) +- Add note to 3PID email token request unit tests that the endpoint being tested can have an expected, artificial delay of up to 1s. ([\#19916](https://github.com/element-hq/synapse/issues/19916)) +- Add an index to `sliding_sync_connection_lazy_members` to speed up deleting old sliding sync connection positions. ([\#19923](https://github.com/element-hq/synapse/issues/19923)) +- Fix `test_lock_contention` being flaky when running against PostgreSQL by budgeting CPU time rather than wall-clock time. ([\#19929](https://github.com/element-hq/synapse/issues/19929)) +- Fix Complement test flake when restarting Synapse workers (cross-test pollution caused by nginx upstreams being temporarily unavailable). ([\#19936](https://github.com/element-hq/synapse/issues/19936)) +- Add clean deploy `FIXME` note for `TestOIDCProviderUnavailable` (problem tracked by [#19937](https://github.com/element-hq/synapse/issues/19937)). ([\#19938](https://github.com/element-hq/synapse/issues/19938)) +- Minor presence performance improvements for large servers. ([\#19939](https://github.com/element-hq/synapse/issues/19939)) +- Reduce replication traffic caused by presence. ([\#19941](https://github.com/element-hq/synapse/issues/19941)) +- Add `last_active_granularity`, `sync_online_timeout` and `idle_timeout` options to the `presence` config section to allow tuning the presence state machine timers. ([\#19942](https://github.com/element-hq/synapse/issues/19942)) + + + + # Synapse 1.156.0 (2026-07-07) No significant changes since 1.156.0rc1. diff --git a/changelog.d/19539.bugfix b/changelog.d/19539.bugfix deleted file mode 100644 index 41beb3a179..0000000000 --- a/changelog.d/19539.bugfix +++ /dev/null @@ -1 +0,0 @@ -[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Update error responses to match their format in the current draft of the MSC. diff --git a/changelog.d/19539.feature b/changelog.d/19539.feature deleted file mode 100644 index 93eb4cc1b9..0000000000 --- a/changelog.d/19539.feature +++ /dev/null @@ -1 +0,0 @@ -[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Limit how many delayed events a user may have scheduled at once. diff --git a/changelog.d/19663.feature b/changelog.d/19663.feature deleted file mode 100644 index 9f61182a90..0000000000 --- a/changelog.d/19663.feature +++ /dev/null @@ -1 +0,0 @@ -Support [MSC4446](https://github.com/matrix-org/matrix-spec-proposals/pull/4446) for moving fully read markers backwards. Contributed by @SpiritCroc @ Beeper. diff --git a/changelog.d/19802.feature b/changelog.d/19802.feature deleted file mode 100644 index 455b64ce0c..0000000000 --- a/changelog.d/19802.feature +++ /dev/null @@ -1 +0,0 @@ -Add before and after time filters to the 'Redact events of a user' Admin API. diff --git a/changelog.d/19808.feature b/changelog.d/19808.feature deleted file mode 100644 index 69eca83681..0000000000 --- a/changelog.d/19808.feature +++ /dev/null @@ -1 +0,0 @@ -Updated experimental support for [MSC4388: Secure out-of-band channel for sign in with QR](https://github.com/matrix-org/matrix-spec-proposals/pull/4388). diff --git a/changelog.d/19826.bugfix b/changelog.d/19826.bugfix deleted file mode 100644 index 771b197b20..0000000000 --- a/changelog.d/19826.bugfix +++ /dev/null @@ -1 +0,0 @@ -Lock Sliding Sync connections when inserting lazy members, to prevent repeated deadlocks. \ No newline at end of file diff --git a/changelog.d/19837.misc b/changelog.d/19837.misc deleted file mode 100644 index 64a531d4ed..0000000000 --- a/changelog.d/19837.misc +++ /dev/null @@ -1 +0,0 @@ -Port the synchronous core of client event serialization to Rust. diff --git a/changelog.d/19871.misc b/changelog.d/19871.misc deleted file mode 100644 index be10ee0540..0000000000 --- a/changelog.d/19871.misc +++ /dev/null @@ -1 +0,0 @@ -Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool). diff --git a/changelog.d/19878.misc b/changelog.d/19878.misc deleted file mode 100644 index 9ea93729c9..0000000000 --- a/changelog.d/19878.misc +++ /dev/null @@ -1 +0,0 @@ -Allow Rust code to have database access via Python database connection pool. diff --git a/changelog.d/19879.misc b/changelog.d/19879.misc deleted file mode 100644 index be10ee0540..0000000000 --- a/changelog.d/19879.misc +++ /dev/null @@ -1 +0,0 @@ -Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool). diff --git a/changelog.d/19888.misc b/changelog.d/19888.misc deleted file mode 100644 index 0506b21d42..0000000000 --- a/changelog.d/19888.misc +++ /dev/null @@ -1 +0,0 @@ -Add `golangci-lint` to CI. \ No newline at end of file diff --git a/changelog.d/19890.misc b/changelog.d/19890.misc deleted file mode 100644 index b74dbc3086..0000000000 --- a/changelog.d/19890.misc +++ /dev/null @@ -1 +0,0 @@ -Remove wall-clock dependency of `test_redact_messages_all_rooms` test, as this caused flakiness. \ No newline at end of file diff --git a/changelog.d/19895.removal b/changelog.d/19895.removal deleted file mode 100644 index 1b39ffb139..0000000000 --- a/changelog.d/19895.removal +++ /dev/null @@ -1 +0,0 @@ -Remove support for experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) auth delegation, in favour of the stable Matrix Authentication Service integration support. See [the upgrade notes](https://element-hq.github.io/synapse/v1.157/upgrade.html#upgrading-to-v11570). diff --git a/changelog.d/19896.misc b/changelog.d/19896.misc deleted file mode 100644 index 2c6a8740ef..0000000000 --- a/changelog.d/19896.misc +++ /dev/null @@ -1 +0,0 @@ -Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint from `POST` to `GET`. diff --git a/changelog.d/19897.misc b/changelog.d/19897.misc deleted file mode 100644 index f1163dc1f1..0000000000 --- a/changelog.d/19897.misc +++ /dev/null @@ -1 +0,0 @@ -Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint paging to match spec conventions. diff --git a/changelog.d/19901.bugfix b/changelog.d/19901.bugfix deleted file mode 100644 index e31c665c18..0000000000 --- a/changelog.d/19901.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the `flag_existing_quarantined_media` background update skipping some quarantined remote media. Introduced in v1.152.0. diff --git a/changelog.d/19902.bugfix b/changelog.d/19902.bugfix deleted file mode 100644 index fb5f75c295..0000000000 --- a/changelog.d/19902.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug introduced in Synapse v1.150.0 where reactivating a deactivated and erased user did not restore their profile, breaking login, name changes, and invitations. -Contributed by @m4us1ne. diff --git a/changelog.d/19911.misc b/changelog.d/19911.misc deleted file mode 100644 index 11331455aa..0000000000 --- a/changelog.d/19911.misc +++ /dev/null @@ -1 +0,0 @@ -Fix storage type mismatches where values were bound with a type that didn't match their database column. diff --git a/changelog.d/19912.misc b/changelog.d/19912.misc deleted file mode 100644 index e9491ab335..0000000000 --- a/changelog.d/19912.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up deletion of old sliding sync connections by adding an index. diff --git a/changelog.d/19916.misc b/changelog.d/19916.misc deleted file mode 100644 index aeec5ae412..0000000000 --- a/changelog.d/19916.misc +++ /dev/null @@ -1 +0,0 @@ -Add note to 3PID email token request unit tests that the endpoint being tested can have an expected, artificial delay of up to 1s. \ No newline at end of file diff --git a/changelog.d/19922.misc b/changelog.d/19922.misc deleted file mode 100644 index 64a531d4ed..0000000000 --- a/changelog.d/19922.misc +++ /dev/null @@ -1 +0,0 @@ -Port the synchronous core of client event serialization to Rust. diff --git a/changelog.d/19923.misc b/changelog.d/19923.misc deleted file mode 100644 index 4a52d1cca2..0000000000 --- a/changelog.d/19923.misc +++ /dev/null @@ -1 +0,0 @@ -Add an index to `sliding_sync_connection_lazy_members` to speed up deleting old sliding sync connection positions. diff --git a/changelog.d/19928.bugfix b/changelog.d/19928.bugfix deleted file mode 100644 index d69e4febbe..0000000000 --- a/changelog.d/19928.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a regression where application services that opted into ephemeral events using the legacy `de.sorunome.msc2409.push_ephemeral` registration flag stopped receiving ephemeral events (including to-device messages used for encryption). Introduced in v1.156.0. diff --git a/changelog.d/19929.misc b/changelog.d/19929.misc deleted file mode 100644 index 4778f9fa32..0000000000 --- a/changelog.d/19929.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `test_lock_contention` being flaky when running against PostgreSQL by budgeting CPU time rather than wall-clock time. diff --git a/changelog.d/19935.feature b/changelog.d/19935.feature deleted file mode 100644 index cd5fa804b8..0000000000 --- a/changelog.d/19935.feature +++ /dev/null @@ -1 +0,0 @@ -Add an `exclude_rooms_from_presence` configuration option to stop presence being routed between users solely because they share one of the listed rooms. diff --git a/changelog.d/19936.misc b/changelog.d/19936.misc deleted file mode 100644 index 5c1f7eef52..0000000000 --- a/changelog.d/19936.misc +++ /dev/null @@ -1 +0,0 @@ -Fix Complement test flake when restarting Synapse workers (cross-test pollution caused by nginx upstreams being temporarily unavailable). diff --git a/changelog.d/19938.misc b/changelog.d/19938.misc deleted file mode 100644 index d02c9bf3b7..0000000000 --- a/changelog.d/19938.misc +++ /dev/null @@ -1 +0,0 @@ -Add clean deploy `FIXME` note for `TestOIDCProviderUnavailable` (problem tracked by [#19937](https://github.com/element-hq/synapse/issues/19937)). diff --git a/changelog.d/19939.misc b/changelog.d/19939.misc deleted file mode 100644 index 59d480c98d..0000000000 --- a/changelog.d/19939.misc +++ /dev/null @@ -1 +0,0 @@ -Minor presence performance improvements for large servers. diff --git a/changelog.d/19941.misc b/changelog.d/19941.misc deleted file mode 100644 index e928f68bc5..0000000000 --- a/changelog.d/19941.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce replication traffic caused by presence. diff --git a/changelog.d/19942.misc b/changelog.d/19942.misc deleted file mode 100644 index bdd95c3c0f..0000000000 --- a/changelog.d/19942.misc +++ /dev/null @@ -1 +0,0 @@ -Add `last_active_granularity`, `sync_online_timeout` and `idle_timeout` options to the `presence` config section to allow tuning the presence state machine timers. diff --git a/changelog.d/19947.bugfix b/changelog.d/19947.bugfix deleted file mode 100644 index c16d6deea0..0000000000 --- a/changelog.d/19947.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug causing device list pruning to skip some rows when the transaction gets retried. \ No newline at end of file diff --git a/changelog.d/19948.bugfix b/changelog.d/19948.bugfix deleted file mode 100644 index 5cd34d1abf..0000000000 --- a/changelog.d/19948.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix presence states being shown to clients forever after presence is disabled, by marking any previously only users as offline. diff --git a/changelog.d/19949.bugfix b/changelog.d/19949.bugfix deleted file mode 100644 index 24ddd4dc4f..0000000000 --- a/changelog.d/19949.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `SYNAPSE_ASYNC_IO_REACTOR=1` on Python 3.14. diff --git a/debian/changelog b/debian/changelog index c1522f068d..8c0e1ccfe8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.157.0~rc1) stable; urgency=medium + + * New synapse release 1.157.0rc1. + + -- Synapse Packaging team Tue, 14 Jul 2026 21:18:16 +0000 + matrix-synapse-py3 (1.156.0) stable; urgency=medium * New Synapse release 1.156.0. diff --git a/pyproject.toml b/pyproject.toml index 6023740d6c..f30ac49847 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.156.0" +version = "1.157.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 62e09377d4..ea5bd7c7af 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.156/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.157/synapse-config.schema.json type: object properties: modules: From f8c913c1c0230af01df4c9b16d9193f7d7a125e3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 14 Jul 2026 16:32:17 -0500 Subject: [PATCH 017/158] Call out upgrade notes --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index c92f9d80d9..d993068549 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # synapse 1.157.0rc1 (2026-07-14) +Please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11570) as this release removes support for the deprecated MSC3861 Auth Delegation (`experimental_features.msc3861`). + ## Features - [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Limit how many delayed events a user may have scheduled at once. ([\#19539](https://github.com/element-hq/synapse/issues/19539)) From 11b48cdbada0243628ef966114d3b118ff658a52 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 14 Jul 2026 16:37:50 -0500 Subject: [PATCH 018/158] Fix title case --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d993068549..8beacb324c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# synapse 1.157.0rc1 (2026-07-14) +# Synapse 1.157.0rc1 (2026-07-14) Please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11570) as this release removes support for the deprecated MSC3861 Auth Delegation (`experimental_features.msc3861`). From 243983e193cc7c5955f6942877a4421ab0596989 Mon Sep 17 00:00:00 2001 From: jakobrss Date: Wed, 15 Jul 2026 11:18:55 +0000 Subject: [PATCH 019/158] Clarified documentation for "List accounts" `guests` parameter (#19963) --- changelog.d/19963.doc | 1 + docs/admin_api/user_admin_api.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19963.doc diff --git a/changelog.d/19963.doc b/changelog.d/19963.doc new file mode 100644 index 0000000000..de11804be5 --- /dev/null +++ b/changelog.d/19963.doc @@ -0,0 +1 @@ +Clarify the usage of the `guests` parameter when using the List user API with the Matrix Authentication Service integration enabled. diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 44d0985697..ca82ccbecd 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -227,7 +227,8 @@ The following parameters should be set in the URL: - `name` - Is optional and filters to only return users with user ID localparts **or** displaynames that contain this value. - `guests` - string representing a bool - Is optional and if `false` will **exclude** guest users. - Defaults to `true` to include guest users. This parameter is not supported when Matrix Authentication Service integration is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) + Defaults to `true` to include guest users. Setting this option to `true` is not supported when Matrix Authentication + Service integration is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) - `admins` - Optional flag to filter admins. If `true`, only admins are queried. If `false`, admins are excluded from the query. When the flag is absent (the default), **both** admins and non-admins are included in the search results. - `deactivated` - string representing a bool - Is optional and if `true` will **include** deactivated users. From ff4e9ec7a7bd755ab2dca67103ff6a591e505973 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:24:15 +0000 Subject: [PATCH 020/158] Bump the minor-and-patches group with 2 updates (#19953) Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 4 ++-- .github/workflows/fix_lint.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8ecf8fb723..5d9ab4b8b4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -54,7 +54,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tailscale - uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2 + uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} audience: ${{ secrets.TS_AUDIENCE }} @@ -151,7 +151,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tailscale - uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2 + uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} audience: ${{ secrets.TS_AUDIENCE }} diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 88decb33d9..aaf6b2ba65 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -49,6 +49,6 @@ jobs: - run: cargo fmt continue-on-error: true - - uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0 + - uses: stefanzweifel/git-auto-commit-action@4a55954c782fc1ea30b9056cd3e7a2b40ca8887d # v7.2.0 with: commit_message: "Attempt to fix linting" From 0512511f87c4436bba63b54e95ee330104cb91cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:25:46 +0000 Subject: [PATCH 021/158] Bump anyhow from 1.0.102 to 1.0.103 in the patches group (#19952) Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ca2265aca..25ceb4d0c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arc-swap" From e1420becdd1560deeef7aa3d834f3a11cacd182e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 15 Jul 2026 09:56:19 -0500 Subject: [PATCH 022/158] Linkify 'Redact events of a user' docs --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8beacb324c..92f6a87a77 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ Please check [the relevant section in the upgrade notes](https://github.com/elem - [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Limit how many delayed events a user may have scheduled at once. ([\#19539](https://github.com/element-hq/synapse/issues/19539)) - Support [MSC4446](https://github.com/matrix-org/matrix-spec-proposals/pull/4446) for moving fully read markers backwards. Contributed by @SpiritCroc @ Beeper. ([\#19663](https://github.com/element-hq/synapse/issues/19663)) -- Add before and after time filters to the 'Redact events of a user' Admin API. ([\#19802](https://github.com/element-hq/synapse/issues/19802)) +- Add before and after time filters to the ['Redact events of a user'](https://element-hq.github.io/synapse/v1.157/admin_api/user_admin_api.html#redact-events-of-a-user) Admin API. ([\#19802](https://github.com/element-hq/synapse/issues/19802)) - Updated experimental support for [MSC4388: Secure out-of-band channel for sign in with QR](https://github.com/matrix-org/matrix-spec-proposals/pull/4388). ([\#19808](https://github.com/element-hq/synapse/issues/19808)) - Add an `exclude_rooms_from_presence` configuration option to stop presence being routed between users solely because they share one of the listed rooms. ([\#19935](https://github.com/element-hq/synapse/issues/19935)) From 30088f38c07683a581bbcb8ad6fd8b7ba89767fc Mon Sep 17 00:00:00 2001 From: catfromplan9 <104175360+catfromplan9@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:37:09 +0000 Subject: [PATCH 023/158] Implement support for generating animated thumbnails in the media thumbnailer (#18831) Signed-off-by: cat --- changelog.d/18831.feature | 1 + .../federation/transport/server/federation.py | 16 +- synapse/media/media_repository.py | 209 +++++++++++------- synapse/media/thumbnailer.py | 120 ++++++++-- synapse/rest/client/media.py | 57 +++-- synapse/rest/media/thumbnail_resource.py | 60 +++-- tests/media/test_media_storage.py | 70 +++++- tests/rest/client/test_media.py | 151 +++++++++++++ 8 files changed, 550 insertions(+), 134 deletions(-) create mode 100644 changelog.d/18831.feature diff --git a/changelog.d/18831.feature b/changelog.d/18831.feature new file mode 100644 index 0000000000..022105c0b5 --- /dev/null +++ b/changelog.d/18831.feature @@ -0,0 +1 @@ +Add animation support to the media thumbnailer, gated behind the `animated` query parameter on the thumbnail endpoint (defaults to off). diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index d783e6da51..8a8914bf4f 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -37,6 +37,7 @@ from synapse.federation.transport.server._base import ( BaseFederationServlet, ) from synapse.http.servlet import ( + parse_boolean, parse_boolean_from_args, parse_integer, parse_integer_from_args, @@ -46,7 +47,7 @@ from synapse.http.servlet import ( ) from synapse.http.site import SynapseRequest from synapse.media._base import DEFAULT_MAX_TIMEOUT_MS, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ANIMATED_THUMBNAIL_TYPE, ThumbnailProvider from synapse.types import JsonDict from synapse.util import SYNAPSE_VERSION from synapse.util.ratelimitutils import FederationRateLimiter @@ -875,8 +876,9 @@ class FederationMediaThumbnailServlet(BaseFederationServerServlet): width = parse_integer(request, "width", required=True) height = parse_integer(request, "height", required=True) method = parse_string(request, "method", "scale") + animated = parse_boolean(request, "animated", default=False) # TODO Parse the Accept header to get an prioritised list of thumbnail types. - m_type = "image/png" + m_type = ANIMATED_THUMBNAIL_TYPE if animated else "image/png" max_timeout_ms = parse_integer( request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) @@ -884,7 +886,15 @@ class FederationMediaThumbnailServlet(BaseFederationServerServlet): if self.dynamic_thumbnails: await self.thumbnail_provider.select_or_generate_local_thumbnail( - request, media_id, width, height, method, m_type, max_timeout_ms, True + request, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + True, + animated=animated, ) else: await self.thumbnail_provider.respond_local_thumbnail( diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 3344e4c7be..bbd31976cb 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -68,7 +68,11 @@ from synapse.media.storage_provider import ( FileStorageProviderBackend, StorageProviderWrapper, ) -from synapse.media.thumbnailer import Thumbnailer, ThumbnailError +from synapse.media.thumbnailer import ( + ANIMATED_THUMBNAIL_TYPE, + Thumbnailer, + ThumbnailError, +) from synapse.media.url_previewer import UrlPreviewer from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.types import UserID @@ -1127,6 +1131,7 @@ class MediaRepository: t_height: int, t_method: str, t_type: str, + animated: bool = False, ) -> BytesIO | None: m_width = thumbnailer.width m_height = thumbnailer.height @@ -1144,12 +1149,12 @@ class MediaRepository: m_width, m_height = thumbnailer.transpose() if t_method == "crop": - return thumbnailer.crop(t_width, t_height, t_type) + return thumbnailer.crop(t_width, t_height, t_type, animated=animated) elif t_method == "scale": t_width, t_height = thumbnailer.aspect(t_width, t_height) t_width = min(m_width, t_width) t_height = min(m_height, t_height) - return thumbnailer.scale(t_width, t_height, t_type) + return thumbnailer.scale(t_width, t_height, t_type, animated=animated) return None @@ -1161,6 +1166,7 @@ class MediaRepository: t_method: str, t_type: str, url_cache: bool, + animated: bool = False, ) -> tuple[str, FileInfo] | None: async with self.media_storage.ensure_media_is_in_local_cache( FileInfo(None, media_id, url_cache=url_cache) @@ -1186,6 +1192,7 @@ class MediaRepository: t_height, t_method, t_type, + animated, ) if t_byte_source: @@ -1199,7 +1206,7 @@ class MediaRepository: height=t_height, method=t_method, type=t_type, - length=t_byte_source.tell(), + length=t_byte_source.getbuffer().nbytes, ), ) @@ -1236,6 +1243,7 @@ class MediaRepository: t_height: int, t_method: str, t_type: str, + animated: bool = False, ) -> str | None: async with self.media_storage.ensure_media_is_in_local_cache( FileInfo(server_name, file_id) @@ -1262,6 +1270,7 @@ class MediaRepository: t_height, t_method, t_type, + animated, ) if t_byte_source: @@ -1274,7 +1283,7 @@ class MediaRepository: height=t_height, method=t_method, type=t_type, - length=t_byte_source.tell(), + length=t_byte_source.getbuffer().nbytes, ), ) @@ -1389,6 +1398,12 @@ class MediaRepository: ) # Now we generate the thumbnails for each dimension, store it + # + # For animated source images we also generate and cache an + # animated WebP thumbnail per (size, method), served only when a + # client requests `?animated=true`. These are deduplicated since + # the animated thumbnail is always WebP regardless of `t_type`. + animated_done: set[tuple[int, int, str]] = set() for (t_width, t_height, t_type), t_method in thumbnails.items(): # Generate the thumbnail if t_method == "crop": @@ -1411,81 +1426,127 @@ class MediaRepository: logger.error("Unrecognized method: %r", t_method) continue - if not t_byte_source: - continue + if t_byte_source: + await self._store_thumbnail( + server_name, + media_id, + file_id, + url_cache, + t_width, + t_height, + t_method, + t_type, + t_byte_source, + ) - file_info = FileInfo( - server_name=server_name, - file_id=file_id, - url_cache=url_cache, - thumbnail=ThumbnailInfo( - width=t_width, - height=t_height, - method=t_method, - type=t_type, - length=t_byte_source.tell(), - ), - ) - - async with self.media_storage.store_into_file(file_info) as ( - f, - fname, + if ( + thumbnailer.is_animated + and (t_width, t_height, t_method) not in animated_done ): - try: - await self.media_storage.write_to_file(t_byte_source, f) - finally: - t_byte_source.close() - - # We flush and close the file to ensure that the bytes have - # been written before getting the size. - f.flush() - f.close() - - t_len = os.path.getsize(fname) - - # Write to database - if server_name: - # Multiple remote media download requests can race (when - # using multiple media repos), so this may throw a violation - # constraint exception. If it does we'll delete the newly - # generated thumbnail from disk (as we're in the ctx - # manager). - # - # However: we've already called `finish()` so we may have - # also written to the storage providers. This is preferable - # to the alternative where we call `finish()` *after* this, - # where we could end up having an entry in the DB but fail - # to write the files to the storage providers. - try: - await self.store.store_remote_media_thumbnail( - server_name, - media_id, - file_id, - t_width, - t_height, - t_type, - t_method, - t_len, - ) - except Exception as e: - thumbnail_exists = ( - await self.store.get_remote_media_thumbnail( - server_name, - media_id, - t_width, - t_height, - t_type, - ) - ) - if not thumbnail_exists: - raise e - else: - await self.store.store_local_thumbnail( - media_id, t_width, t_height, t_type, t_method, t_len + animated_done.add((t_width, t_height, t_method)) + a_byte_source = await defer_to_thread( + self.hs.get_reactor(), + thumbnailer.crop + if t_method == "crop" + else thumbnailer.scale, + t_width, + t_height, + ANIMATED_THUMBNAIL_TYPE, + True, + ) + if a_byte_source: + await self._store_thumbnail( + server_name, + media_id, + file_id, + url_cache, + t_width, + t_height, + t_method, + ANIMATED_THUMBNAIL_TYPE, + a_byte_source, ) return {"width": m_width, "height": m_height} + async def _store_thumbnail( + self, + server_name: str | None, + media_id: str, + file_id: str, + url_cache: bool, + t_width: int, + t_height: int, + t_method: str, + t_type: str, + t_byte_source: BytesIO, + ) -> None: + """Store a generated thumbnail to the configured storage and database.""" + file_info = FileInfo( + server_name=server_name, + file_id=file_id, + url_cache=url_cache, + thumbnail=ThumbnailInfo( + width=t_width, + height=t_height, + method=t_method, + type=t_type, + length=t_byte_source.getbuffer().nbytes, + ), + ) + + async with self.media_storage.store_into_file(file_info) as (f, fname): + try: + await self.media_storage.write_to_file(t_byte_source, f) + finally: + t_byte_source.close() + + # We flush and close the file to ensure that the bytes have + # been written before getting the size. + f.flush() + f.close() + + t_len = os.path.getsize(fname) + + # Write to database + if server_name: + # Multiple remote media download requests can race (when + # using multiple media repos), so this may throw a violation + # constraint exception. If it does we'll delete the newly + # generated thumbnail from disk (as we're in the ctx + # manager). + # + # However: we've already called `finish()` so we may have + # also written to the storage providers. This is preferable + # to the alternative where we call `finish()` *after* this, + # where we could end up having an entry in the DB but fail + # to write the files to the storage providers. + try: + await self.store.store_remote_media_thumbnail( + server_name, + media_id, + file_id, + t_width, + t_height, + t_type, + t_method, + t_len, + ) + except Exception as e: + thumbnail_exists = await self.store.get_remote_media_thumbnail( + server_name, + media_id, + t_width, + t_height, + t_type, + ) + if not thumbnail_exists: + raise e + else: + await self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + async def _apply_media_retention_rules(self) -> None: """ Purge old local and remote media according to the media retention rules diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index cb86a0ff66..d60ca16f26 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -20,11 +20,12 @@ # # import logging +from collections.abc import Callable from io import BytesIO from types import TracebackType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any -from PIL import Image +from PIL import Image, ImageSequence from synapse.api.errors import Codes, NotFoundError, SynapseError, cs_error from synapse.config.repository import THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP @@ -49,6 +50,11 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# Content type used when generating animated thumbnails. The spec recommends servers +# prefer WebP when supporting animation. See +# https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1mediathumbnailservernamemediaid +ANIMATED_THUMBNAIL_TYPE = "image/webp" + EXIF_ORIENTATION_TAG = 0x0112 EXIF_TRANSPOSE_MAPPINGS = { 2: Image.FLIP_LEFT_RIGHT, @@ -66,7 +72,7 @@ class ThumbnailError(Exception): class Thumbnailer: - FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"} + FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG", "image/webp": "WEBP"} # Which image formats we allow Pillow to open. # This should intentionally be kept restrictive, because the decoder of any @@ -143,33 +149,72 @@ class Thumbnailer: else: return max((max_height * self.width) // self.height, 1), max_height - def _resize(self, width: int, height: int) -> Image.Image: + def _resize_image(self, image: Image.Image, width: int, height: int) -> Image.Image: # 1-bit or 8-bit color palette images need converting to RGB # otherwise they will be scaled using nearest neighbour which # looks awful. # # If the image has transparency, use RGBA instead. - if self.image.mode in ["1", "L", "P"]: - if self.image.info.get("transparency", None) is not None: - with self.image: - self.image = self.image.convert("RGBA") + if image.mode in ["1", "L", "P"]: + if image.info.get("transparency", None) is not None: + converted = image.convert("RGBA") else: - with self.image: - self.image = self.image.convert("RGB") - return self.image.resize((width, height), Image.LANCZOS) + converted = image.convert("RGB") + else: + converted = image + return converted.resize((width, height), Image.LANCZOS) + + @property + def is_animated(self) -> bool: + return getattr(self.image, "is_animated", False) + + def _encode_animated_from( + self, transform: Callable[[Image.Image], Image.Image] + ) -> BytesIO: + """Apply `transform` to every frame of the source image and encode the + result as an animated thumbnail.""" + frames = [] + durations = [] + loop = self.image.info.get("loop", 0) + for frame in ImageSequence.Iterator(self.image): + # Copy the frame to avoid referencing the original image memory. + f = frame.copy() + if f.mode != "RGBA": + f = f.convert("RGBA") + frames.append(transform(f)) + # A duration of 0 is valid (interpretation is implementation-defined, + # see RFC 9649 section 2.7.1.1), so only fall back when unset. The + # 100ms default matches libwebp's animation tools. + duration = frame.info.get("duration") + if duration is None: + duration = self.image.info.get("duration", 100) + durations.append(duration) + return self._encode_animated(frames, durations, loop) @trace - def scale(self, width: int, height: int, output_type: str) -> BytesIO: + def scale( + self, width: int, height: int, output_type: str, animated: bool = False + ) -> BytesIO: """Rescales the image to the given dimensions. + If `animated` is set and the source image is animated, the animation is + preserved. Otherwise a static thumbnail of the first frame is produced. + Returns: The bytes of the encoded image ready to be written to disk """ - with self._resize(width, height) as scaled: + if animated and self.is_animated: + return self._encode_animated_from( + lambda f: self._resize_image(f, width, height) + ) + + with self._resize_image(self.image, width, height) as scaled: return self._encode_image(scaled, output_type) @trace - def crop(self, width: int, height: int, output_type: str) -> BytesIO: + def crop( + self, width: int, height: int, output_type: str, animated: bool = False + ) -> BytesIO: """Rescales and crops the image to the given dimensions preserving aspect:: (w_in / h_in) = (w_scaled / h_scaled) @@ -196,7 +241,14 @@ class Thumbnailer: crop_right = width + crop_left crop = (crop_left, 0, crop_right, height) - with self._resize(scaled_width, scaled_height) as scaled_image: + if animated and self.is_animated: + return self._encode_animated_from( + lambda f: self._resize_image(f, scaled_width, scaled_height).crop(crop) + ) + + with self._resize_image( + self.image, scaled_width, scaled_height + ) as scaled_image: with scaled_image.crop(crop) as cropped: return self._encode_image(cropped, output_type) @@ -206,6 +258,35 @@ class Thumbnailer: if fmt == "JPEG" or fmt == "PNG" and output_image.mode == "CMYK": output_image = output_image.convert("RGB") output_image.save(output_bytes_io, fmt, quality=80) + output_bytes_io.seek(0) + return output_bytes_io + + def _encode_animated( + self, + frames: list[Image.Image], + durations: list[int], + loop: int, + ) -> BytesIO: + """ + Encode a list of RGBA frames into an animated WebP, preserving per-frame + durations and the loop count. + """ + output_bytes_io = BytesIO() + if not frames: + raise ThumbnailError("No frames to encode for animated thumbnail") + + save_kwargs: dict[str, Any] = { + "format": "WEBP", + "save_all": True, + "append_images": frames[1:], + "loop": loop, + "duration": durations, + "minimize_size": True, + "quality": 80, + } + + frames[0].save(output_bytes_io, **save_kwargs) + output_bytes_io.seek(0) return output_bytes_io def close(self) -> None: @@ -318,6 +399,7 @@ class ThumbnailProvider: max_timeout_ms: int, for_federation: bool, allow_authenticated: bool = True, + animated: bool = False, ) -> None: media_info = await self.media_repo.get_local_media_info( request, media_id, max_timeout_ms @@ -379,6 +461,7 @@ class ThumbnailProvider: desired_method, desired_type, url_cache=bool(media_info.url_cache), + animated=animated, ) if thumbnail_result: @@ -413,6 +496,7 @@ class ThumbnailProvider: ip_address: str, use_federation: bool, allow_authenticated: bool = True, + animated: bool = False, ) -> None: media_info = await self.media_repo.get_remote_media_info( server_name, @@ -474,6 +558,7 @@ class ThumbnailProvider: desired_height, desired_method, desired_type, + animated=animated, ) if file_path: @@ -633,6 +718,9 @@ class ThumbnailProvider: # width/height/method so we can just call the "generate exact" # methods. + # A stored animated thumbnail is always of the animated type, so + # regenerate it as animated. + regen_animated = file_info.thumbnail.type == ANIMATED_THUMBNAIL_TYPE if server_name: await self.media_repo.generate_remote_exact_thumbnail( server_name, @@ -642,6 +730,7 @@ class ThumbnailProvider: t_height=file_info.thumbnail.height, t_method=file_info.thumbnail.method, t_type=file_info.thumbnail.type, + animated=regen_animated, ) else: await self.media_repo.generate_local_exact_thumbnail( @@ -651,6 +740,7 @@ class ThumbnailProvider: t_method=file_info.thumbnail.method, t_type=file_info.thumbnail.type, url_cache=url_cache, + animated=regen_animated, ) responder = await self.media_storage.fetch_media(file_info) diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index c740659cdd..c2e30b687b 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -36,7 +36,12 @@ from synapse.http.server import ( set_corp_headers, set_cors_headers, ) -from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_boolean, + parse_integer, + parse_string, +) from synapse.http.site import SynapseRequest from synapse.media._base import ( DEFAULT_MAX_TIMEOUT_MS, @@ -45,7 +50,7 @@ from synapse.media._base import ( ) from synapse.media.media_repository import MediaRepository from synapse.media.media_storage import MediaStorage -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ANIMATED_THUMBNAIL_TYPE, ThumbnailProvider from synapse.server import HomeServer from synapse.util.stringutils import parse_and_validate_server_name @@ -163,8 +168,9 @@ class ThumbnailResource(RestServlet): width = parse_integer(request, "width", required=True) height = parse_integer(request, "height", required=True) method = parse_string(request, "method", "scale") + animated = parse_boolean(request, "animated", default=False) # TODO Parse the Accept header to get an prioritised list of thumbnail types. - m_type = "image/png" + m_type = ANIMATED_THUMBNAIL_TYPE if animated else "image/png" max_timeout_ms = parse_integer( request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) @@ -181,6 +187,7 @@ class ThumbnailResource(RestServlet): m_type, max_timeout_ms, False, + animated=animated, ) else: await self.thumbnailer.respond_local_thumbnail( @@ -204,23 +211,33 @@ class ThumbnailResource(RestServlet): return ip_address = request.getClientAddress().host - remote_resp_function = ( - self.thumbnailer.select_or_generate_remote_thumbnail - if self.dynamic_thumbnails - else self.thumbnailer.respond_remote_thumbnail - ) - await remote_resp_function( - request, - server_name, - media_id, - width, - height, - method, - m_type, - max_timeout_ms, - ip_address, - True, - ) + if self.dynamic_thumbnails: + await self.thumbnailer.select_or_generate_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + True, + animated=animated, + ) + else: + await self.thumbnailer.respond_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + True, + ) self.media_repo.mark_recently_accessed(server_name, media_id) diff --git a/synapse/rest/media/thumbnail_resource.py b/synapse/rest/media/thumbnail_resource.py index 536fea4c32..35afb7ab54 100644 --- a/synapse/rest/media/thumbnail_resource.py +++ b/synapse/rest/media/thumbnail_resource.py @@ -25,7 +25,12 @@ import re from typing import TYPE_CHECKING from synapse.http.server import set_corp_headers, set_cors_headers -from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_boolean, + parse_integer, + parse_string, +) from synapse.http.site import SynapseRequest from synapse.media._base import ( DEFAULT_MAX_TIMEOUT_MS, @@ -33,7 +38,7 @@ from synapse.media._base import ( respond_404, ) from synapse.media.media_storage import MediaStorage -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ANIMATED_THUMBNAIL_TYPE, ThumbnailProvider from synapse.util.stringutils import parse_and_validate_server_name if TYPE_CHECKING: @@ -78,8 +83,9 @@ class ThumbnailResource(RestServlet): width = parse_integer(request, "width", required=True) height = parse_integer(request, "height", required=True) method = parse_string(request, "method", "scale") + animated = parse_boolean(request, "animated", default=False) # TODO Parse the Accept header to get an prioritised list of thumbnail types. - m_type = "image/png" + m_type = ANIMATED_THUMBNAIL_TYPE if animated else "image/png" max_timeout_ms = parse_integer( request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) @@ -97,6 +103,7 @@ class ThumbnailResource(RestServlet): max_timeout_ms, False, allow_authenticated=False, + animated=animated, ) else: await self.thumbnail_provider.respond_local_thumbnail( @@ -121,22 +128,33 @@ class ThumbnailResource(RestServlet): return ip_address = request.getClientAddress().host - remote_resp_function = ( - self.thumbnail_provider.select_or_generate_remote_thumbnail - if self.dynamic_thumbnails - else self.thumbnail_provider.respond_remote_thumbnail - ) - await remote_resp_function( - request, - server_name, - media_id, - width, - height, - method, - m_type, - max_timeout_ms, - ip_address, - use_federation=False, - allow_authenticated=False, - ) + if self.dynamic_thumbnails: + await self.thumbnail_provider.select_or_generate_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + use_federation=False, + allow_authenticated=False, + animated=animated, + ) + else: + await self.thumbnail_provider.respond_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + use_federation=False, + allow_authenticated=False, + ) self.media_repo.mark_recently_accessed(server_name, media_id) diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 855a623ec0..54d1262054 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -52,7 +52,11 @@ from synapse.media.storage_provider import ( FileStorageProviderBackend, StorageProviderWrapper, ) -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ( + ANIMATED_THUMBNAIL_TYPE, + Thumbnailer, + ThumbnailProvider, +) from synapse.module_api import ModuleApi from synapse.module_api.callbacks.spamchecker_callbacks import load_legacy_spam_checkers from synapse.rest import admin @@ -1407,3 +1411,67 @@ class MediaRepoSizeModuleCallbackTestCase(unittest.HomeserverTestCase): self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=413) assert self.last_user_id == self.user assert self.last_size == len(SMALL_PNG) + + +def _make_animated_gif() -> bytes: + """Build a small two-frame animated GIF.""" + frames = [Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255))] + out = BytesIO() + frames[0].save( + out, + format="GIF", + save_all=True, + append_images=frames[1:], + duration=100, + loop=0, + ) + return out.getvalue() + + +class ThumbnailerAnimatedTestCase(unittest.TestCase): + """Tests that the thumbnailer only animates when explicitly asked to.""" + + def setUp(self) -> None: + super().setUp() + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) + + self.gif_path = os.path.join(self.tempdir, "animated.gif") + with open(self.gif_path, "wb") as f: + f.write(_make_animated_gif()) + + self.png_path = os.path.join(self.tempdir, "static.png") + with open(self.png_path, "wb") as f: + f.write(SMALL_PNG) + + def test_scale_static_by_default(self) -> None: + """An animated source produces a static thumbnail unless animated=True.""" + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.scale(32, 32, "image/png") + result = Image.open(out) + self.assertFalse(getattr(result, "is_animated", False)) + + def test_scale_animated_when_requested(self) -> None: + """An animated source produces an animated thumbnail when animated=True.""" + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.scale(32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True) + result = Image.open(out) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(result.n_frames, 2) + + def test_crop_animated_when_requested(self) -> None: + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.crop(32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True) + result = Image.open(out) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(result.size, (32, 32)) + + def test_static_source_never_animates(self) -> None: + """A non-animated source stays static even when animated=True.""" + with Thumbnailer(self.png_path) as thumbnailer: + self.assertFalse(thumbnailer.is_animated) + out = thumbnailer.scale(1, 1, ANIMATED_THUMBNAIL_TYPE, animated=True) + result = Image.open(out) + self.assertFalse(getattr(result, "is_animated", False)) diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 3409581c5e..8757e27d48 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -29,6 +29,7 @@ from unittest.mock import MagicMock, Mock, patch from urllib import parse from urllib.parse import quote, urlencode +from matrix_common.types.mxc_uri import MXCUri from parameterized import parameterized, parameterized_class from PIL import Image as Image @@ -3208,3 +3209,153 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): ) self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 500) self.assertEqual(self.last_media_upload_limit_exceeded["attempted_bytes"], 800) + + +class AnimatedThumbnailTestCase(unittest.HomeserverTestCase): + """End-to-end tests for the `animated` query parameter on the local + thumbnail endpoint.""" + + servlets = [ + media.register_servlets, + login.register_servlets, + admin.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.storage_path = self.mktemp() + self.media_store_path = self.mktemp() + os.mkdir(self.storage_path) + os.mkdir(self.media_store_path) + + config = self.default_config() + config["media_store_path"] = self.media_store_path + config["media_storage_providers"] = [ + { + "module": "synapse.media.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + ] + + return self.setup_test_homeserver(config=config) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.repo = hs.get_media_repository() + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + + self.content_uri = self._upload(self._make_animated_gif(), "image/gif") + + def _make_animated_gif(self) -> bytes: + frames = [ + Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255)) + ] + out = io.BytesIO() + frames[0].save( + out, + format="GIF", + save_all=True, + append_images=frames[1:], + duration=100, + loop=0, + ) + return out.getvalue() + + def _upload(self, data: bytes, content_type: str) -> MXCUri: + return self.get_success( + self.repo.create_or_update_content( + content_type, + "test", + io.BytesIO(data), + len(data), + UserID.from_string(self.user), + ) + ) + + def _thumbnail(self, content_uri: MXCUri, animated: str | None) -> FakeChannel: + params = "?width=32&height=32&method=scale" + if animated is not None: + params += f"&animated={animated}" + channel = self.make_request( + "GET", + f"/_matrix/client/v1/media/thumbnail/{content_uri.server_name}" + f"/{content_uri.media_id}{params}", + shorthand=False, + access_token=self.tok, + ) + self.pump() + self.assertEqual(channel.code, 200, channel.result) + return channel + + def test_default_is_static(self) -> None: + """Without the `animated` parameter a static thumbnail is returned.""" + channel = self._thumbnail(self.content_uri, animated=None) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + def test_animated_false_is_static(self) -> None: + channel = self._thumbnail(self.content_uri, animated="false") + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + def test_animated_true_is_animated(self) -> None: + """With `animated=true` an animated WebP thumbnail is returned.""" + channel = self._thumbnail(self.content_uri, animated="true") + self.assertEqual( + channel.headers.getRawHeaders(b"Content-Type"), [b"image/webp"] + ) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(result.n_frames, 2) + + def test_animated_thumbnail_is_cached(self) -> None: + """Animated thumbnails are generated at upload and served from cache, + not regenerated on each request.""" + # The animated (WebP) thumbnail is stored alongside the static ones at + # upload time. + thumbnails = self.get_success( + self.store.get_local_media_thumbnails(self.content_uri.media_id) + ) + self.assertTrue( + any(info.type == "image/webp" for info in thumbnails), + thumbnails, + ) + + # Two consecutive requests return identical cached bytes. + first = self._thumbnail(self.content_uri, animated="true") + second = self._thumbnail(self.content_uri, animated="true") + self.assertEqual(first.result["body"], second.result["body"]) + + def test_non_animatable_source_falls_back_to_static(self) -> None: + """`animated=true` on a non-animatable source behaves as `animated=false`.""" + png_uri = self._upload(SMALL_PNG, "image/png") + channel = self._thumbnail(png_uri, animated="true") + self.assertEqual(channel.headers.getRawHeaders(b"Content-Type"), [b"image/png"]) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + @override_config({"dynamic_thumbnails": True}) + def test_dynamic_thumbnails_generates_and_caches_animated(self) -> None: + """With dynamic thumbnails, animated thumbnails are generated on demand + and then cached.""" + content_uri = self._upload(self._make_animated_gif(), "image/gif") + + channel = self._thumbnail(content_uri, animated="true") + self.assertEqual( + channel.headers.getRawHeaders(b"Content-Type"), [b"image/webp"] + ) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertTrue(getattr(result, "is_animated", False)) + + # The on-demand generated thumbnail is now cached. + thumbnails = self.get_success( + self.store.get_local_media_thumbnails(content_uri.media_id) + ) + self.assertTrue( + any(info.type == "image/webp" for info in thumbnails), + thumbnails, + ) From 4679ed4b06c79f27d613f6cafea68f430b3fbbf9 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 15 Jul 2026 11:45:52 -0500 Subject: [PATCH 024/158] Silencing alerts is no longer necessary during a deploy (release script instructions) (#19968) As discussed in [`#element-backend-internal:matrix.org`](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$29ZzRe7gg62UZmT0bgeseMs320Kb-Ub6DyaQE20-3ng?via=jki.re&via=element.io&via=matrix.org) Our assumptions on why this was done previously: in the olden days you'd get paged doing the redeploy even if everything was actually fine (probably before we started doing rolling restarts?) --- changelog.d/19968.misc | 1 + scripts-dev/release.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/19968.misc diff --git a/changelog.d/19968.misc b/changelog.d/19968.misc new file mode 100644 index 0000000000..1526938614 --- /dev/null +++ b/changelog.d/19968.misc @@ -0,0 +1 @@ +Remove alert silencing deploy step from release script instructions as it's no longer necessary. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index ea4fb0f142..68e0b3c772 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -806,7 +806,6 @@ def full(gh_token: str) -> None: _prepare() click.echo("Deploy to matrix.org and ensure that it hasn't fallen over.") - click.echo("Remember to silence the alerts to prevent alert spam.") click.confirm("Deployed?", abort=True) click.echo("\n*** tag ***") From d930ac615b1c41e9d7ecdb5ca21a9e4c8be75908 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 15 Jul 2026 12:01:09 -0500 Subject: [PATCH 025/158] Update release script JSON schema find/replace to be compatible with macOS (#19962) BSD vs GNU `sed` problems: ```shell $ sed -i '0,/^\$id: .*/s||$id: https://element-hq.github.io/synapse/schema/synapse/v1.157/synapse-config.schema.json|' schema/synapse-config.schema.yaml sed: 1: "schema/synapse-config.s ...": bad flag in substitute command: 'h' ``` --- changelog.d/19962.misc | 1 + scripts-dev/release.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19962.misc diff --git a/changelog.d/19962.misc b/changelog.d/19962.misc new file mode 100644 index 0000000000..2f47794a67 --- /dev/null +++ b/changelog.d/19962.misc @@ -0,0 +1 @@ +Update release script JSON schema find/replace task to be compatible with macOS. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 68e0b3c772..d1ba2e5822 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -31,6 +31,7 @@ import sys import time import urllib.request from os import path +from pathlib import Path from tempfile import TemporaryDirectory from typing import Any @@ -257,10 +258,16 @@ def _prepare() -> None: subprocess.check_output(["poetry", "version", new_version]) # Update config schema $id. - schema_file = "schema/synapse-config.schema.yaml" + schema_file_path = Path("schema/synapse-config.schema.yaml") major_minor_version = ".".join(new_version.split(".")[:2]) url = f"https://element-hq.github.io/synapse/schema/synapse/v{major_minor_version}/synapse-config.schema.json" - subprocess.check_output(["sed", "-i", f"0,/^\\$id: .*/s||$id: {url}|", schema_file]) + # Find/replace the `$id: ...` line in `schema/synapse-config.schema.yaml` with a new + # unique identifier for this release + schema_file_content = schema_file_path.read_text() + new_schema_file_content = re.sub( + r"^\$id: .*", f"$id: {url}", schema_file_content, count=1, flags=re.MULTILINE + ) + schema_file_path.write_text(new_schema_file_content) # Generate changelogs. generate_and_write_changelog(synapse_repo, current_version, new_version) From 5ed830b3b4c74c89d876cc07756c5d98a100cbed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=9C=E5=9D=82=E9=9B=85?= <23130178+ShadowRZ@users.noreply.github.com> Date: Fri, 17 Jul 2026 01:33:13 +0800 Subject: [PATCH 026/158] Change default room version to 11 (MSC4239) (#18680) Fix #18530 Complement test changes: https://github.com/matrix-org/complement/pull/858 SyTest changes: https://github.com/matrix-org/sytest/pull/1422 --- changelog.d/18680.feature | 1 + .../configuration/config_documentation.md | 3 +- schema/synapse-config.schema.yaml | 6 ++- synapse/config/server.py | 2 +- .../test_federation_out_of_band_membership.py | 5 +-- tests/module_api/test_api.py | 19 ---------- .../client/sliding_sync/test_rooms_invites.py | 8 ++-- tests/rest/client/test_redactions.py | 7 +++- tests/rest/client/test_rooms.py | 38 +++++++++---------- tests/storage/databases/main/test_room.py | 21 ++++++++-- 10 files changed, 57 insertions(+), 53 deletions(-) create mode 100644 changelog.d/18680.feature diff --git a/changelog.d/18680.feature b/changelog.d/18680.feature new file mode 100644 index 0000000000..e4280d4678 --- /dev/null +++ b/changelog.d/18680.feature @@ -0,0 +1 @@ +Change default room version to 11, implementing [MSC4239](https://github.com/matrix-org/matrix-spec-proposals/pull/4239) as part of Matrix v1.14. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 62a4676adb..7f69cf23b1 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -363,8 +363,9 @@ Known room versions are listed [here](https://spec.matrix.org/latest/rooms/#comp For example, for room version 1, `default_room_version` should be set to "1". _Changed in Synapse 1.76:_ the default version room version was increased from [9](https://spec.matrix.org/v1.5/rooms/v9/) to [10](https://spec.matrix.org/v1.5/rooms/v10/). +_Changed in Synapse 1.157:_ the default version room version was increased from [10](https://spec.matrix.org/v1.12/rooms/v10/) to [11](https://spec.matrix.org/v1.12/rooms/v11/). -Defaults to `"10"`. +Defaults to `"11"`. Example configuration: ```yaml diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index ea5bd7c7af..68e9e2676b 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -309,7 +309,11 @@ properties: _Changed in Synapse 1.76:_ the default version room version was increased from [9](https://spec.matrix.org/v1.5/rooms/v9/) to [10](https://spec.matrix.org/v1.5/rooms/v10/). - default: "10" + + _Changed in Synapse 1.157:_ the default version room version was increased + from [10](https://spec.matrix.org/v1.12/rooms/v10/) to + [11](https://spec.matrix.org/v1.12/rooms/v11/). + default: "11" examples: - "8" gc_thresholds: diff --git a/synapse/config/server.py b/synapse/config/server.py index 832183f32b..a892d7b1e1 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -176,7 +176,7 @@ DEFAULT_IP_RANGE_BLOCKLIST = [ "fec0::/10", ] -DEFAULT_ROOM_VERSION = "10" +DEFAULT_ROOM_VERSION = "11" # Defaults for the presence state machine timers, in milliseconds. Overridden # by the corresponding options in the `presence` config section. diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py index 1707081863..85e221a64a 100644 --- a/tests/federation/test_federation_out_of_band_membership.py +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -189,7 +189,7 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): # Create a remote room room_creator_user_id = f"@remote-user:{self.OTHER_SERVER_NAME}" remote_room_id = f"!remote-room:{self.OTHER_SERVER_NAME}" - room_version = RoomVersions.V10 + room_version = RoomVersions.V11 room_create_event = make_test_event( self.add_hashes_and_signatures_from_other_server( @@ -201,9 +201,6 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): "type": EventTypes.Create, "state_key": "", "content": { - # The `ROOM_CREATOR` field could be removed if we used a room - # version > 10 (in favor of relying on `sender`) - EventContentFields.ROOM_CREATOR: room_creator_user_id, EventContentFields.ROOM_VERSION: room_version.identifier, }, "auth_events": [], diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index 3114675052..b4b14c87b9 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -743,7 +743,6 @@ class ModuleApiTestCase(BaseModuleApiTestCase): # Now do the happy path. user_id = self.register_user("user", "password") - access_token = self.login(user_id, "password") room_id, room_alias = self.get_success( self.module_api.create_room( @@ -751,15 +750,6 @@ class ModuleApiTestCase(BaseModuleApiTestCase): ) ) - # Check room creator. - channel = self.make_request( - "GET", - f"/_matrix/client/v3/rooms/{room_id}/state/m.room.create", - access_token=access_token, - ) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body["creator"], user_id) - # Check room alias. self.assertEqual(room_alias, f"#foo-bar:{self.module_api.server_name}") @@ -768,15 +758,6 @@ class ModuleApiTestCase(BaseModuleApiTestCase): self.module_api.create_room(user_id=user_id, config={}, ratelimit=False) ) - # Check room creator. - channel = self.make_request( - "GET", - f"/_matrix/client/v3/rooms/{room_id}/state/m.room.create", - access_token=access_token, - ) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body["creator"], user_id) - # Check room alias. self.assertIsNone(room_alias) diff --git a/tests/rest/client/sliding_sync/test_rooms_invites.py b/tests/rest/client/sliding_sync/test_rooms_invites.py index 5a463303dd..85628833f5 100644 --- a/tests/rest/client/sliding_sync/test_rooms_invites.py +++ b/tests/rest/client/sliding_sync/test_rooms_invites.py @@ -143,7 +143,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"creator": user2_id, "room_version": "10"}, + "content": {"room_version": "11"}, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -253,7 +253,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"creator": user2_id, "room_version": "10"}, + "content": {"room_version": "11"}, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -374,7 +374,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"creator": user2_id, "room_version": "10"}, + "content": {"room_version": "11"}, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -500,7 +500,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"creator": user2_id, "room_version": "10"}, + "content": {"room_version": "11"}, "sender": user2_id, "state_key": "", "type": "m.room.create", diff --git a/tests/rest/client/test_redactions.py b/tests/rest/client/test_redactions.py index 997ca5f9ca..c8ab9577a3 100644 --- a/tests/rest/client/test_redactions.py +++ b/tests/rest/client/test_redactions.py @@ -63,7 +63,12 @@ class RedactionsTestCase(HomeserverTestCase): # Create a room self.room_id = self.helper.create_room_as( - self.mod_user_id, tok=self.mod_access_token + self.mod_user_id, + tok=self.mod_access_token, + # FIXME: Using room version 10 here as using later version requires + # https://github.com/element-hq/synapse/pull/19782 + # Remove the override when the above PR is merged. + room_version="10", ) # Invite the other user diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 90063ccb06..4df2002a86 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -4916,7 +4916,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -4924,7 +4924,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -4959,7 +4959,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5010,7 +5010,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5035,7 +5035,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -5043,7 +5043,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -5078,7 +5078,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5119,7 +5119,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): # user should be able to join again channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -5167,7 +5167,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5281,7 +5281,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -5289,7 +5289,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -5324,7 +5324,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5375,7 +5375,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5397,7 +5397,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -5405,7 +5405,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -5440,7 +5440,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5476,7 +5476,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): # user re-joins after kick channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -5484,7 +5484,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -5524,7 +5524,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( diff --git a/tests/storage/databases/main/test_room.py b/tests/storage/databases/main/test_room.py index 4ed775ad76..ef6193da98 100644 --- a/tests/storage/databases/main/test_room.py +++ b/tests/storage/databases/main/test_room.py @@ -20,6 +20,7 @@ # import json +from typing import Optional from twisted.internet.testing import MemoryReactor @@ -45,9 +46,13 @@ class RoomBackgroundUpdateStoreTestCase(HomeserverTestCase): self.user_id = self.register_user("foo", "pass") self.token = self.login("foo", "pass") - def _generate_room(self) -> str: + def _generate_room(self, room_version: Optional[str] = None) -> str: """Create a room and return the room ID.""" - return self.helper.create_room_as(self.user_id, tok=self.token) + return self.helper.create_room_as( + self.user_id, + tok=self.token, + room_version=room_version, + ) def run_background_updates(self, update_name: str) -> None: """Insert and run the background update.""" @@ -70,7 +75,17 @@ class RoomBackgroundUpdateStoreTestCase(HomeserverTestCase): """ # Insert a room without the creator - room_id = self._generate_room() + room_id = self._generate_room( + # Create the room as v10, because the `POPULATE_ROOMS_CREATOR_COLUMN` + # background update assumes that the room `creator` field is available + # in the `m.room.create` event content, which was removed in room + # version 11. That assumption is safe as the background update only + # backfills rooms whose `creator` column is unset, which can only be + # rooms created before Synapse started populating the column at room + # creation time (Synapse 1.43, 2021), all of which predate room version + # 11 (2023). + room_version="10" + ) self.get_success( self.store.db_pool.simple_update( table="rooms", From e0f251ce2c0e20eff7af423c68ee349923f6d719 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 Jul 2026 10:21:37 -0500 Subject: [PATCH 027/158] 1.157.0 --- CHANGES.md | 7 ++++++- debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 92f6a87a77..68a05aa943 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,12 @@ -# Synapse 1.157.0rc1 (2026-07-14) +# synapse 1.157.0 (2026-07-21) + +No significant changes since 1.157.0rc1. Please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11570) as this release removes support for the deprecated MSC3861 Auth Delegation (`experimental_features.msc3861`). + +# Synapse 1.157.0rc1 (2026-07-14) + ## Features - [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Limit how many delayed events a user may have scheduled at once. ([\#19539](https://github.com/element-hq/synapse/issues/19539)) diff --git a/debian/changelog b/debian/changelog index 8c0e1ccfe8..f5dec1026f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.157.0) stable; urgency=medium + + * New synapse release 1.157.0. + + -- Synapse Packaging team Tue, 21 Jul 2026 15:17:02 +0000 + matrix-synapse-py3 (1.157.0~rc1) stable; urgency=medium * New synapse release 1.157.0rc1. diff --git a/pyproject.toml b/pyproject.toml index f30ac49847..9e53fec4bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.157.0rc1" +version = "1.157.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 7cb94ab2667c49aee1dd00c48290a95afef6b803 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 Jul 2026 09:07:29 -0500 Subject: [PATCH 028/158] Fix config regression around falsy `experimental_features` (`None`) (#19987) Fix https://github.com/element-hq/synapse/issues/19986 Regressed in https://github.com/element-hq/synapse/pull/19539 --- changelog.d/19987.bugfix | 1 + synapse/config/server.py | 7 +-- tests/config/test_experimental.py | 71 +++++++++++++++++++++++++++++++ tests/config/test_server.py | 24 +++++++++++ 4 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19987.bugfix create mode 100644 tests/config/test_experimental.py diff --git a/changelog.d/19987.bugfix b/changelog.d/19987.bugfix new file mode 100644 index 0000000000..73f97c1f7d --- /dev/null +++ b/changelog.d/19987.bugfix @@ -0,0 +1 @@ +Fix config regression around falsy `experimental_features` no longer being accepted. diff --git a/synapse/config/server.py b/synapse/config/server.py index 832183f32b..16479170d0 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -966,9 +966,10 @@ class ServerConfig(Config): # The maximum number of delayed events a user may have scheduled at a time. # (Defined here despite being experimental to be near the other MSC4140 config) - self.max_delayed_events_per_user: int = config.get( - "experimental_features", {} - ).get("msc4140_max_delayed_events_per_user", 100) + experimental = config.get("experimental_features") or {} + self.max_delayed_events_per_user: int = experimental.get( + "msc4140_max_delayed_events_per_user", 100 + ) if ( not isinstance(self.max_delayed_events_per_user, int) or self.max_delayed_events_per_user < 0 diff --git a/tests/config/test_experimental.py b/tests/config/test_experimental.py new file mode 100644 index 0000000000..0fede98ee9 --- /dev/null +++ b/tests/config/test_experimental.py @@ -0,0 +1,71 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + + +import yaml +from parameterized import parameterized + +from synapse.config._base import RootConfig +from synapse.config.experimental import ExperimentalConfig +from synapse.config.homeserver import HomeServerConfig +from synapse.types import JsonDict + +from tests import unittest + + +class ExperimentalConfigTestCase(unittest.TestCase): + @parameterized.expand( + [ + [ + "single", + { + "experimental_features": { + "msc3575_enabled": True, + } + }, + ], + [ + "multi", + { + "experimental_features": { + "msc3575_enabled": True, + "msc3030_enabled": True, + } + }, + ], + # This has historically worked and this is being added as a regression test + ["none", {"experimental_features": None}], + ] + ) + def test_experimental_features_parsing( + self, test_description: str, config_values: JsonDict + ) -> None: + """ + Test the that `experimental_features` parses with these values + """ + + _read_config(config_values) + + +def _read_config(config_values: JsonDict) -> None: + ExperimentalConfig(RootConfig()).read_config( + yaml.safe_load( + HomeServerConfig().generate_config( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + ) + ) + | config_values, + allow_secrets_in_config=False, + ) diff --git a/tests/config/test_server.py b/tests/config/test_server.py index 41ea8fb5b1..f718e20a2a 100644 --- a/tests/config/test_server.py +++ b/tests/config/test_server.py @@ -22,6 +22,7 @@ from typing import Any import yaml +from parameterized import parameterized from synapse.config._base import ConfigError, RootConfig from synapse.config.homeserver import HomeServerConfig @@ -228,6 +229,29 @@ class ServerConfigTestCase(unittest.TestCase): with self.assertRaises(ConfigError): _read_config(generate_config(disallowed_value)) + @parameterized.expand( + [ + [ + "single", + { + "experimental_features": { + "msc4140_max_delayed_events_per_user": 3, + } + }, + ], + # This has historically worked and this is being added as a regression test + ["none", {"experimental_features": None}], + ] + ) + def test_experimental_features_parsing( + self, test_description: str, config_values: JsonDict + ) -> None: + """ + Test the that `experimental_features` parses with these values + """ + + _read_config(config_values) + def _read_config(config_values: JsonDict) -> None: ServerConfig(RootConfig()).read_config( From 2b5a0623bace645cd524676211393aa1caf26a2e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 Jul 2026 09:31:47 -0500 Subject: [PATCH 029/158] 1.157.1 --- CHANGES.md | 9 ++++++++- changelog.d/19987.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/19987.bugfix diff --git a/CHANGES.md b/CHANGES.md index 68a05aa943..d122a74e19 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,11 @@ -# synapse 1.157.0 (2026-07-21) +# Synapse 1.157.1 (2026-07-22) + +## Bugfixes + +- Fix config regression around falsy `experimental_features` no longer being accepted. ([\#19987](https://github.com/element-hq/synapse/issues/19987)) + + +# Synapse 1.157.0 (2026-07-21) No significant changes since 1.157.0rc1. diff --git a/changelog.d/19987.bugfix b/changelog.d/19987.bugfix deleted file mode 100644 index 73f97c1f7d..0000000000 --- a/changelog.d/19987.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix config regression around falsy `experimental_features` no longer being accepted. diff --git a/debian/changelog b/debian/changelog index f5dec1026f..6a8d4663a4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.157.1) stable; urgency=medium + + * New synapse release 1.157.1. + + -- Synapse Packaging team Wed, 22 Jul 2026 14:30:55 +0000 + matrix-synapse-py3 (1.157.0) stable; urgency=medium * New synapse release 1.157.0. diff --git a/pyproject.toml b/pyproject.toml index 9e53fec4bf..64e2838085 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.157.0" +version = "1.157.1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 343581a1a0328cc87ca68a62323283c26c2b5555 Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Thu, 23 Jul 2026 13:13:21 +0200 Subject: [PATCH 030/158] fixes related to room v12 and sytest (#19898) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19898.bugfix | 3 ++ synapse/handlers/federation.py | 9 +++- synapse/handlers/room.py | 87 ++++++++++++++++++++++++------- tests/handlers/test_federation.py | 40 ++++++++++++++ tests/handlers/test_room.py | 41 +++++++++++++++ 5 files changed, 159 insertions(+), 21 deletions(-) create mode 100644 changelog.d/19898.bugfix diff --git a/changelog.d/19898.bugfix b/changelog.d/19898.bugfix new file mode 100644 index 0000000000..0d2a6d5063 --- /dev/null +++ b/changelog.d/19898.bugfix @@ -0,0 +1,3 @@ +Fix third-party (3pid) invites over federation failing intermittently in version 12 rooms, whose room IDs no longer encode a server name. ([#19898](https://github.com/element-hq/synapse/issues/19898)) + +Fix `/createRoom` intermittently failing with a 500 error in version 12 rooms when the same user creates several rooms at once, due to colliding room IDs. \ No newline at end of file diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ba83d4fd26..aacbd00afe 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1539,7 +1539,14 @@ class FederationHandler: if i == max_retries - 1: raise e else: - destinations = {x.split(":", 1)[-1] for x in (sender_user_id, room_id)} + # The sender always tells us a server to try. Pre-v12 room IDs also + # encode the resident server's domain, but v12+ room IDs are a hash + # with no domain component, so we must not treat them as a server + # name -- doing so raises an invalid-destination error which can + # abort the whole exchange before the valid destination is tried. + destinations = {get_domain_from_id(sender_user_id)} + if ":" in room_id: + destinations.add(get_domain_from_id(room_id)) try: await self.federation_client.forward_third_party_invite( diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c75fff9170..9029cda1f5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1409,30 +1409,77 @@ class RoomCreationHandler: is_public: bool, room_version: RoomVersion, ) -> tuple[EventBase, synapse.events.snapshot.EventContext]: - ( - creation_event, - new_unpersisted_context, - ) = await self.event_creation_handler.create_event( - creator, - { + """Create and store the create event for a v12+ room, retrying on room ID collision. + + In v12+ rooms the room ID is derived from the create event, so this + builds the create event, stores the room under the resulting ID, and + retries with a slightly older `origin_server_ts` if that ID is already taken. + + Args: + creator: The user creating the room. + creation_content: The content for the create event. + is_public: Whether the room is published to the room directory. + room_version: The version of the room being created. + + Returns: + A tuple of the create event and its event context. + + Raises: + StoreError: if a unique room ID could not be generated after several + attempts. + """ + # In v12+ rooms, the room ID is the reference hash of the create event, + # so two rooms whose create events have identical content collide on the + # same room ID. This happens in practice when the same user creates + # several rooms at once (e.g. concurrent /createRoom requests with the + # same config), as the only entropy in the create event is + # `origin_server_ts`, which has millisecond resolution. + # + # Retry a few times on collision, perturbing `origin_server_ts` + # so the create event hashes to a fresh room ID. This mirrors the + # collision handling in `_generate_and_create_room_id` used for + # older room versions. + attempts = 0 + while attempts < 5: + event_dict: JsonDict = { "content": creation_content, "sender": creator.user.to_string(), "type": EventTypes.Create, "state_key": "", - }, - prev_event_ids=[], - depth=1, - state_map={}, - for_batch=False, - ) - await self.store.store_room( - room_id=creation_event.room_id, - room_creator_user_id=creator.user.to_string(), - is_public=is_public, - room_version=room_version, - ) - creation_context = await new_unpersisted_context.persist(creation_event) - return (creation_event, creation_context) + } + if attempts > 0: + # Bump the timestamp to give the create event (and hence the + # room ID it hashes to) different content from the colliding one. + event_dict["origin_server_ts"] = ( + self.clock.time_msec() + random.randint(1, 10) + ) + + ( + creation_event, + new_unpersisted_context, + ) = await self.event_creation_handler.create_event( + creator, + event_dict, + prev_event_ids=[], + depth=1, + state_map={}, + for_batch=False, + ) + try: + await self.store.store_room( + room_id=creation_event.room_id, + room_creator_user_id=creator.user.to_string(), + is_public=is_public, + room_version=room_version, + ) + except StoreError: + attempts += 1 + continue + + creation_context = await new_unpersisted_context.persist(creation_event) + return (creation_event, creation_context) + + raise StoreError(500, "Couldn't generate a unique room ID.") async def _send_events_for_new_room( self, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 0c7edbaa2d..1f42490d96 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -108,6 +108,46 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): self.assertEqual(failure.errcode, Codes.FORBIDDEN, failure) self.assertEqual(failure.msg, "You are not invited to this room.") + def test_exchange_third_party_invite_forwards_to_sender_for_v12_room( + self, + ) -> None: + """When we are not resident in the room, a 3pid invite is forwarded to + a remote server for exchange. Pre-v12 room IDs encode the resident + server's domain, but v12+ room IDs are a content hash with no domain, + so the room ID must not be treated as a destination as doing so raises + an invalid-destination error and aborts the exchange. + + Regression test for 3pid invites over federation failing intermittently + in v12 rooms. + """ + sender_user_id = "@sender:remote.example.com" + # A v12-style room ID: a reference hash with no ":domain" suffix. + room_id = "!somereferencehashwithnodomain" + + # Pretend we're not in the room so we take the "forward to a remote + # server" branch. + self.handler._event_auth_handler.is_host_in_room = AsyncMock( # type: ignore[method-assign] + return_value=False + ) + forward = AsyncMock(return_value=None) + self.handler.federation_client.forward_third_party_invite = forward # type: ignore[method-assign] + + self.get_success( + self.handler.exchange_third_party_invite( + sender_user_id=sender_user_id, + target_user_id="@target:localhost", + room_id=room_id, + signed={"mxid": "@target:localhost", "token": "sometoken"}, + ) + ) + + forward.assert_called_once() + destinations = forward.call_args.args[0] + # Only the sender's server is a valid destination; the domainless room + # ID must not be misinterpreted as one. + self.assertEqual(destinations, {"remote.example.com"}) + self.assertNotIn(room_id, destinations) + def test_rejected_message_event_state(self) -> None: """ Check that we store the state group correctly for rejected non-state events. diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index df95490d3b..e1d21441d6 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -1,6 +1,9 @@ +from unittest.mock import patch + import synapse from synapse.api.constants import EventTypes, RoomEncryptionAlgorithms from synapse.rest.client import login, room +from synapse.types import create_requester from tests import unittest from tests.unittest import override_config @@ -106,3 +109,41 @@ class EncryptedByDefaultTestCase(unittest.HomeserverTestCase): tok=user_token, expect_code=404, ) + + +class RoomIDCollisionTestCase(unittest.HomeserverTestCase): + servlets = [ + login.register_servlets, + synapse.rest.admin.register_servlets_for_client_rest_resource, + room.register_servlets, + ] + + def test_colliding_v12_room_ids_are_retried(self) -> None: + """In v12+ rooms the room ID is the reference hash of the + create event, so two rooms whose create events have identical content + collide on the same room ID. This happens when the same user creates + several rooms at once (e.g. concurrent /createRoom requests with the + same config within the same millisecond). + + Regression test: the collision must be retried transparently and both + rooms created with distinct IDs. + """ + handler = self.hs.get_room_creation_handler() + user_id = self.register_user("alice", "pass") + requester = create_requester(user_id) + + # Freeze the clock so both create events carry the same + # `origin_server_ts`; with identical config this forces the two room + # IDs to hash to the same value, reproducing the collision. + with patch.object(self.hs.get_clock(), "time_msec", return_value=1234567890000): + room_id1, _, _ = self.get_success( + handler.create_room(requester, {"room_version": "12"}, ratelimit=False) + ) + room_id2, _, _ = self.get_success( + handler.create_room(requester, {"room_version": "12"}, ratelimit=False) + ) + + self.assertNotEqual(room_id1, room_id2) + # v12 room IDs are content hashes with no domain component. + self.assertNotIn(":", room_id1) + self.assertNotIn(":", room_id2) From 444564f0bef538b387987c453c64635b162d9929 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:10:04 +0000 Subject: [PATCH 031/158] Bump pyasn1 from 0.6.3 to 0.6.4 (#19989) Signed-off-by: dependabot[bot] --- poetry.lock | 56 ++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2d03b641e8..b2551ada51 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -532,7 +532,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -557,7 +557,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -607,7 +607,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -890,7 +890,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -931,7 +931,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1123,7 +1123,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1240,7 +1240,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, @@ -1548,7 +1548,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1833,7 +1833,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -2032,7 +2032,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2050,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2066,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2076,14 +2076,14 @@ psycopg2 = "*" [[package]] name = "pyasn1" -version = "0.6.3" +version = "0.6.4" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, - {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, + {file = "pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b"}, + {file = "pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81"}, ] [[package]] @@ -2346,7 +2346,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2478,7 +2478,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2503,7 +2503,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2531,7 +2531,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2935,7 +2935,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3135,7 +3135,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3151,7 +3151,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3226,7 +3226,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163"}, {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100"}, @@ -3358,7 +3358,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3619,7 +3619,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3640,7 +3640,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, From bdd53836ef848ca61346856801e23dac71f4aecb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:19:25 +0000 Subject: [PATCH 032/158] Bump gitpython from 3.1.50 to 3.1.52 (#19990) Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index b2551ada51..f33cd0d976 100644 --- a/poetry.lock +++ b/poetry.lock @@ -583,14 +583,14 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.50" +version = "3.1.54" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9"}, - {file = "gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc"}, + {file = "gitpython-3.1.54-py3-none-any.whl", hash = "sha256:b90d7b3d9bc0238681d24369130826f0dcdb0ceaa45db67cf1d4ffa4c302dedf"}, + {file = "gitpython-3.1.54.tar.gz", hash = "sha256:53f2085e24a2cda300eed7c3fc5f1559ae289634b725e98acaf4791940247aa0"}, ] [package.dependencies] From c39c92e9c9f393e8930d1d2749ba97e26a2b2bb3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:20:15 +0000 Subject: [PATCH 033/158] Bump setuptools from 82.0.0 to 83.0.0 (#19988) Signed-off-by: dependabot[bot] --- poetry.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index f33cd0d976..403a2297dd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3020,24 +3020,24 @@ tests = ["coverage[toml] (>=5.0.2)", "pytest"] [[package]] name = "setuptools" -version = "82.0.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" +version = "83.0.0" +description = "Most extensible Python build backend with support for C/C++ extension modules" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "setuptools-82.0.0-py3-none-any.whl", hash = "sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0"}, - {file = "setuptools-82.0.0.tar.gz", hash = "sha256:22e0a2d69474c6ae4feb01951cb69d515ed23728cf96d05513d36e42b62b37cb"}, + {file = "setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3"}, + {file = "setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.13.0) ; sys_platform != \"cygwin\""] -core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.13.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -enabler = ["pytest-enabler (>=2.2)"] +enabler = ["pytest-enabler (>=3.4)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.18.*)", "pytest-mypy"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.18.*)", "pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "setuptools-rust" From eaba03b5775fc392b93c863b904bc48acf420423 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:21:30 +0000 Subject: [PATCH 034/158] Bump pillow from 12.2.0 to 12.3.0 (#19983) Signed-off-by: dependabot[bot] --- poetry.lock | 182 +++++++++++++++++++++++++--------------------------- 1 file changed, 89 insertions(+), 93 deletions(-) diff --git a/poetry.lock b/poetry.lock index 403a2297dd..3b37350eaf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1901,103 +1901,99 @@ files = [ [[package]] name = "pillow" -version = "12.2.0" +version = "12.3.0" description = "Python Imaging Library (fork)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "pillow-12.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:a4e8f36e677d3336f35089648c8955c51c6d386a13cf6ee9c189c5f5bd713a9f"}, - {file = "pillow-12.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e589959f10d9824d39b350472b92f0ce3b443c0a3442ebf41c40cb8361c5b97"}, - {file = "pillow-12.2.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a52edc8bfff4429aaabdf4d9ee0daadbbf8562364f940937b941f87a4290f5ff"}, - {file = "pillow-12.2.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:975385f4776fafde056abb318f612ef6285b10a1f12b8570f3647ad0d74b48ec"}, - {file = "pillow-12.2.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd9c0c7a0c681a347b3194c500cb1e6ca9cab053ea4d82a5cf45b6b754560136"}, - {file = "pillow-12.2.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88d387ff40b3ff7c274947ed3125dedf5262ec6919d83946753b5f3d7c67ea4c"}, - {file = "pillow-12.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:51c4167c34b0d8ba05b547a3bb23578d0ba17b80a5593f93bd8ecb123dd336a3"}, - {file = "pillow-12.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:34c0d99ecccea270c04882cb3b86e7b57296079c9a4aff88cb3b33563d95afaa"}, - {file = "pillow-12.2.0-cp310-cp310-win32.whl", hash = "sha256:b85f66ae9eb53e860a873b858b789217ba505e5e405a24b85c0464822fe88032"}, - {file = "pillow-12.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:673aa32138f3e7531ccdbca7b3901dba9b70940a19ccecc6a37c77d5fdeb05b5"}, - {file = "pillow-12.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:3e080565d8d7c671db5802eedfb438e5565ffa40115216eabb8cd52d0ecce024"}, - {file = "pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab"}, - {file = "pillow-12.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65"}, - {file = "pillow-12.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7"}, - {file = "pillow-12.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e"}, - {file = "pillow-12.2.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705"}, - {file = "pillow-12.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176"}, - {file = "pillow-12.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b"}, - {file = "pillow-12.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909"}, - {file = "pillow-12.2.0-cp311-cp311-win32.whl", hash = "sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808"}, - {file = "pillow-12.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60"}, - {file = "pillow-12.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe"}, - {file = "pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5"}, - {file = "pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421"}, - {file = "pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987"}, - {file = "pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76"}, - {file = "pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005"}, - {file = "pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780"}, - {file = "pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5"}, - {file = "pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5"}, - {file = "pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940"}, - {file = "pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5"}, - {file = "pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414"}, - {file = "pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c"}, - {file = "pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2"}, - {file = "pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c"}, - {file = "pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795"}, - {file = "pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f"}, - {file = "pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed"}, - {file = "pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9"}, - {file = "pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed"}, - {file = "pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3"}, - {file = "pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9"}, - {file = "pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795"}, - {file = "pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e"}, - {file = "pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b"}, - {file = "pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06"}, - {file = "pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b"}, - {file = "pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f"}, - {file = "pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612"}, - {file = "pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c"}, - {file = "pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea"}, - {file = "pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4"}, - {file = "pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4"}, - {file = "pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea"}, - {file = "pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24"}, - {file = "pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98"}, - {file = "pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453"}, - {file = "pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8"}, - {file = "pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b"}, - {file = "pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295"}, - {file = "pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed"}, - {file = "pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae"}, - {file = "pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601"}, - {file = "pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be"}, - {file = "pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f"}, - {file = "pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286"}, - {file = "pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50"}, - {file = "pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104"}, - {file = "pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7"}, - {file = "pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150"}, - {file = "pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1"}, - {file = "pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463"}, - {file = "pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3"}, - {file = "pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166"}, - {file = "pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe"}, - {file = "pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd"}, - {file = "pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e"}, - {file = "pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06"}, - {file = "pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43"}, - {file = "pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354"}, - {file = "pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1"}, - {file = "pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e"}, - {file = "pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5"}, + {file = "pillow-12.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a"}, + {file = "pillow-12.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7"}, + {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f"}, + {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec"}, + {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468"}, + {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed"}, + {file = "pillow-12.3.0-cp310-cp310-win32.whl", hash = "sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1"}, + {file = "pillow-12.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb"}, + {file = "pillow-12.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f"}, + {file = "pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756"}, + {file = "pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6"}, + {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd"}, + {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd"}, + {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c"}, + {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5"}, + {file = "pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b"}, + {file = "pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a"}, + {file = "pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26"}, + {file = "pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965"}, + {file = "pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7"}, + {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9"}, + {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91"}, + {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c"}, + {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df"}, + {file = "pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f"}, + {file = "pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09"}, + {file = "pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec"}, + {file = "pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66"}, + {file = "pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35"}, + {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65"}, + {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3"}, + {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a"}, + {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e"}, + {file = "pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f"}, + {file = "pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8"}, + {file = "pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930"}, + {file = "pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8"}, + {file = "pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0"}, + {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321"}, + {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b"}, + {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198"}, + {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130"}, + {file = "pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a"}, + {file = "pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d"}, + {file = "pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838"}, + {file = "pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e"}, + {file = "pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17"}, + {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385"}, + {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c"}, + {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d"}, + {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931"}, + {file = "pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7"}, + {file = "pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c"}, + {file = "pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c"}, + {file = "pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f"}, + {file = "pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701"}, + {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace"}, + {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4"}, + {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39"}, + {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71"}, + {file = "pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827"}, + {file = "pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5"}, + {file = "pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658"}, + {file = "pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf"}, + {file = "pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64"}, + {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e"}, + {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777"}, + {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1"}, + {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9"}, + {file = "pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8"}, + {file = "pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418"}, + {file = "pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a"}, + {file = "pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce"}, ] [package.extras] @@ -2005,7 +2001,7 @@ docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-autobuild", "sphinx-copybut fpx = ["olefile"] mic = ["olefile"] test-arrow = ["arro3-compute", "arro3-core", "nanoarrow", "pyarrow"] -tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma (>=5)", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "trove-classifiers (>=2024.10.12)"] +tests = ["coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "setuptools", "trove-classifiers (>=2024.10.12)"] xmp = ["defusedxml"] [[package]] From a58ad44eb0ce3cc14153f921206f322e17339dc5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:23:18 +0000 Subject: [PATCH 035/158] Bump the minor-and-patches group with 5 updates (#19982) Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 20 ++++++++++---------- .github/workflows/push_complement_image.yml | 4 ++-- .github/workflows/release-artifacts.yml | 2 +- .github/workflows/tests.yml | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5d9ab4b8b4..b087e7f8cf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -41,13 +41,13 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -79,7 +79,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -87,7 +87,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: push: true labels: | @@ -136,14 +136,14 @@ jobs: merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io @@ -176,20 +176,20 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} password: ${{ steps.import-secrets.outputs.OCI_PASSWORD }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Install Cosign uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Calculate docker image tag - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ${{ matrix.repository }} flavor: | diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 34f86520b0..ac74cf9cf8 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -52,14 +52,14 @@ jobs: with: poetry-version: "2.4.1" - name: Login to registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Work out labels for complement image id: meta - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ghcr.io/${{ github.repository }}/complement-synapse tags: | diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index a54722e95b..2482662726 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -67,7 +67,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Set up docker layer caching uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 58974ad7db..cd967f6ade 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }} golangci: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.golangci }} steps: - - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 id: filter # We only check on PRs if: startsWith(github.ref, 'refs/pull/') From 97fb38eca66f74c4a761e1b1b394e4d19486e61b Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 24 Jul 2026 22:39:04 +0100 Subject: [PATCH 036/158] Return M_USER_LIMIT_EXCEEDED error code for media upload limits from MSC4335 (#18876) This PR uses the new `M_USER_LIMIT_EXCEEDED` common error code instead of the malformed `M_RESOURCE_LIMIT_EXCEEDED` error code (as reported by #18749) (spec: MSC4335). The error code is also changed from `400` to `403` as this matches what is in the spec for the endpoints: https://spec.matrix.org/v1.18/client-server-api/#post_matrixmediav3upload and https://spec.matrix.org/v1.18/client-server-api/#put_matrixmediav3uploadservernamemediaid (albeit the latter says that `M_FORBIDDEN` would be returned) By default a new built-in `media_upload_limit_exceeded.html` template will be served for the `info_uri`. Administrators can specify an external URI in config instead. Compatibility is retained for any existing modules making use the `MediaUploadLimit` (e.g. via the `get_media_upload_limits_for_user` callback). --- changelog.d/18876.feature | 1 + .../configuration/config_documentation.md | 13 ++ schema/synapse-config.schema.yaml | 40 +++- synapse/api/errors.py | 30 +++ synapse/config/repository.py | 75 +++++- synapse/media/media_repository.py | 17 +- .../callbacks/media_repository_callbacks.py | 5 +- .../media_upload_limit_exceeded.html | 6 + synapse/rest/synapse/client/__init__.py | 11 + .../client/media_upload_limit_exceeded.py | 42 ++++ tests/rest/client/test_media.py | 215 ++++++++++++++++-- 11 files changed, 419 insertions(+), 36 deletions(-) create mode 100644 changelog.d/18876.feature create mode 100644 synapse/res/templates/media_upload_limit_exceeded.html create mode 100644 synapse/rest/synapse/client/media_upload_limit_exceeded.py diff --git a/changelog.d/18876.feature b/changelog.d/18876.feature new file mode 100644 index 0000000000..d62540f2f6 --- /dev/null +++ b/changelog.d/18876.feature @@ -0,0 +1 @@ +Return `M_USER_LIMIT_EXCEEDED` error code for media upload limits from [MSC4335](https://github.com/matrix-org/matrix-spec-proposals/pull/4335). diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 7f69cf23b1..7013e29f9f 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2228,13 +2228,26 @@ These settings can be overridden using the `get_media_upload_limits_for_user` mo Defaults to `[]`. +Options for each entry include: + +* `time_period` (duration): The time period over which the limit applies. Required. + +* `max_size` (byte size): Amount of data that can be uploaded in the time period by the user. Required. + +* `info_uri` (string): URI returned to the client for where the user can find information about the upload limit and how users can reduce their upload usage or request an upload limit increase. Optional. If not set, Synapse serves a built-in page (customisable via the `media_upload_limit_exceeded.html` template) and uses its URL instead. + +* `can_upgrade` (boolean): Value returned to the client for whether the limit can be increased. Defaults to `false`. + Example configuration: ```yaml media_upload_limits: - time_period: 1h max_size: 100M + info_uri: https://example.com/quota#hour - time_period: 1w max_size: 500M + info_uri: https://example.com/quota + can_upgrade: true ``` --- ### `max_image_pixels` diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 68e9e2676b..b547a4fe94 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2505,20 +2505,42 @@ properties: module API [callback](../../modules/media_repository_callbacks.md#get_media_upload_limits_for_user). default: [] items: - time_period: - type: "#/$defs/duration" - description: >- - The time period over which the limit applies. Required. - max_size: - type: "#/$defs/bytes" - description: >- - Amount of data that can be uploaded in the time period by the user. - Required. + type: object + required: + - time_period + - max_size + properties: + time_period: + $ref: "#/$defs/duration" + description: >- + The time period over which the limit applies. Required. + max_size: + $ref: "#/$defs/bytes" + description: >- + Amount of data that can be uploaded in the time period by the user. + Required. + info_uri: + type: string + description: >- + URI returned to the client for where the user can find information + about the upload limit and how users can reduce their upload usage + or request an upload limit increase. Optional. + If not set, Synapse serves a built-in page (customisable via the + `media_upload_limit_exceeded.html` template) and uses its URL + instead. + can_upgrade: + type: boolean + description: >- + Value returned to the client for whether the limit can be increased. + default: false examples: - - time_period: 1h max_size: 100M + info_uri: https://example.com/quota#hour - time_period: 1w max_size: 500M + info_uri: https://example.com/quota + can_upgrade: true max_image_pixels: $ref: "#/$defs/bytes" description: Maximum number of pixels that will be thumbnailed. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0c35b4a7ba..7797f5062e 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -152,6 +152,8 @@ class Codes(str, Enum): # Part of MSC4326 UNKNOWN_DEVICE = "ORG.MATRIX.MSC4326.M_UNKNOWN_DEVICE" + USER_LIMIT_EXCEEDED = "M_USER_LIMIT_EXCEEDED" + class CodeMessageException(RuntimeError): """An exception with integer code, a message string attributes and optional headers. @@ -513,6 +515,34 @@ class ResourceLimitError(SynapseError): ) +class UserLimitExceededError(SynapseError): + """ + Implementation of M_USER_LIMIT_EXCEEDED error + """ + + def __init__( + self, + code: int, + msg: str, + *, + info_uri: str, + can_upgrade: bool = False, + ): + additional_fields: dict[str, str | bool] = { + "info_uri": info_uri, + } + + if can_upgrade: + additional_fields["can_upgrade"] = can_upgrade + + super().__init__( + code, + msg, + Codes.USER_LIMIT_EXCEEDED, + additional_fields=additional_fields, + ) + + class EventSizeError(SynapseError): """An error raised when an event is too big.""" diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 373e518ddc..6d06c910d7 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -3,6 +3,7 @@ # # Copyright 2014, 2015 OpenMarket Ltd # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -21,14 +22,16 @@ import logging import os -from typing import Any +from typing import Annotated, Any import attr +from pydantic import AnyUrl, BeforeValidator, ValidationError from synapse.config.server import generate_ip_set, parse_proxy_config from synapse.types import JsonDict from synapse.util.check_dependencies import check_requirements from synapse.util.module_loader import load_module +from synapse.util.pydantic_models import ParseModel from ._base import Config, ConfigError @@ -134,6 +137,27 @@ class MediaUploadLimit: time_period_ms: int """The time period in milliseconds.""" + info_uri: str | None = None + """The URI to return with the M_USER_LIMIT_EXCEEDED error. + + If left unset (`None`), Synapse falls back to a static page served by itself + (see `MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH`), which explains that the limit has been + exceeded and can be customized by server administrators via a custom + template.""" + + can_upgrade: bool = False + """Whether the user can upgrade their plan to increase the limit. This is returned in the M_USER_LIMIT_EXCEEDED error.""" + + +class MediaUploadLimitConfigModel(ParseModel): + """Internal model for parsing a single media_upload_limits config entry.""" + + max_size: Annotated[int, BeforeValidator(Config.parse_size)] + time_period: Annotated[int, BeforeValidator(Config.parse_duration)] + info_uri: AnyUrl | None = None + """We accept AnyUrl as a subset of valid URIs. It could be widened in future if needed.""" + can_upgrade: bool = False + class ContentRepositoryConfig(Config): section = "media" @@ -143,6 +167,18 @@ class ContentRepositoryConfig(Config): # is not a media repo worker, as it's exposed in `/capabilities` self.url_preview_enabled = bool(config.get("url_preview_enabled", False)) + # Load the template used to render the fallback page. + # + # We set this up on all workers (not just the media repo) as the + # fallback page is served by whichever process handles + # `/_synapse/client/media_upload_limit_exceeded`, so every process must + # be able to render it. This must happen before the early return below, + # which is taken by workers that do not load the media repo. + self.media_upload_limit_exceeded_template = self.read_templates( + ["media_upload_limit_exceeded.html"], + (td for td in (self.root.server.custom_template_directory,) if td), + )[0] + # Only enable the media repo if either the media repo is enabled or the # current worker app is the media repo. if ( @@ -308,11 +344,40 @@ class ContentRepositoryConfig(Config): self.enable_authenticated_media = config.get("enable_authenticated_media", True) self.media_upload_limits: list[MediaUploadLimit] = [] - for limit_config in config.get("media_upload_limits", []): - time_period_ms = self.parse_duration(limit_config["time_period"]) - max_bytes = self.parse_size(limit_config["max_size"]) + for raw_entry in config.get("media_upload_limits", []): + try: + entry = MediaUploadLimitConfigModel.model_validate( + raw_entry, strict=True + ) + except ValidationError as e: + raise ConfigError( + "Could not validate media_upload_limits entry", + ("media_upload_limits",), + ) from e - self.media_upload_limits.append(MediaUploadLimit(max_bytes, time_period_ms)) + info_uri = str(entry.info_uri) if entry.info_uri is not None else None + self.media_upload_limits.append( + MediaUploadLimit( + max_bytes=entry.max_size, + time_period_ms=entry.time_period, + info_uri=info_uri, + can_upgrade=entry.can_upgrade, + ) + ) + + # The absolute URI of the static fallback page, used as the `info_uri` + # for any media upload limit (whether from config or a module callback) + # that doesn't specify one. Built from public_baseurl so that it is a + # usable absolute URL. We import here to avoid a circular import at + # module load time. + from synapse.rest.synapse.client.media_upload_limit_exceeded import ( + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, + ) + + self.media_upload_limit_fallback_info_uri = ( + self.root.server.public_baseurl + + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH.lstrip("/") + ) def generate_config_section(self, data_dir_path: str, **kwargs: Any) -> str: assert data_dir_path is not None diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index bbd31976cb..180adc20a0 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -40,6 +40,7 @@ from synapse.api.errors import ( NotFoundError, RequestSendFailed, SynapseError, + UserLimitExceededError, cs_error, ) from synapse.api.ratelimiting import Ratelimiter @@ -400,8 +401,20 @@ class MediaRepository: sent_bytes=uploaded_media_size, attempted_bytes=content_length, ) - raise SynapseError( - 400, "Media upload limit exceeded", Codes.RESOURCE_LIMIT_EXCEEDED + + # Fall back to the static page served by Synapse when the limit + # doesn't specify its own `info_uri` (e.g. limits returned by a + # module callback without one). + info_uri = ( + limit.info_uri + or self.hs.config.media.media_upload_limit_fallback_info_uri + ) + + raise UserLimitExceededError( + 403, + "Media upload limit exceeded", + info_uri=info_uri, + can_upgrade=limit.can_upgrade, ) if is_new_media: diff --git a/synapse/module_api/callbacks/media_repository_callbacks.py b/synapse/module_api/callbacks/media_repository_callbacks.py index f1e6ea4c38..5b58229b77 100644 --- a/synapse/module_api/callbacks/media_repository_callbacks.py +++ b/synapse/module_api/callbacks/media_repository_callbacks.py @@ -150,7 +150,10 @@ class MediaRepositoryModuleApiCallbacks: ): # Use a copy of the data in case the module modifies it limit_copy = MediaUploadLimit( - max_bytes=limit.max_bytes, time_period_ms=limit.time_period_ms + max_bytes=limit.max_bytes, + time_period_ms=limit.time_period_ms, + info_uri=limit.info_uri, + can_upgrade=limit.can_upgrade, ) await delay_cancellation( callback(user_id, limit_copy, sent_bytes, attempted_bytes) diff --git a/synapse/res/templates/media_upload_limit_exceeded.html b/synapse/res/templates/media_upload_limit_exceeded.html new file mode 100644 index 0000000000..62be18a138 --- /dev/null +++ b/synapse/res/templates/media_upload_limit_exceeded.html @@ -0,0 +1,6 @@ +{% extends "_base.html" %} +{% block title %}Media upload limit exceeded{% endblock %} + +{% block body %} +

You have exceeded a media upload limit. Ask your server administrator for more information.

+{% endblock %} diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py index e04b84ecac..3534d59120 100644 --- a/synapse/rest/synapse/client/__init__.py +++ b/synapse/rest/synapse/client/__init__.py @@ -3,6 +3,7 @@ # # Copyright 2021 The Matrix.org Foundation C.I.C. # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -24,6 +25,10 @@ from typing import TYPE_CHECKING, Mapping from twisted.web.resource import Resource from synapse.rest.synapse.client.federation_whitelist import FederationWhitelistResource +from synapse.rest.synapse.client.media_upload_limit_exceeded import ( + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, + MediaUploadLimitExceededResource, +) from synapse.rest.synapse.client.new_user_consent import NewUserConsentResource from synapse.rest.synapse.client.pick_idp import PickIdpResource from synapse.rest.synapse.client.pick_username import pick_username_resource @@ -54,6 +59,12 @@ def build_synapse_client_resource_tree(hs: "HomeServer") -> Mapping[str, Resourc "/_synapse/client/sso_register": SsoRegisterResource(hs), # Unsubscribe to notification emails link "/_synapse/client/unsubscribe": UnsubscribeResource(hs), + # Fallback page served as the `info_uri` for media upload limits that + # don't have an explicit `info_uri`. Mounted unconditionally: in a + # worker deployment, the media repo (which generates the errors) + # typically runs on a different worker than the one serving + # `/_synapse/client`. + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH: MediaUploadLimitExceededResource(hs), } if hs.config.mas.enabled: diff --git a/synapse/rest/synapse/client/media_upload_limit_exceeded.py b/synapse/rest/synapse/client/media_upload_limit_exceeded.py new file mode 100644 index 0000000000..48acdd2fd6 --- /dev/null +++ b/synapse/rest/synapse/client/media_upload_limit_exceeded.py @@ -0,0 +1,42 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import TYPE_CHECKING + +from synapse.http.server import DirectServeHtmlResource, respond_with_html +from synapse.http.site import SynapseRequest + +if TYPE_CHECKING: + from synapse.server import HomeServer + +# The path at which the fallback media upload limit exceeded info page is served. +# This is used as the `info_uri` returned in the `M_USER_LIMIT_EXCEEDED` error +# when no `info_uri` has been specified for a media upload limit. +MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH = "/_synapse/client/media_upload_limit_exceeded" + + +class MediaUploadLimitExceededResource(DirectServeHtmlResource): + """Serves a static fallback page explaining that a media upload limit has + been exceeded. + + This is used as the `info_uri` for the `M_USER_LIMIT_EXCEEDED` error when no + `info_uri` has been specified for a media upload limit. + """ + + def __init__(self, hs: "HomeServer"): + super().__init__(clock=hs.get_clock()) + self._template = hs.config.media.media_upload_limit_exceeded_template + + async def _async_render_GET(self, request: SynapseRequest) -> None: + respond_with_html(request, 200, self._template.render()) diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 8757e27d48..1cf401ed4f 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -3,6 +3,7 @@ # # Copyright 2022 The Matrix.org Foundation C.I.C. # Copyright (C) 2024 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -58,6 +59,10 @@ from synapse.media.url_previewer import IMAGE_CACHE_EXPIRY_MS from synapse.module_api import MediaUploadLimit from synapse.rest import admin from synapse.rest.client import login, media +from synapse.rest.synapse.client import build_synapse_client_resource_tree +from synapse.rest.synapse.client.media_upload_limit_exceeded import ( + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, +) from synapse.server import HomeServer from synapse.types import JsonDict, UserID from synapse.util.clock import Clock @@ -2933,11 +2938,20 @@ class MediaUploadLimits(unittest.HomeserverTestCase): config["media_storage_providers"] = [provider_config] - # These are the limits that we are testing - config["media_upload_limits"] = [ - {"time_period": "1d", "max_size": "1K"}, - {"time_period": "1w", "max_size": "3K"}, - ] + # These are the limits that we are testing unless overridden + if config.get("media_upload_limits") is None: + config["media_upload_limits"] = [ + { + "time_period": "1d", + "max_size": "1K", + "info_uri": "https://example.com/limits#daily", + }, + { + "time_period": "1w", + "max_size": "3K", + "info_uri": "https://example.com/limits#weekly", + }, + ] return self.setup_test_homeserver(config=config) @@ -2951,6 +2965,9 @@ class MediaUploadLimits(unittest.HomeserverTestCase): def create_resource_dict(self) -> dict[str, Resource]: resources = super().create_resource_dict() resources["/_matrix/media"] = self.hs.get_media_repository_resource() + # Mount the `/_synapse/client` tree, which includes the fallback page + # served at `MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH` which we use in the tests. + resources.update(build_synapse_client_resource_tree(self.hs)) return resources def upload_media(self, size: int) -> FakeChannel: @@ -2975,7 +2992,8 @@ class MediaUploadLimits(unittest.HomeserverTestCase): self.assertEqual(channel.code, 200) channel = self.upload_media(800) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") def test_under_daily_limit(self) -> None: """Test that uploading media under the daily limit fails.""" @@ -3013,7 +3031,8 @@ class MediaUploadLimits(unittest.HomeserverTestCase): # This will fail as the weekly limit has been exceeded channel = self.upload_media(900) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") # Reset the weekly limit by advancing a week self.reactor.advance(7 * 60 * 60 * 24) # Advance by 7 days @@ -3022,6 +3041,104 @@ class MediaUploadLimits(unittest.HomeserverTestCase): channel = self.upload_media(900) self.assertEqual(channel.code, 200) + @override_config( + { + "media_upload_limits": [ + { + "time_period": "1d", + "max_size": "1K", + # No `info_uri` is configured, so the fallback resource + # should be used instead. + } + ], + } + ) + def test_falls_back_to_served_page_when_no_info_uri(self) -> None: + """When no info_uri is configured, the error should point at a page + served by Synapse, and that page should return HTML.""" + channel = self.upload_media(1300) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") + + expected_info_uri = ( + self.hs.config.server.public_baseurl + + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH.lstrip("/") + ) + self.assertEqual(channel.json_body["info_uri"], expected_info_uri) + + # The fallback page should be served by Synapse and return HTML. + page = self.make_request( + "GET", + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, + shorthand=False, + ) + self.assertEqual(page.code, 200) + self.assertEqual( + page.headers.getRawHeaders("Content-Type"), + ["text/html; charset=utf-8"], + ) + self.assertIn(b"upload limit", page.result["body"]) + + def test_fallback_page_mounted(self) -> None: + """The fallback resource should always be mounted, even if every configured + limit has an explicit info_uri, since module callbacks can return + limits without an info_uri at any time.""" + # The default config for this test case sets an info_uri on every limit. + page = self.make_request( + "GET", + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, + shorthand=False, + ) + self.assertEqual(page.code, 200) + self.assertEqual( + page.headers.getRawHeaders("Content-Type"), + ["text/html; charset=utf-8"], + ) + + @override_config( + { + "media_upload_limits": [ + { + "time_period": "1d", + "max_size": "1K", + "info_uri": "https://example.com", + } + ], + } + ) + def test_returns_hard_user_limit_exceeded_by_default(self) -> None: + """Test that the error is returned with can_upgrade False by default.""" + channel = self.upload_media(1300) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") + self.assertEqual(channel.json_body["info_uri"], "https://example.com/") + # the spec says that can_upgrade should not be included if it is False + self.assertIsNone(channel.json_body.get("can_upgrade")) + + @override_config( + { + "media_upload_limits": [ + { + "time_period": "1d", + "max_size": "1K", + "info_uri": "https://example.com", + "can_upgrade": True, + } + ], + } + ) + def test_returns_soft_user_limit_exceeded(self) -> None: + """Test that the M_USER_LIMIT_EXCEEDED error is returned with + can_upgrade True when specified in config.""" + channel = self.upload_media(500) + self.assertEqual(channel.code, 200) + + channel = self.upload_media(800) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") + self.assertEqual(channel.json_body["info_uri"], "https://example.com/") + self.assertEqual(channel.json_body["can_upgrade"], True) + class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): """ @@ -3054,10 +3171,19 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): config["media_storage_providers"] = [provider_config] # default limits to use - config["media_upload_limits"] = [ - {"time_period": "1d", "max_size": "1K"}, - {"time_period": "1w", "max_size": "3K"}, - ] + if config.get("media_upload_limits") is None: + config["media_upload_limits"] = [ + { + "time_period": "1d", + "max_size": "1K", + "info_uri": "https://example.com/limits#daily", + }, + { + "time_period": "1w", + "max_size": "3K", + "info_uri": "https://example.com/limits#weekly", + }, + ] return self.setup_test_homeserver(config=config) @@ -3070,10 +3196,14 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): # n.b. we return these in increasing duration order and Synapse will need to sort them correctly return [ MediaUploadLimit( - time_period_ms=Config.parse_duration("1d"), max_bytes=5000 + time_period_ms=Config.parse_duration("1d"), + max_bytes=5000, + info_uri="https://override.example.com/limits#daily", ), MediaUploadLimit( - time_period_ms=Config.parse_duration("1w"), max_bytes=15000 + time_period_ms=Config.parse_duration("1w"), + max_bytes=15000, + info_uri="https://override.example.com/limits#weekly", ), ] # user2 has no limits @@ -3154,13 +3284,16 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): # User 1 attempts to upload 4000 bytes taking it over the limit channel = self.upload_media(4000, self.tok1) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) assert self.last_media_upload_limit_exceeded is not None self.assertEqual(self.last_media_upload_limit_exceeded["user_id"], self.user1) self.assertEqual( self.last_media_upload_limit_exceeded["limit"], MediaUploadLimit( - max_bytes=5000, time_period_ms=Config.parse_duration("1d") + max_bytes=5000, + time_period_ms=Config.parse_duration("1d"), + info_uri="https://override.example.com/limits#daily", + can_upgrade=False, ), ) self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 3000) @@ -3169,13 +3302,16 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): # User 1 attempts to upload 20000 bytes which is over the weekly limit # This tests that the limits have been sorted as expected channel = self.upload_media(20000, self.tok1) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) assert self.last_media_upload_limit_exceeded is not None self.assertEqual(self.last_media_upload_limit_exceeded["user_id"], self.user1) self.assertEqual( self.last_media_upload_limit_exceeded["limit"], MediaUploadLimit( - max_bytes=15000, time_period_ms=Config.parse_duration("1w") + max_bytes=15000, + time_period_ms=Config.parse_duration("1w"), + info_uri="https://override.example.com/limits#weekly", + can_upgrade=False, ), ) self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 3000) @@ -3198,18 +3334,59 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): # User 3 uploads 800 bytes which is over the limit channel = self.upload_media(800, self.tok3) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) assert self.last_media_upload_limit_exceeded is not None self.assertEqual(self.last_media_upload_limit_exceeded["user_id"], self.user3) self.assertEqual( self.last_media_upload_limit_exceeded["limit"], MediaUploadLimit( - max_bytes=1024, time_period_ms=Config.parse_duration("1d") + max_bytes=1024, + time_period_ms=Config.parse_duration("1d"), + info_uri="https://example.com/limits#daily", + can_upgrade=False, ), ) self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 500) self.assertEqual(self.last_media_upload_limit_exceeded["attempted_bytes"], 800) + def test_module_limit_without_info_uri_falls_back(self) -> None: + """A limit returned by a module callback without an info_uri falls back + to the static page served by Synapse when the error is generated.""" + + async def _limits_without_info_uri( + user_id: str, + ) -> list[MediaUploadLimit] | None: + # info_uri is omitted (defaults to None). + return [ + MediaUploadLimit( + time_period_ms=Config.parse_duration("1d"), + max_bytes=123, + ) + ] + + self.hs.get_module_api().register_media_repository_callbacks( + get_media_upload_limits_for_user=_limits_without_info_uri, + ) + + # This is over the limit of 123 bytes, but below the default limit of 1k bytes + channel = self.upload_media(130, self.tok3) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") + + # Assert that the upload limit from the module callback was used (identified by + # max_bytes=123) (as opposed to the global default). + assert self.last_media_upload_limit_exceeded is not None + limit = self.last_media_upload_limit_exceeded["limit"] + assert isinstance(limit, MediaUploadLimit) + self.assertEqual(limit.max_bytes, 123) + + # Assert that the info_uri was correctly populated (even though not provided by the module) + expected_info_uri = ( + self.hs.config.server.public_baseurl + + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH.lstrip("/") + ) + self.assertEqual(channel.json_body["info_uri"], expected_info_uri) + class AnimatedThumbnailTestCase(unittest.HomeserverTestCase): """End-to-end tests for the `animated` query parameter on the local From 51672809c7c01de3ba5581f53376b15ae643fbfe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Jul 2026 09:29:58 +0100 Subject: [PATCH 037/158] Fix `_get_server_keys_json` invalidations over replication as JSON (#19966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache key of `_get_server_keys_json` is a single argument which is itself a `(server_name, key_id)` tuple, and `store_server_keys_response` passed that nested tuple straight into the cache invalidation stream. psycopg2 quietly serialises the inner tuple as a Postgres *record*, so the `keys` column of `cache_invalidation_stream_by_instance` ended up holding the record literal as a single string (e.g. `{"(srv,ed25519:abc)"}`) — which never matches the real cache key on the receiving side, i.e. the invalidation has always been a silent no-op on workers. The native Rust backend's stricter parameter conversion turns the same nested tuple into a loud `TypeError: unsupported parameter type for postgres: tuple`. Fix it the same way https://github.com/element-hq/synapse/pull/18899 did for `_get_e2e_cross_signing_signatures_for_device`, which has the same nested-tuple key shape: invalidate the local cache directly, JSON-encode the key for the replication row, and decode it again in `process_replication_rows`. Found as part of the the effort to port the database pool to Rust. --- changelog.d/19966.bugfix | 1 + synapse/storage/databases/main/cache.py | 22 ++++++++++ synapse/storage/databases/main/keys.py | 16 +++++-- tests/storage/databases/main/test_cache.py | 51 ++++++++++++++++++++++ 4 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19966.bugfix diff --git a/changelog.d/19966.bugfix b/changelog.d/19966.bugfix new file mode 100644 index 0000000000..5940159f56 --- /dev/null +++ b/changelog.d/19966.bugfix @@ -0,0 +1 @@ +Fix server key cache invalidations being silently dropped on workers. diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index a4530796f2..45fe605d8b 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -70,6 +70,10 @@ GET_E2E_CROSS_SIGNING_SIGNATURES_FOR_DEVICE_CACHE_NAME = ( "_get_e2e_cross_signing_signatures_for_device" ) +# As above: this cache takes a single argument which is itself a tuple, which +# requires special handling. +GET_SERVER_KEYS_JSON_CACHE_NAME = "_get_server_keys_json" + # How long between cache invalidation table cleanups, once we have caught up # with the backlog. REGULAR_CLEANUP_INTERVAL = Duration(hours=1) @@ -305,6 +309,24 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._get_e2e_cross_signing_signatures_for_device.invalidate( # type: ignore[attr-defined] ((user_id, device_id),) ) + elif row.cache_func == GET_SERVER_KEYS_JSON_CACHE_NAME: + # As above: each entry in "keys" is a JSON-encoded + # (server_name, key_id) pair, since the cache takes a single + # argument which is itself a tuple and we cannot send nested + # information over replication. + for json_str in row.keys: + try: + server_name, key_id = json.loads(json_str) + except (json.JSONDecodeError, TypeError, ValueError): + logger.error( + "Failed to deserialise cache key as valid JSON: %s", + json_str, + ) + continue + + self._get_server_keys_json.invalidate( # type: ignore[attr-defined] + ((server_name, key_id),) + ) else: self._attempt_to_invalidate_cache(row.cache_func, row.keys) diff --git a/synapse/storage/databases/main/keys.py b/synapse/storage/databases/main/keys.py index f81257b5a1..3e733643e7 100644 --- a/synapse/storage/databases/main/keys.py +++ b/synapse/storage/databases/main/keys.py @@ -113,10 +113,20 @@ class KeyStore(CacheInvalidationWorkerStore): # invalidate takes a tuple corresponding to the params of # _get_server_keys_json. _get_server_keys_json only takes one # param, which is itself the 2-tuple (server_name, key_id). - self._invalidate_cache_and_stream_bulk( + # + # Invalidate the local cache directly, but we can only send + # primitive types per argument over replication, so JSON-encode the + # nested key and unpack it on the receiving side (see + # `CacheInvalidationWorkerStore.process_replication_rows`). + for key_id in verify_keys: + txn.call_after( + self._get_server_keys_json.invalidate, + ((server_name, key_id),), + ) + self._send_invalidation_to_replication_bulk( txn, - self._get_server_keys_json, - [((server_name, key_id),) for key_id in verify_keys], + self._get_server_keys_json.__name__, + [(json.dumps([server_name, key_id]),) for key_id in verify_keys], ) self._invalidate_cache_and_stream_bulk( txn, diff --git a/tests/storage/databases/main/test_cache.py b/tests/storage/databases/main/test_cache.py index 3900a35290..1e7ae7e5a3 100644 --- a/tests/storage/databases/main/test_cache.py +++ b/tests/storage/databases/main/test_cache.py @@ -20,7 +20,10 @@ # from unittest.mock import Mock, call +from signedjson.key import generate_signing_key, get_verify_key + from synapse.storage.database import LoggingTransaction +from synapse.storage.keys import FetchKeyResult from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.unittest import HomeserverTestCase @@ -122,3 +125,51 @@ class CacheInvalidationOverReplicationTestCase(BaseMultiWorkerStreamTestCase): [call(key_list) for key_list in keys_to_invalidate], any_order=True, ) + + def test_server_keys_json_invalidation_replicates(self) -> None: + """`_get_server_keys_json` takes a single argument which is itself a + tuple, and we can't send nested keys over replication: the keys are + JSON-encoded on the sending side and decoded again in + `process_replication_rows`. Check the round trip invalidates the right + cache entry on the worker. + """ + master_invalidate = Mock() + worker_invalidate = Mock() + + self.store._get_server_keys_json.invalidate = master_invalidate + worker = self.make_worker_hs("synapse.app.generic_worker") + worker_ds = worker.get_datastores().main + worker_ds._get_server_keys_json.invalidate = worker_invalidate + + signing_key = generate_signing_key("ver1") + verify_key = get_verify_key(signing_key) + key_id = f"{verify_key.alg}:{verify_key.version}" + + assert self.store._cache_id_gen is not None + initial_token = self.store._cache_id_gen.get_current_token() + + self.get_success( + self.store.store_server_keys_response( + "server1", + from_server="server2", + ts_added_ms=self.clock.time_msec(), + verify_keys={ + key_id: FetchKeyResult( + verify_key=verify_key, valid_until_ts=200_000 + ), + }, + response_json={}, + ) + ) + second_token = self.store._cache_id_gen.get_current_token() + self.assertGreater(second_token, initial_token) + + self.get_success( + worker.get_replication_data_handler().wait_for_stream_position( + "master", "caches", second_token + ) + ) + + expected_key = (("server1", key_id),) + master_invalidate.assert_called_once_with(expected_key) + worker_invalidate.assert_called_once_with(expected_key) From befd2e2b7a09dee7a0912429459938ce3d54a994 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 27 Jul 2026 18:53:37 -0500 Subject: [PATCH 038/158] Add changelog --- changelog.d/20009.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/20009.bugfix diff --git a/changelog.d/20009.bugfix b/changelog.d/20009.bugfix new file mode 100644 index 0000000000..474afcef09 --- /dev/null +++ b/changelog.d/20009.bugfix @@ -0,0 +1 @@ +Fix `HomeServer.shutdown()` not being able to cleanly shutdown the homeserver (caused by Rust code referencing Python `DatabasePool`). From 408b0936acd66ca4ee77a3c1392c6421c07dc59f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 27 Jul 2026 18:56:17 -0500 Subject: [PATCH 039/158] Revert "Add changelog" This reverts commit befd2e2b7a09dee7a0912429459938ce3d54a994. --- changelog.d/20009.bugfix | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changelog.d/20009.bugfix diff --git a/changelog.d/20009.bugfix b/changelog.d/20009.bugfix deleted file mode 100644 index 474afcef09..0000000000 --- a/changelog.d/20009.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `HomeServer.shutdown()` not being able to cleanly shutdown the homeserver (caused by Rust code referencing Python `DatabasePool`). From c5e5abb8226763855791bea2bd57056c7ad3798e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 28 Jul 2026 04:04:48 -0500 Subject: [PATCH 040/158] Use weakref to Python `DatabasePool` to allow homeserver to cleanly shutdown (#20009) Use weakref to Python `DatabasePool` to allow homeserver to cleanly shutdown Spawning from [upgrading](https://github.com/element-hq/synapse-small-hosts/pull/420#discussion_r3661700252) the Synapse version in Synapse Pro for small hosts and seeing our tenant deprovision tests failing (end-to-end Complement tests). Specifically, it was failing with a test in [`TestLogging`](https://github.com/element-hq/synapse-small-hosts/blob/f24ee731ff419890680d1d80ef6eafe44f87f2c2/complement/tests/multi_synapse/logging_test.go#L328-L385) in CI but was equally reproducible with any test where we deprovision (`deployment.StopServer(...)`) like [`TestProvisionHomeserverTenant/deprovision_homeserver_tenant`](https://github.com/element-hq/synapse-small-hosts/blob/f24ee731ff419890680d1d80ef6eafe44f87f2c2/complement/tests/multi_synapse/provision_test.go#L48-L58). Clean homeserver shutdown specifically regressed with the changes from https://github.com/element-hq/synapse/pull/19878. The `PythonDatabasePoolWrapper` in Rust holds onto a reference to the Python `DatabasePool` which references the Python `HomeServer` and keeps the `HomeServer` from being garbage collected on the Python side. This PR updates `PythonDatabasePoolWrapper` to use a weak reference. ### Dev notes Previous PR where we had to deal with Rust <-> Python reference cycles, https://github.com/element-hq/synapse/pull/19837#discussion_r3425358096 PyO3 garbage collector docs (`__traverse__`, `__clear__`): https://pyo3.rs/v0.28.3/class/protocols#garbage-collector-integration ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20009.bugfix | 1 + rust/src/handlers/mod.rs | 9 +++-- rust/src/storage/db/python_db_pool.rs | 48 +++++++++++++++++++++------ tests/app/test_homeserver_shutdown.py | 10 ++++++ 4 files changed, 52 insertions(+), 16 deletions(-) create mode 100644 changelog.d/20009.bugfix diff --git a/changelog.d/20009.bugfix b/changelog.d/20009.bugfix new file mode 100644 index 0000000000..474afcef09 --- /dev/null +++ b/changelog.d/20009.bugfix @@ -0,0 +1 @@ +Fix `HomeServer.shutdown()` not being able to cleanly shutdown the homeserver (caused by Rust code referencing Python `DatabasePool`). diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs index d560423166..5df019adff 100644 --- a/rust/src/handlers/mod.rs +++ b/rust/src/handlers/mod.rs @@ -41,15 +41,14 @@ impl RustHandlers { // The Twisted reactor, used both to drive our Tokio runtime and to // marshal database work back onto the reactor thread. - let reactor: Py = homeserver.call_method0("get_reactor")?.unbind(); + let reactor = homeserver.call_method0("get_reactor")?.unbind(); // hs.get_datastores().main.db_pool - let db_pool_py: Py = homeserver + let db_pool_py = homeserver .call_method0("get_datastores")? .getattr("main")? - .getattr("db_pool")? - .unbind(); - let db_pool = PythonDatabasePoolWrapper::new(db_pool_py, reactor.clone_ref(py)); + .getattr("db_pool")?; + let db_pool = PythonDatabasePoolWrapper::new(&db_pool_py, reactor.clone_ref(py))?; // Store is shared across all of the handlers so let's use an `Arc` let store = Arc::new(Store { diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 5e62b06656..89003d574b 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -34,7 +34,9 @@ use pyo3::{ exceptions::{PyAssertionError, PyRuntimeError, PyTypeError}, intern, prelude::*, - types::{PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString}, + types::{ + PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString, PyWeakrefMethods, PyWeakrefReference, + }, }; use crate::deferred::run_python_awaitable; @@ -95,23 +97,32 @@ impl DatabaseEngine { /// Wrapper for a `DatabasePool` from the Python side of Synapse. pub struct PythonDatabasePoolWrapper { - /// The underlying Python `DatabasePool` - database_pool_py: Py, + /// A *weak* reference to the underlying Python `DatabasePool`. + /// + /// We use a *weak* reference to ensure the homeserver can cleanly shut down as + /// otherwise we hold onto `DatabasePool` which references the homeserver and keeps + /// the homeserver from being garbage collected. + database_pool_py_ref: Py, - /// The Twisted reactor. We need this to marshal back onto the reactor thread - /// (via `callFromThread`) when starting transactions, since Twisted's thread - /// pool machinery must be driven from there. + /// The Twisted reactor. We need this to marshal database work back onto the + /// reactor thread (via `callFromThread`) when starting transactions, since + /// Twisted's thread pool machinery must be driven from there. + /// + /// A strong reference is fine here: the reactor is a process-global singleton that + /// never gets garbage collected and never points back at the homeserver, so it is + /// not part of any reference cycle. Ideally, we could worry about it but + /// practically probably doesn't matter. reactor: Py, } impl PythonDatabasePoolWrapper { /// Build a wrapper around the Python `DatabasePool` (e.g. /// `hs.get_datastores().main.db_pool`) and the Twisted `reactor`. - pub fn new(database_pool_py: Py, reactor: Py) -> Self { - Self { - database_pool_py, + pub fn new(database_pool: &Bound<'_, PyAny>, reactor: Py) -> PyResult { + Ok(Self { + database_pool_py_ref: PyWeakrefReference::new(database_pool)?.unbind(), reactor, - } + }) } } @@ -193,9 +204,24 @@ impl DatabasePool for PythonDatabasePoolWrapper { )? .unbind(); + // Upgrade our weak reference to the Python `DatabasePool` into a strong + // one for the duration of this interaction. + let database_pool_py = self + .database_pool_py_ref + .bind(py) + .upgrade() + .ok_or_else(|| { + PyRuntimeError::new_err( + "The Python `DatabasePool` has already been dropped \ + (the homeserver is likely shutdown), so we cannot \ + run the database interaction.", + ) + })? + .unbind(); + Ok(( callback, - self.database_pool_py.clone_ref(py), + database_pool_py, self.reactor.clone_ref(py), )) }) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index 20d314cb68..1ff07bdcbf 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -58,6 +58,11 @@ class HomeserverCleanShutdownTestCase(HomeserverTestCase): homeserver_to_use=SynapseHomeServer, clock=self.clock, ) + # Since we're driving the entire homeserver creation in these tests (something + # normally handled by `HomeserverTestCase` if we defined some `servlets`), make + # sure we get the Rust side instantiated to verify nothing on the Rust side is + # holding onto a Python reference that would prevent shutdown. + self.hs.get_rust_handlers() self.wait_for_background_updates() hs_ref = weakref.ref(self.hs) @@ -109,6 +114,11 @@ class HomeserverCleanShutdownTestCase(HomeserverTestCase): homeserver_to_use=SynapseHomeServer, clock=self.clock, ) + # Since we're driving the entire homeserver creation in these tests (something + # normally handled by `HomeserverTestCase` if we defined some `servlets`), make + # sure we get the Rust side instantiated to verify nothing on the Rust side is + # holding onto a Python reference that would prevent shutdown. + self.hs.get_rust_handlers() # Pump the background updates by a single iteration, just to ensure any extra # resources it uses have been started. From e8601840677a1c7ce1ccf42509c4e0dae20fc225 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 7 Jul 2026 11:28:32 +0100 Subject: [PATCH 041/158] Fix `multipart/form-data` mitigation being case-sensitive. Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-6wjm-9p2x-gvpm Fixes: https://github.com/matrix-org/internal-config/issues/1740 Relevant library code to check against: - CPython `EmailMessage.get_content_type()` (https://github.com/python/cpython/blob/v3.13.3/Lib/email/message.py#L608-L630) - Twisted `_parseContentType()` (https://github.com/twisted/twisted/blob/twisted-25.5.0/src/twisted/web/http.py#L306-L313) - Twisted `Request.requestReceived()` (https://github.com/twisted/twisted/blob/twisted-25.5.0/src/twisted/web/http.py#L1079-L1097) ----- Signed-off-by: Olivier 'reivilibre Reviewed-on: https://github.com/element-hq/synapse-private/pull/133 --- synapse/http/site.py | 2 +- tests/http/test_site.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/synapse/http/site.py b/synapse/http/site.py index 9b7fd5c936..4b841a21e2 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -278,7 +278,7 @@ class SynapseRequest(Request): # See: https://github.com/element-hq/synapse/security/advisories/GHSA-rfq8-j7rh-8hf2 if command == b"POST": ctype = self.requestHeaders.getRawHeaders(b"content-type") - if ctype and b"multipart/form-data" in ctype[0]: + if ctype and b"multipart/form-data" in ctype[0].lower(): logger.warning( "Aborting connection from %s because `content-type: multipart/form-data` is unsupported: %s %s", self.client, diff --git a/tests/http/test_site.py b/tests/http/test_site.py index 654ec3190b..c93aadfee1 100644 --- a/tests/http/test_site.py +++ b/tests/http/test_site.py @@ -19,6 +19,8 @@ # # +from parameterized import parameterized + from twisted.internet.address import IPv6Address from twisted.internet.testing import MemoryReactor, StringTransport @@ -92,7 +94,16 @@ class SynapseRequestTestCase(HomeserverTestCase): # that. self.assertEqual(sent, 50 * 1024 * 1024 + 1024) - def test_content_type_multipart(self) -> None: + @parameterized.expand( + [ + (b"multipart/form-data",), + # Also check with a boundary + (b"multipart/form-data; boundary=abc123",), + # Headers are case-insensitive, so test that too. + (b"Multipart/Form-Data",), + ] + ) + def test_content_type_multipart(self, content_type: bytes) -> None: """HTTP POST requests with `content-type: multipart/form-data` should be rejected""" self.hs.start_listening() @@ -133,7 +144,7 @@ class SynapseRequestTestCase(HomeserverTestCase): b"POST / HTTP/1.1\r\n" b"Connection: close\r\n" b"Transfer-Encoding: chunked\r\n" - b"Content-Type: multipart/form-data\r\n" + b"Content-Type: " + content_type + b"\r\n" b"\r\n" b"0\r\n" b"\r\n" From e39303af40afc495cb433aaf61de8a5b77f9704a Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 14 Jul 2026 11:42:19 +0100 Subject: [PATCH 042/158] Fix some paths being accessible at unintended locations with extra prefix components Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-vh4c-pqh4-w3wq Fixes: https://github.com/matrix-org/internal-config/issues/1703 The key thing to understand is that in `synapse/util/httpresourcetree.py`, we create `UnrecognizedRequestResource` and then dangle children (with real resources) off them. Since `UnrecognizedRequestResource` returns itself as a catch-all 'dynamic child', this means any `UnrecognizedRequestResource`s with real children can have unlimited path components inserted between it and its child. So `/_matrix/INSERTED/static/client/login/style.css` or `/_matrix/INSERTED/AS/MANY/AS/I/WANT/static/client/login/style.css` would unexpectedly resolve to the resource. Client, Federation and Admin APIs wouldn't have been affected because you wouldn't get through the regex routing that they use. ----- Reviewed-on: https://github.com/element-hq/synapse-private/pull/143 --- synapse/http/server.py | 30 ++++++++++++- tests/util/test_httpresourcetree.py | 66 +++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 tests/util/test_httpresourcetree.py diff --git a/synapse/http/server.py b/synapse/http/server.py index 2c235e04f4..a0ae20be16 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -33,6 +33,7 @@ from typing import ( Any, Awaitable, Callable, + Final, Iterable, Iterator, Pattern, @@ -673,8 +674,33 @@ class UnrecognizedRequestResource(resource.Resource): # or the response bytes as a return value. return NOT_DONE_YET - def getChild(self, name: str, request: Request) -> resource.Resource: - return self + def getChild(self, path: str, request: Request) -> resource.Resource: + # The child of a catch-all unrecognised request handler + # is itself another unrecognised request handler. + # We can return any UnrecognizedRequestResource that doesn't + # have children. + assert len(_BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE.children) == 0 + return _BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE + + +class _LeafUnrecognisedRequestResource(UnrecognizedRequestResource): + """ + UnrecognizedRequestResource, but with the added caveat that it can't have any children. + This makes it safe for it to return itself as a dynamic child. + + Constructed as a singleton; use `_BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE` + """ + + def putChild(self, path: bytes, child: IResource) -> None: + raise RuntimeError("_LeafUnrecognisedRequestResource does not accept children") + + +_BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE: Final[_LeafUnrecognisedRequestResource] = ( + _LeafUnrecognisedRequestResource() +) +""" +An UnrecognizedRequestResource that is guaranteed not to have children. +""" class RootRedirect(resource.Resource): diff --git a/tests/util/test_httpresourcetree.py b/tests/util/test_httpresourcetree.py new file mode 100644 index 0000000000..b5dc2e0657 --- /dev/null +++ b/tests/util/test_httpresourcetree.py @@ -0,0 +1,66 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import os +from http import HTTPStatus + +from twisted.web.resource import Resource + +import synapse +from synapse.api.errors import Codes +from synapse.api.urls import STATIC_PREFIX +from synapse.http.server import StaticResource + +from tests import unittest + + +class ResourceTreeTestCase(unittest.HomeserverTestCase): + servlets = [] + + def create_resource_dict(self) -> dict[str, Resource]: + """ + Register /_matrix/static for the test. + """ + resources = super().create_resource_dict() + resources[STATIC_PREFIX] = StaticResource( + # as in `synapse/app/homeserver.py` `_configure_named_resource` + os.path.join(os.path.dirname(synapse.__file__), "static") + ) + return resources + + def test_inserted_segment_is_silently_swallowed(self) -> None: + """ + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-vh4c-pqh4-w3wq + + The path `/_matrix/INSERTED/static/client/login/style.css` used to resolve to the same + as `/_matrix/static/client/login/style.css`. + """ + PATH_SUFFIX = "/static/client/login/style.css" + correct_channel = self.make_request( + "GET", + f"/_matrix{PATH_SUFFIX}", + shorthand=False, + ) + # The correct path should give a 200 OK static resource + self.assertEqual(correct_channel.code, HTTPStatus.OK, correct_channel.result) + + wrong_channel = self.make_request( + "GET", + f"/_matrix/INSERTED{PATH_SUFFIX}", + shorthand=False, + ) + # This prefixed version of the same path should give a 404 + self.assertEqual(wrong_channel.code, HTTPStatus.NOT_FOUND, wrong_channel.result) + self.assertEqual( + wrong_channel.json_body["errcode"], Codes.UNRECOGNIZED, wrong_channel.result + ) From 51988c509fed850f868836188f935d3b83e6e9ef Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 14 Jul 2026 12:04:44 +0100 Subject: [PATCH 043/158] Fix spoofed to-device messages over federation not being dropped. Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-rgv2-84w7-5j9p Fixes: https://github.com/matrix-org/internal-config/issues/1520 Introduced in: d4a35ada28302e096efd42e1a2a28542ed7ebd6f The code was obviously intended to drop them, but the if block only logged without actually taking any action. ----- Reviewed-on: https://github.com/element-hq/synapse-private/pull/145 --- synapse/handlers/devicemessage.py | 1 + tests/rest/client/test_sendtodevice.py | 54 ++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 63f73bb832..6a64c1ccf7 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -121,6 +121,7 @@ class DeviceMessageHandler: origin, sender_user_id, ) + return message_type = content["type"] message_id = content["message_id"] for user_id, by_device in content["messages"].items(): diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index c06c7312f2..e9b607cf23 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -489,6 +489,60 @@ class SendToDeviceTestCase(HomeserverTestCase): }, ) + def test_remote_spoofed_sender(self) -> None: + """ + Tests that a to-device message whose sender domain does not match the origin + server is dropped. + """ + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + federation_registry = self.hs.get_federation_registry() + + # Send a spoofed to-device message EDU + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "example.org", + { + "sender": "@user:not.the.same.example.org", + "type": "org.example.test", + "messages": {user2: {"d2": {"foo": "bar"}}}, + "message_id": "1", + }, + ) + ) + # Also send a valid one as a sentinel, to make sure our test setup + # is working as we'd expect + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "example.org", + { + "sender": "@user:example.org", + "type": "org.example.test", + "messages": {user2: {"d2": {"hiss": "meow"}}}, + "message_id": "1", + }, + ) + ) + + # Then do a /sync and be sure it didn't come down + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + messages = channel.json_body.get("to_device", {}).get("events", []) + self.assertEqual( + messages, + [ + # Only the sentinel (valid) to-device message came down, not the spoofed one + { + "content": {"hiss": "meow"}, + "sender": "@user:example.org", + "type": "org.example.test", + } + ], + ) + def test_limited_sync(self) -> None: """If a limited sync for to-devices happens the next /sync should respond immediately.""" From 83672faf9c641cb7a03b78d0af36a9327b90d1a0 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 14 Jul 2026 12:11:54 +0100 Subject: [PATCH 044/158] Add missing 'host in room' check for `/timestamp_to_event/{roomId}` Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-r66v-qhwx-8rg4 Fixes: https://github.com/matrix-org/internal-config/issues/1718 --------- Co-authored-by: Eric Eastwood Reviewed-on: https://github.com/element-hq/synapse-private/pull/144 --- synapse/federation/federation_server.py | 1 + tests/federation/test_federation_server.py | 89 ++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 2fc78c012e..6d8836f71a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -265,6 +265,7 @@ class FederationServer(FederationBase): body including `event_id`. """ async with self._server_linearizer.queue((origin, room_id)): + await self._event_auth_handler.assert_host_in_room(room_id, origin) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 1bf0817bd2..75490d0f0f 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -324,6 +324,95 @@ class StateQueryTests(unittest.FederatingHomeserverTestCase): self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") +class TimestampToEventTests(unittest.FederatingHomeserverTestCase): + """Tests for `GET /_matrix/federation/v1/timestamp_to_event/`.""" + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # Create a room and join the remote server so it's allowed to query + user = self.register_user("u1", "pass") + tok = self.login("u1", "pass") + self.room_id = self.helper.create_room_as(user, tok=tok) + # Send one event at time = 1000s + self.reactor.advance(1000) + self.event_at_1000 = self.helper.send_messages(self.room_id, 1, tok=tok)[0] + + # Send another event at time = 4000s + self.reactor.advance(3000) + self.event_at_4000 = self.helper.send_messages(self.room_id, 1, tok=tok)[0] + + # Send another event at time = 8000s + self.reactor.advance(4000) + self.event_at_8000 = self.helper.send_messages(self.room_id, 1, tok=tok)[0] + + super().prepare(reactor, clock, hs) + + @parameterized.expand( + [ + # Query backwards from 5000s, should find the event at 4000s + (5000000, "b"), + # Query forwards from 1100s, should find the event at 4000s + (1100000, "f"), + ] + ) + def test_happy_path(self, ts: int, dir: str) -> None: + """ + Tests that a server in the room gets 200 OK + with the closest event IDs as requested for a given timestamp, + in both forward and backward directions. + """ + # Join the remote server to the room + self.inject_room_member(self.room_id, "@user:" + self.OTHER_SERVER_NAME, "join") + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/timestamp_to_event/{self.room_id}?ts={ts}&dir={dir}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + self.assertEqual(channel.json_body["event_id"], self.event_at_4000) + + @parameterized.expand( + [ + # Query backwards at 0s, no events to be found. + (0, "b"), + # Query forwards from 8100s, no events to be found. + (8100000, "f"), + ] + ) + def test_no_matching_event(self, ts: int, dir: str) -> None: + """ + Tests that a 404 / M_NOT_FOUND is returned when no event occurs + in the requested direction of a timestamp. + """ + # Join the remote server to the room + self.inject_room_member(self.room_id, "@user:" + self.OTHER_SERVER_NAME, "join") + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/timestamp_to_event/{self.room_id}?ts={ts}&dir={dir}", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_requires_server_in_room(self) -> None: + """ + Tests that a server not in the room is rejected with 403 / M_FORBIDDEN. + """ + # Notably: _don't_ join the remote server to the room + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/timestamp_to_event/{self.room_id}?ts=2000000&dir=b", + ) + self.assertEqual(channel.code, HTTPStatus.FORBIDDEN, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") + + class UnstableGetExtremitiesTests(unittest.FederatingHomeserverTestCase): servlets = [ admin.register_servlets, From 3772859fbcaac6ffa2baeea0025307fbdc192cbb Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 14 Jul 2026 12:14:08 +0100 Subject: [PATCH 045/158] Add missing same-room check on `/get_missing_events` handler Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-27p5-4f45-gx76 Fixes: https://github.com/matrix-org/internal-config/issues/1717 I've tried my best to make the tests a good narrative for the thought process here, but essentially the rationale is to make `/get_missing_events` not distinguish between 'event is in wrong room' and 'event is unknown to me'. ----- Reviewed-on: https://github.com/element-hq/synapse-private/pull/141 --- .../databases/main/event_federation.py | 50 ++- tests/federation/test_federation_server.py | 289 +++++++++++++++++- 2 files changed, 331 insertions(+), 8 deletions(-) diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index d84c58dcf8..4cb55f8c46 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1983,6 +1983,13 @@ class EventFederationWorkerStore( latest_events: list[str], limit: int, ) -> list[EventBase]: + """ + Walk backwards in the DAG of events, + starting at `latest_events` and stopping at `earliest_events` (or when having reached `limit` events). + + This function will check that `latest_events` and `earliest_events` are in the correct + room (`room_id`), appropriately ignoring any that aren't. + """ ids = await self.db_pool.runInteraction( "get_missing_events", self._get_missing_events, @@ -2001,20 +2008,49 @@ class EventFederationWorkerStore( latest_events: list[str], limit: int, ) -> list[str]: + # It's OK that this has not been filtered by correct-room, + # because we will only compare based on event ID from the events + # we happen to run into. seen_events = set(earliest_events) - front = set(latest_events) - seen_events - event_results: list[str] = [] - query = ( - "SELECT prev_event_id FROM event_edges " - "WHERE event_id = ? AND NOT is_state " - "LIMIT ?" + # Pre-filter the `latest_events` to only include those + # that are in this room (and that we know about) + # This makes events in the wrong room get treated the same as unknown events. + events_clause, events_args = make_in_list_sql_clause( + self.database_engine, + "event_id", + # Don't waste time looking at events that the requester told us + # they already know about. + # (They probably shouldn't send this in the first place) + set(latest_events) - seen_events, ) + txn.execute( + f""" + SELECT event_id + FROM events + WHERE {events_clause} AND room_id = ? + """, + (*events_args, room_id), + ) + # Start walking back from the legitimate and known `latest_events` + front = {latest_event_id for (latest_event_id,) in txn} + + event_results: list[str] = [] while front and len(event_results) < limit: new_front = set() for event_id in front: - txn.execute(query, (event_id, limit - len(event_results))) + txn.execute( + """ + SELECT ee.prev_event_id FROM event_edges AS ee + JOIN events ON events.event_id = ee.prev_event_id + WHERE ee.event_id = ? + AND events.room_id = ? + AND NOT ee.is_state + LIMIT ? + """, + (event_id, room_id, limit - len(event_results)), + ) new_results = {t[0] for t in txn} - seen_events new_front |= new_results diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 75490d0f0f..cdb05a9b7a 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -38,7 +38,7 @@ from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.storage.controllers.state import server_acl_evaluator_from_event -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from tests import unittest @@ -94,6 +94,293 @@ class FederationServerTests(unittest.FederatingHomeserverTestCase): self.assertEqual(500, channel.code, channel.result) +class GetMissingEventsRoomCheckTests(unittest.FederatingHomeserverTestCase): + """ + Regression tests for room confusion in /get_missing_events + https://github.com/element-hq/synapse/security/advisories/GHSA-27p5-4f45-gx76 + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + # Local user + self.local_user_id = self.register_user("alice", "pass") + self.local_user_token = self.login("alice", "pass") + self.local_user = UserID.from_string(self.local_user_id) + + # Create 2 rooms (one with the remote server, one without). + # - The remote server will be in this room + self.room_allowed = self.helper.create_room_as( + self.local_user_id, tok=self.local_user_token + ) + self.inject_room_member( + self.room_allowed, f"@remote:{self.OTHER_SERVER_NAME}", "join" + ) + # - The remote server will _not_ be in this room + self.room_blocked = self.helper.create_room_as( + self.local_user_id, tok=self.local_user_token + ) + + # Insert a linear chain of events in both rooms + self.room_allowed_event_ids = self.helper.send_messages( + self.room_allowed, num_events=5, tok=self.local_user_token + ) + self.room_blocked_event_ids = self.helper.send_messages( + self.room_blocked, num_events=5, tok=self.local_user_token + ) + + def _extract_returned_event_ids(self, json_body: JsonDict) -> set[str]: + """ + Given the response body of `/get_missing_events`, return the event IDs + of the events that were returned in the response. + This only includes event IDs from `self.room_allowed_event_ids` and + `self.room_blocked_event_ids`; other events are ignored. + + As the federation PDU format doesn't include event IDs + (at least not for every room version), we match on the + `(room_id, content.body, prev_events)` triple against the events + we sent in the setup. + """ + store = self.hs.get_datastores().main + events = self.get_success( + store.get_events_as_list( + list(self.room_allowed_event_ids) + list(self.room_blocked_event_ids) + ) + ) + # (room_id, content.body, prev_events) -> event ID + event_lookup: dict[tuple[str, str, tuple[str, ...]], str] = {} + for event in events: + key = ( + event.room_id, + event.content["body"], + tuple(event.prev_event_ids()), + ) + event_lookup[key] = event.event_id + + returned_event_ids: set[str] = set() + for pdu in json_body["events"]: + key = ( + pdu.get("room_id"), + pdu.get("content", {}).get("body"), + tuple(pdu.get("prev_events", [])), + ) + event_id = event_lookup.get(key) + if event_id is None: + # Not one of the events we created; ignore it. + continue + returned_event_ids.add(event_id) + return returned_event_ids + + def test_get_missing_events_returns_events_from_correct_room(self) -> None: + """ + Tests the happy path when `latest_events` and `earliest_events` + are both in the correct room. + + returned + | + v + e1 <- e2 <- e3 <- e4 <- e5 + ^ ^ + | | + earliest latest + + Not a regression test; I'm just filling a gap in our (in-repo) testing + as far as I can tell. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [self.room_allowed_event_ids[1]], + "latest_events": [self.room_allowed_event_ids[3]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + {self.room_allowed_event_ids[2]}, + ) + + def test_get_missing_events_with_empty_earliest_events(self) -> None: + """ + Tests that `/get_missing_events`, when given no `earliest_events`, + walks back to the start of the room, capped at `limit`. + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [], + "latest_events": [self.room_allowed_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + set(self.room_allowed_event_ids[:-1]), + ) + + def test_get_missing_events_with_unknown_earliest_event(self) -> None: + """ + Tests that `/get_missing_events` ignores unknown event IDs given in + `earliest_events`. + + This makes sense as the `earliest_events` are intuitively + 'events to stop at' when walking backwards. + Since we don't know about those events, we don't use them as stopping conditions. + (In other words, this falls back to the same behaviour as + `test_get_missing_events_with_empty_earliest_events`.) + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": ["$someUnknownEventId"], + "latest_events": [self.room_allowed_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + set(self.room_allowed_event_ids[:-1]), + ) + + def test_get_missing_events_with_no_latest_event(self) -> None: + """ + Tests that when the `/get_missing_events` request references + no events in `latest_events`, the response is 200 OK + with an empty `events` list. + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": ["$someOtherUnknownEventId"], + "latest_events": [], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_with_unknown_latest_event(self) -> None: + """ + Tests that when the `/get_missing_events` request references + unknown events in `latest_events`, the response is 200 OK + with an empty `events` list. + + I imagine this makes sense as you might request several events + in `latest_events` to start walking back from and we need to be + tolerant of the fact that servers don't always know about every event. + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": ["$someOtherUnknownEventId"], + "latest_events": ["$someUnknownEventId"], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_ignores_events_from_other_room(self) -> None: + """ + Tests that providing `earliest_events` and `latest_events` from the wrong room + treats them the same as being unknown. + + From `test_get_missing_events_with_unknown_latest_event` we established that + unknown events in `latest_events` get skipped (to the point of returning an empty + `events: []` response) + + From `test_get_missing_events_with_unknown_earliest_event` we established that + unknown events in `earliest_events` get ignored as stopping conditions. + + This regression test previously failed. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [self.room_blocked_event_ids[0]], + "latest_events": [self.room_blocked_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_skips_latest_events_from_other_room(self) -> None: + """ + Tests that providing `latest_events` from the wrong room + treats it as being unknown, even if `earliest_events` are from the correct + room. + + From `test_get_missing_events_with_unknown_latest_event` we established that + unknown events in `latest_events` get skipped (to the point of returning an empty + `events: []` response) + + This regression test previously failed. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [self.room_allowed_event_ids[0]], + "latest_events": [self.room_blocked_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_ignores_earliest_events_from_other_room(self) -> None: + """ + Tests that providing `earliest_events` from the wrong room causes those + events to be ignored as stopping conditions, + even though `latest_events` are from the correct room. + + From `test_get_missing_events_with_unknown_earliest_event` we established that + unknown events in `earliest_events` get ignored as stopping conditions. + + This test was previously fine, but is an obvious extra case. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + # Use [-3] here as we want to see if the walk-back algorithm + # confuses depth (topological ordering) across the two rooms. + "earliest_events": [self.room_blocked_event_ids[-3]], + "latest_events": [self.room_allowed_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + set(self.room_allowed_event_ids[:-1]), + ) + + def _create_acl_event(content: JsonDict) -> EventBase: return make_test_event( { From 68139e5234724c441b3b3beaa30a644d4b546fae Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 16 Jul 2026 12:26:23 +0100 Subject: [PATCH 046/158] Add missing 'event in expected room' check on `/event_auth` Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-qcjr-46gf-7f4r Fixes: https://github.com/matrix-org/internal-config/issues/1714 The `/event_auth` endpoint could be tricked to give you the auth chain for an event in a foreign room, because it trusted the requester to provide the correct `room_id` for the event. Now we pass the `room_id` through all the way to `get_event`'s `check_room_id`, which (correctly IMO) treats mismatches as unknown events (seems correct as it prevents divulging what events we know about). The `test_event_auth_wrong_room_returns_404` test failed before the fix. ----- Reviewed-on: https://github.com/element-hq/synapse-private/pull/147 --- synapse/federation/federation_server.py | 2 +- synapse/handlers/federation.py | 4 +- tests/federation/test_federation_server.py | 80 +++++++++++++++++++++- 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 6d8836f71a..83566050f6 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -1129,7 +1129,7 @@ class FederationServer(FederationBase): await self.check_server_matches_acl(origin_host, room_id) time_now = self._clock.time_msec() - auth_pdus = await self.handler.on_event_auth(event_id) + auth_pdus = await self.handler.on_event_auth(event_id, room_id) res = {"auth_chain": serialize_and_filter_pdus(auth_pdus, time_now)} return 200, res diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ba83d4fd26..a99daaba48 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -582,8 +582,8 @@ class FederationHandler: return pdu - async def on_event_auth(self, event_id: str) -> list[EventBase]: - event = await self.store.get_event(event_id) + async def on_event_auth(self, event_id: str, room_id: str) -> list[EventBase]: + event = await self.store.get_event(event_id, check_room_id=room_id) auth = await self.store.get_auth_chain( event.room_id, list(event.auth_event_ids()), include_given=True ) diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index cdb05a9b7a..6056448f20 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -27,7 +27,7 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import FederationError +from synapse.api.errors import Codes, FederationError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.crypto.event_signing import add_hashes_and_signatures @@ -849,6 +849,84 @@ class UnstableGetExtremitiesTests(unittest.FederatingHomeserverTestCase): self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED") +class EventAuthFederationTests(unittest.FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # Create a local user + self.user_id = self.register_user("alice", "password") + self.user_tok = self.login("alice", "password") + + # Set up a room and join the remote server to it + self.room_id = self.helper.create_room_as( + self.user_id, + is_public=True, + room_version=RoomVersions.V10.identifier, + tok=self.user_tok, + ) + self.inject_room_member( + self.room_id, f"@remote:{self.OTHER_SERVER_NAME}", Membership.JOIN + ) + + # Create a known event whose auth chain we can request back. + self.event_id = self.helper.send_messages( + self.room_id, num_events=1, tok=self.user_tok + )[0] + + return super().prepare(reactor, clock, hs) + + def test_event_auth_unknown_event_returns_404(self) -> None: + """ + Tests that requesting the auth chain of an unknown event + returns 404 / M_NOT_FOUND. + """ + + # Request an event that doesn't exist in self.room_id. + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/event_auth/{self.room_id}/$unknownevent", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual( + channel.json_body["errcode"], Codes.NOT_FOUND, channel.json_body + ) + + def test_event_auth_wrong_room_returns_404(self) -> None: + """ + Tests that a request whose `room_id` is wrong for the event + acts the same as though it were an unknown event. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-qcjr-46gf-7f4r + """ + + # Create a second room with its own event. + other_room_id = self.helper.create_room_as( + self.user_id, + is_public=True, + room_version=RoomVersions.V10.identifier, + tok=self.user_tok, + ) + other_room_event_id = self.helper.send_messages( + other_room_id, num_events=1, tok=self.user_tok + )[0] + + # Request the chain of other_room_id's event, but pretend it's part of the room + # we are in. + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/event_auth/{self.room_id}/{other_room_event_id}", + ) + + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual( + channel.json_body["errcode"], Codes.NOT_FOUND, channel.json_body + ) + + class SendJoinFederationTests(unittest.FederatingHomeserverTestCase): servlets = [ admin.register_servlets, From cbc6934821aab314506c5957223c60c74eeda091 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 23 Jul 2026 18:13:15 +0100 Subject: [PATCH 047/158] Prevent theft of room aliases via remote join to room with illegitimate predecessor Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-cjh7-rcpx-xpf8 Fixes: https://github.com/matrix-org/internal-config/issues/1729 --------- Co-authored-by: Eric Eastwood Reviewed-on: https://github.com/element-hq/synapse-private/pull/136 --- synapse/handlers/federation.py | 49 ++- tests/federation/_remote_join.py | 354 ++++++++++++++++++ .../test_federation_join_upgraded_room.py | 308 +++++++++++++++ 3 files changed, 709 insertions(+), 2 deletions(-) create mode 100644 tests/federation/_remote_join.py create mode 100644 tests/federation/test_federation_join_upgraded_room.py diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index a99daaba48..261ccd909b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -797,10 +797,55 @@ class FederationHandler: if not predecessor or not isinstance(predecessor.get("room_id"), str): return event.event_id, max_stream_id old_room_id = predecessor["room_id"] - logger.debug( - "Found predecessor for %s during remote join: %s", room_id, old_room_id + + # We can't take the new room's word for it. + # Check to see that the predecessor room consents to the + # room upgrade. + if not await self._event_auth_handler.is_host_in_room( + room_id=old_room_id, host=self.hs.hostname + ): + logger.info( + "Ignoring unverified predecessor for %s during remote join: %s (not in old room)", + room_id, + old_room_id, + ) + return event.event_id, max_stream_id + + tombstone = await self._state_storage_controller.get_current_state_event( + old_room_id, + event_type=EventTypes.Tombstone, + state_key="", ) + if tombstone is None: + logger.warning( + "Ignoring unverified predecessor for %s during remote join: %s (no tombstone in old room)", + room_id, + old_room_id, + ) + return event.event_id, max_stream_id + + intended_successor_room = tombstone.content.get( + EventContentFields.TOMBSTONE_SUCCESSOR_ROOM, None + ) + + if not isinstance(intended_successor_room, str): + logger.warning( + "Ignoring unverified predecessor for %s during remote join: %s (tombstone is invalid)", + room_id, + old_room_id, + ) + return event.event_id, max_stream_id + + if intended_successor_room != room_id: + logger.warning( + "Ignoring unverified predecessor for %s during remote join: predecessor defined as %s (the old room ID) but the old room's tombstone points to %r which doesn't match", + room_id, + old_room_id, + intended_successor_room, + ) + return event.event_id, max_stream_id + # We retrieve the room member handler here as to not cause a cyclic dependency member_handler = self.hs.get_room_member_handler() await member_handler.transfer_room_state_on_room_upgrade( diff --git a/tests/federation/_remote_join.py b/tests/federation/_remote_join.py new file mode 100644 index 0000000000..94fa44e200 --- /dev/null +++ b/tests/federation/_remote_join.py @@ -0,0 +1,354 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +import urllib.parse +from http import HTTPStatus +from typing import Callable, TypeVar +from unittest.mock import Mock + +import attr + +from synapse.api.constants import EventContentFields, EventTypes, Membership +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.events import EventBase +from synapse.events.utils import strip_event +from synapse.federation.transport.client import SendJoinResponse +from synapse.http.matrixfederationclient import ByteParser +from synapse.http.types import QueryParams +from synapse.types import JsonDict + +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event +from tests.unittest import FederatingHomeserverTestCase + +logger = logging.getLogger(__name__) + + +@attr.s(slots=True, auto_attribs=True) +class RemoteStateEvent: + """ + A state event to insert into the remote room `/send_join` response. + """ + + type: str + state_key: str + content: JsonDict + # Defaults to the remote room creator. + sender: str | None = None + + +class RemoteJoinHelper: + """ + Helps a `FederatingHomeserverTestCase` join a remote (non-resident) room + over federation. + + Spiritually inspired by `test_federation_out_of_band_membership.py` + but in a more reusable form. + + Constructor Args: + test_case: the current `FederatingHomeserverTestCase` + federation_http_client: Mocked form of the main test homeserver's federation HTTP client; + should have method slots for `get_json` and `put_json`. + remote_creator_user_id: User ID of the remote user creating the room + room_version: Desired room version + create_content: Desired `content` of the `m.room.create` event + state_events: Extra state events to create in the mock room + (They will be made available in the `/send_join` response) + + Usage: + + helper = RemoteJoinHelper( + self, + create_content={"predecessor": {"room_id": old_room_id, ...}}, + ) + # helper.room_id is available now; you can e.g. point a tombstone at it. + helper.join(local_user_id, local_user_tok) + """ + + room_id: str + """ + The room ID of the mock remote room that will be joined. + """ + + def __init__( + self, + test_case: FederatingHomeserverTestCase, + federation_http_client: Mock, + *, + remote_creator_user_id: str | None = None, + room_version: RoomVersion = RoomVersions.V10, + create_content: JsonDict | None = None, + state_events: list[RemoteStateEvent] | None = None, + ) -> None: + if remote_creator_user_id is None: + remote_creator_user_id = f"@remote-user:{test_case.OTHER_SERVER_NAME}" + + self._test_case = test_case + self._federation_http_client = federation_http_client + self._remote_creator_user_id = remote_creator_user_id + self._room_version = room_version + + # 1. Create the room creation event + create_content_full: JsonDict = { + EventContentFields.ROOM_CREATOR: remote_creator_user_id, + EventContentFields.ROOM_VERSION: room_version.identifier, + } + if create_content is not None: + create_content_full.update(create_content) + + create_event_dict: JsonDict = { + "sender": remote_creator_user_id, + "depth": 1, + "origin_server_ts": 1, + "type": EventTypes.Create, + "state_key": "", + "content": create_content_full, + "auth_events": [], + "prev_events": [], + } + if not room_version.msc4291_room_ids_as_hashes: + # For room versions that _don't_ derive the room ID from the content, + # we need to set our own. + # We could consider exposing a parameter to allow varying the localpart + # (and perturb the create event for hashes-as-room-ID rooms) + create_event_dict["room_id"] = f"!remote-room:{test_case.OTHER_SERVER_NAME}" + + room_create_event = make_test_event( + test_case.add_hashes_and_signatures_from_other_server( + create_event_dict, + ), + room_version=room_version, + ) + + self.room_id = room_create_event.room_id + + # 2. Create the room creator's membership event + creator_membership_event = make_test_event( + test_case.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": remote_creator_user_id, + "depth": 2, + "origin_server_ts": 2, + "type": EventTypes.Member, + "state_key": remote_creator_user_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [room_create_event.event_id], + "prev_events": [room_create_event.event_id], + } + ), + room_version=room_version, + ) + + # 3. Create requested extra state events (in a linear chain from the membership) + extra_state_events: list[EventBase] = [] + prev_event = creator_membership_event + depth = 3 + for spec in state_events or []: + sender = spec.sender or remote_creator_user_id + event = make_test_event( + test_case.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": sender, + "depth": depth, + "origin_server_ts": depth, + "type": spec.type, + "state_key": spec.state_key, + "content": spec.content, + "auth_events": [ + room_create_event.event_id, + creator_membership_event.event_id, + ], + "prev_events": [prev_event.event_id], + } + ), + room_version=room_version, + ) + extra_state_events.append(event) + prev_event = event + depth += 1 + + self._room_create_event = room_create_event + self._creator_membership_event = creator_membership_event + self._extra_state_events = extra_state_events + + def join(self, local_user_id: str, local_user_tok: str) -> None: + """ + Invite `local_user_id` and perform the federation join dance. + """ + remote_room_id = self.room_id + room_version = self._room_version + + room_create_event = self._room_create_event + creator_membership_event = self._creator_membership_event + extra_events = self._extra_state_events + + # 1. Create an invite event and make it appear on the 'real' homeserver + depth = 3 + len(extra_events) + + invite_membership_event = make_test_event( + self._test_case.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": self._remote_creator_user_id, + "depth": depth, + "origin_server_ts": depth, + "type": EventTypes.Member, + "state_key": local_user_id, + "content": {"membership": Membership.INVITE}, + "auth_events": [ + room_create_event.event_id, + creator_membership_event.event_id, + ], + "prev_events": [ + extra_events[-1].event_id + if extra_events + else creator_membership_event.event_id + ], + } + ), + room_version=room_version, + ) + + channel = self._test_case.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/invite/{remote_room_id}/{invite_membership_event.event_id}", + content={ + "event": invite_membership_event.get_dict(), + "invite_room_state": [ + strip_event(room_create_event), + ], + "room_version": room_version.identifier, + }, + ) + assert channel.code == HTTPStatus.OK, channel.json_body + + # 2. Mock `/make_join` and `/send_join`. + # Start by creating a join membership event. + join_membership_event_template = make_test_event( + { + "room_id": remote_room_id, + "sender": local_user_id, + "depth": depth + 1, + "origin_server_ts": depth + 1, + "type": EventTypes.Member, + "state_key": local_user_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + room_create_event.event_id, + invite_membership_event.event_id, + ], + "prev_events": [invite_membership_event.event_id], + }, + room_version=room_version, + ) + + T = TypeVar("T") + + async def _get_json( + destination: str, + path: str, + args: QueryParams | None = None, + retry_on_dns_fail: bool = True, + timeout: int | None = None, + ignore_backoff: bool = False, + try_trailing_slash_on_400: bool = False, + parser: ByteParser[T] | None = None, + ) -> JsonDict | T: + make_join_path = ( + f"/_matrix/federation/v1/make_join/" + f"{urllib.parse.quote_plus(remote_room_id)}/{urllib.parse.quote_plus(local_user_id)}" + ) + if path == make_join_path: + return { + "event": join_membership_event_template.get_pdu_json(), + "room_version": room_version.identifier, + } + raise NotImplementedError( + "We have not mocked a response for `get_json(...)` for the following endpoint yet: " + + f"{destination}{path}" + ) + + self._federation_http_client.get_json.side_effect = _get_json + + send_join_state = [ + room_create_event, + creator_membership_event, + *extra_events, + invite_membership_event, + ] + + async def _put_json( + destination: str, + path: str, + args: QueryParams | None = None, + data: JsonDict | None = None, + json_data_callback: Callable[[], JsonDict] | None = None, + long_retries: bool = False, + timeout: int | None = None, + ignore_backoff: bool = False, + backoff_on_404: bool = False, + try_trailing_slash_on_400: bool = False, + parser: ByteParser[T] | None = None, + backoff_on_all_error_codes: bool = False, + ) -> JsonDict | T | SendJoinResponse: + if ( + path.startswith( + f"/_matrix/federation/v2/send_join/{urllib.parse.quote_plus(remote_room_id)}/" + ) + and data is not None + and data.get("type") == EventTypes.Member + and data.get("state_key") == local_user_id + and parser is not None + ): + # As the remote server, sign the join event before returning it. + join_membership_event_signed = make_test_event( + self._test_case.add_hashes_and_signatures_from_other_server(data), + room_version=room_version, + ) + return SendJoinResponse( + auth_events=[ + room_create_event, + invite_membership_event, + ], + state=send_join_state, + event_dict=join_membership_event_signed.get_pdu_json(), + event=join_membership_event_signed, + members_omitted=False, + servers_in_room=[ + self._test_case.OTHER_SERVER_NAME, + ], + ) + + if path.startswith("/_matrix/federation/v1/send/") and data is not None: + # Just acknowledge everything. + return { + make_test_pdu_event(pdu, room_version).event_id: {} + for pdu in data.get("pdus", []) + } + + raise NotImplementedError( + "We have not mocked a response for `put_json(...)` for the following endpoint yet: " + + f"{destination}{path} with the following body data: {data}" + ) + + self._federation_http_client.put_json.side_effect = _put_json + + # 3. Issue the client-server API request to join the room + self._test_case.helper.join(remote_room_id, local_user_id, tok=local_user_tok) + + # 4. Reset mocks + self._federation_http_client.get_json.side_effect = None + self._federation_http_client.put_json.side_effect = None diff --git a/tests/federation/test_federation_join_upgraded_room.py b/tests/federation/test_federation_join_upgraded_room.py new file mode 100644 index 0000000000..4557cf1b55 --- /dev/null +++ b/tests/federation/test_federation_join_upgraded_room.py @@ -0,0 +1,308 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from unittest.mock import Mock + +from twisted.internet.testing import MemoryReactor + +from synapse.api.constants import EventTypes +from synapse.rest import admin +from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.types import JsonDict, RoomAlias +from synapse.util.clock import Clock + +from tests import unittest +from tests.federation._remote_join import RemoteJoinHelper + +logger = logging.getLogger(__name__) + + +def _predecessor(room_id: str) -> JsonDict: + """`create_content` for a remote room that claims `room_id` as its predecessor.""" + return { + "predecessor": { + "room_id": room_id, + # inert dummy + "event_id": "$some_tombstone_event:test", + } + } + + +class FederationJoinUpgradedRoomTestCase(unittest.FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self._federation_http_client = Mock( + # The problem with using `spec=MatrixFederationHttpClient` here is that it + # requires everything to be mocked which is a lot of work that I don't want + # to do when the code only uses a few methods (`get_json` and `put_json`). + ) + return self.setup_test_homeserver( + federation_http_client=self._federation_http_client + ) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + self.store = self.hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + + def _room_is_public(self, room_id: str) -> bool: + """`is_public` flag from the `rooms` table (asserts the row exists).""" + room = self.get_success(self.store.get_room(room_id)) + assert room is not None, f"no rooms row for {room_id}" + is_public, _ = room + return is_public + + def test_alias_transferred_on_federation_join_to_upgraded_room(self) -> None: + """ + Tests that joining an upgraded room over federation, + where the predecessor has a valid corresponding tombstone, + should transfer all room aliases from the old room to the new room. + + The old room is set up with a couple of local aliases and a tombstone + event pointing at the new (remote) room. + After the federation join, all room aliases should have been + transferred from the old room to the new room. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up an old room and point 2 room aliases at it + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + + alias1 = RoomAlias.from_string("#old_room:test") + alias2 = RoomAlias.from_string("#old_room_alt:test") + for alias in (alias1, alias2): + self.get_success( + self.store.create_room_alias_association( + alias, old_room_id, [self.hs.hostname] + ) + ) + + # Now set up a replacement room, which we will remote-room-join in a moment + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + create_content=_predecessor(old_room_id), + ) + + # Place a tombstone in the old room, essentially authorising + # the replacement room to replace it + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": join_helper.room_id}, + tok=local_user_tok, + ) + + # Trigger the remote room join + join_helper.join(local_user_id, local_user_tok) + + # The new room should have acquired the aliases... + new_aliases = self.get_success( + self.store.get_aliases_for_room(join_helper.room_id) + ) + self.assertCountEqual(new_aliases, [alias1.to_string(), alias2.to_string()]) + + # ...and the old room therefore must have given them up + old_aliases = self.get_success(self.store.get_aliases_for_room(old_room_id)) + self.assertEqual(old_aliases, []) + + def test_room_directory_visibility_transferred(self) -> None: + """ + On a valid federation join of an upgraded room, the room directory public + flag should move from the old room to the new room. + + A public old room becomes private (so people don't accidentally join it) + and the newly-joined room is marked public. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up an old room and mark it as public (for room directory purposes) + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + self.get_success(self.store.set_room_is_public(old_room_id, True)) + self.assertTrue(self._room_is_public(old_room_id)) + + # Now set up a replacement room, which we will remote-room-join in a moment + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + create_content=_predecessor(old_room_id), + ) + + # Place a tombstone in the old room, essentially authorising + # the replacement room to replace it + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": join_helper.room_id}, + tok=local_user_tok, + ) + + # Trigger the remote room join + join_helper.join(local_user_id, local_user_tok) + + # The room directory publicity should have shifted to the new room + # (So users don't accidentally join the old room from the directory) + self.assertFalse(self._room_is_public(old_room_id)) + self.assertTrue(self._room_is_public(join_helper.room_id)) + + def test_room_directory_visibility_not_transferred_for_private_room(self) -> None: + """ + On a valid federation join of an upgraded room, a private old room should + leave the room directory visibility untouched for both the old and new + rooms (both remain private). + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up an old room and mark it as public (for room directory purposes) + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + self.assertFalse(self._room_is_public(old_room_id)) + + # Now set up a replacement room, which we will remote-room-join in a moment + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + create_content=_predecessor(old_room_id), + ) + + # Place a tombstone in the old room, essentially authorising + # the replacement room to replace it + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": join_helper.room_id}, + tok=local_user_tok, + ) + + # Trigger the remote room join + join_helper.join(local_user_id, local_user_tok) + + self.assertFalse(self._room_is_public(old_room_id)) + self.assertFalse(self._room_is_public(join_helper.room_id)) + + def test_no_transfer_when_predecessor_room_has_no_tombstone(self) -> None: + """ + Tests that when joining a remote room over federation, + if the room has an illegitimate predecessor (a predecessor pointing + to a room that does not have a corresponding tombstone to vouch for it + as the successor), room aliases are not transferred. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up a room with an alias + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + + alias = RoomAlias.from_string("#old_room:test") + self.get_success( + self.store.create_room_alias_association( + alias, old_room_id, [self.hs.hostname] + ) + ) + + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + # The new room (illegitimately) claims to be the successor + # of the old room. + create_content=_predecessor(old_room_id), + ) + + # Notably, we do NOT set up a tombstone in the 'old' room. + + # Do the remote room join dance + join_helper.join(local_user_id, local_user_tok) + + # Check that the room alias did _not_ get transferred... + new_aliases = self.get_success( + self.store.get_aliases_for_room(join_helper.room_id) + ) + self.assertCountEqual(new_aliases, []) + + # ...and that the old room still has it + old_aliases = self.get_success(self.store.get_aliases_for_room(old_room_id)) + self.assertCountEqual(old_aliases, [alias.to_string()]) + + def test_no_transfer_when_tombstone_does_not_match(self) -> None: + """ + A predecessor room whose tombstone points to a different room than the + one being joined must not trigger an alias transfer. + + The tombstone's `replacement_room` must match the joined room for the + upgrade link to be considered valid. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up a room with an alias + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + + alias = RoomAlias.from_string("#old_room:test") + self.get_success( + self.store.create_room_alias_association( + alias, old_room_id, [self.hs.hostname] + ) + ) + + # Tombstone points at a _different_, room. + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": "!the_real_replacement_room:example.com"}, + tok=local_user_tok, + ) + + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + # The new room (illegitimately) claims to be the successor + # of the old room. + create_content=_predecessor(old_room_id), + ) + join_helper.join(local_user_id, local_user_tok) + + # Check that the room alias did _not_ get transferred... + new_aliases = self.get_success( + self.store.get_aliases_for_room(join_helper.room_id) + ) + self.assertCountEqual(new_aliases, []) + + # ...and that the old room still has it + old_aliases = self.get_success(self.store.get_aliases_for_room(old_room_id)) + self.assertCountEqual(old_aliases, [alias.to_string()]) From 4f8037ab592d1ddc4bc5c1841f02096fba3a7ed9 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 24 Jul 2026 11:19:56 +0100 Subject: [PATCH 048/158] Anchor the ends of our path patterns to prevent path traversal vulnerabilities Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-hgcg-p9gx-fq5f Fixes: https://github.com/element-hq/backend-internal/issues/217 -> https://github.com/matrix-org/internal-config/issues/1760 ----- Reviewed-on: https://github.com/element-hq/synapse-private/pull/146 --- synapse/rest/admin/experimental_features.py | 2 +- synapse/rest/admin/users.py | 14 +++++++------- synapse/rest/client/account_data.py | 8 ++++---- synapse/rest/client/appservice_ping.py | 2 +- synapse/rest/client/auth.py | 2 +- synapse/rest/client/devices.py | 2 +- synapse/rest/client/filter.py | 4 ++-- synapse/rest/client/knock.py | 2 +- synapse/rest/client/login.py | 2 +- synapse/rest/client/openid.py | 2 +- synapse/rest/client/presence.py | 2 +- synapse/rest/client/profile.py | 10 +++++----- synapse/rest/client/register.py | 4 ++-- synapse/rest/client/relations.py | 2 +- synapse/rest/client/room.py | 2 +- synapse/rest/client/tags.py | 2 +- synapse/rest/client/thirdparty.py | 2 +- synapse/rest/client/tokenrefresh.py | 2 +- synapse/rest/media/create_resource.py | 2 +- 19 files changed, 34 insertions(+), 34 deletions(-) diff --git a/synapse/rest/admin/experimental_features.py b/synapse/rest/admin/experimental_features.py index abdb937793..c91c5b6a49 100644 --- a/synapse/rest/admin/experimental_features.py +++ b/synapse/rest/admin/experimental_features.py @@ -62,7 +62,7 @@ class ExperimentalFeaturesRestServlet(RestServlet): for a given user """ - PATTERNS = admin_patterns("/experimental_features/(?P[^/]*)") + PATTERNS = admin_patterns("/experimental_features/(?P[^/]*)$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 1bd48e18cc..53d1f2d366 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -1371,7 +1371,7 @@ class RateLimitRestServlet(RestServlet): class AccountDataRestServlet(RestServlet): """Retrieve the given user's account data""" - PATTERNS = admin_patterns("/users/(?P[^/]*)/accountdata") + PATTERNS = admin_patterns("/users/(?P[^/]*)/accountdata$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1409,7 +1409,7 @@ class UserReplaceMasterCrossSigningKeyRestServlet(RestServlet): """ PATTERNS = admin_patterns( - "/users/(?P[^/]*)/_allow_cross_signing_replacement_without_uia" + "/users/(?P[^/]*)/_allow_cross_signing_replacement_without_uia$" ) REPLACEMENT_PERIOD_MS = 10 * 60 * 1000 # 10 minutes @@ -1443,7 +1443,7 @@ class UserByExternalId(RestServlet): """Find a user based on an external ID from an auth provider""" PATTERNS = admin_patterns( - "/auth_providers/(?P[^/]*)/users/(?P[^/]*)" + "/auth_providers/(?P[^/]*)/users/(?P[^/]*)$" ) def __init__(self, hs: "HomeServer"): @@ -1469,7 +1469,7 @@ class UserByExternalId(RestServlet): class UserByThreePid(RestServlet): """Find a user based on 3PID of a particular medium""" - PATTERNS = admin_patterns("/threepid/(?P[^/]*)/users/(?P
[^/]*)") + PATTERNS = admin_patterns("/threepid/(?P[^/]*)/users/(?P
[^/]*)$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1503,7 +1503,7 @@ class RedactUser(RestServlet): If only one parameter is sent, then all messages before or after given time will be redacted. """ - PATTERNS = admin_patterns("/user/(?P[^/]*)/redact") + PATTERNS = admin_patterns("/user/(?P[^/]*)/redact$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1614,7 +1614,7 @@ class UserInvitesCount(RestServlet): Return the count of invites that the user has sent after the given timestamp """ - PATTERNS = admin_patterns("/users/(?P[^/]*)/sent_invite_count") + PATTERNS = admin_patterns("/users/(?P[^/]*)/sent_invite_count$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1639,7 +1639,7 @@ class UserJoinedRoomCount(RestServlet): if they have subsequently left/been banned from those rooms. """ - PATTERNS = admin_patterns("/users/(?P[^/]*)/cumulative_joined_room_count") + PATTERNS = admin_patterns("/users/(?P[^/]*)/cumulative_joined_room_count$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() diff --git a/synapse/rest/client/account_data.py b/synapse/rest/client/account_data.py index b18232fc56..cccc0473e8 100644 --- a/synapse/rest/client/account_data.py +++ b/synapse/rest/client/account_data.py @@ -61,7 +61,7 @@ class AccountDataServlet(RestServlet): """ PATTERNS = client_patterns( - "/user/(?P[^/]*)/account_data/(?P[^/]*)" + "/user/(?P[^/]*)/account_data/(?P[^/]*)$" ) CATEGORY = "Account data requests" @@ -136,7 +136,7 @@ class UnstableAccountDataServlet(RestServlet): PATTERNS = client_patterns( "/org.matrix.msc3391/user/(?P[^/]*)" - "/account_data/(?P[^/]*)", + "/account_data/(?P[^/]*)$", unstable=True, releases=(), ) @@ -174,7 +174,7 @@ class RoomAccountDataServlet(RestServlet): PATTERNS = client_patterns( "/user/(?P[^/]*)" "/rooms/(?P[^/]*)" - "/account_data/(?P[^/]*)" + "/account_data/(?P[^/]*)$" ) CATEGORY = "Account data requests" @@ -271,7 +271,7 @@ class UnstableRoomAccountDataServlet(RestServlet): PATTERNS = client_patterns( "/org.matrix.msc3391/user/(?P[^/]*)" "/rooms/(?P[^/]*)" - "/account_data/(?P[^/]*)", + "/account_data/(?P[^/]*)$", unstable=True, releases=(), ) diff --git a/synapse/rest/client/appservice_ping.py b/synapse/rest/client/appservice_ping.py index 2c6ad5bcf0..4cbf6935c5 100644 --- a/synapse/rest/client/appservice_ping.py +++ b/synapse/rest/client/appservice_ping.py @@ -46,7 +46,7 @@ logger = logging.getLogger(__name__) class AppservicePingRestServlet(RestServlet): PATTERNS = client_patterns( - "/appservice/(?P[^/]*)/ping", + "/appservice/(?P[^/]*)/ping$", releases=("v1",), ) diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index b1775346f6..215779a355 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -47,7 +47,7 @@ class AuthRestServlet(RestServlet): Current use is for web fallback auth. """ - PATTERNS = client_patterns(r"/auth/(?P[\w\.]*)/fallback/web") + PATTERNS = client_patterns(r"/auth/(?P[\w\.]*)/fallback/web$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 3d766cda3f..115ad543fc 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -77,7 +77,7 @@ class DeleteDevicesRestServlet(RestServlet): key which lists the device_ids to delete. Requires user interactive auth. """ - PATTERNS = client_patterns("/delete_devices") + PATTERNS = client_patterns("/delete_devices$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/filter.py b/synapse/rest/client/filter.py index cfe82e1473..6718daaf23 100644 --- a/synapse/rest/client/filter.py +++ b/synapse/rest/client/filter.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) class GetFilterRestServlet(RestServlet): - PATTERNS = client_patterns("/user/(?P[^/]*)/filter/(?P[^/]*)") + PATTERNS = client_patterns("/user/(?P[^/]*)/filter/(?P[^/]*)$") CATEGORY = "Encryption requests" def __init__(self, hs: "HomeServer"): @@ -76,7 +76,7 @@ class GetFilterRestServlet(RestServlet): class CreateFilterRestServlet(RestServlet): - PATTERNS = client_patterns("/user/(?P[^/]*)/filter") + PATTERNS = client_patterns("/user/(?P[^/]*)/filter$") CATEGORY = "Encryption requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/knock.py b/synapse/rest/client/knock.py index cd3afda11e..cce856f0ef 100644 --- a/synapse/rest/client/knock.py +++ b/synapse/rest/client/knock.py @@ -46,7 +46,7 @@ class KnockRoomAliasServlet(RestServlet): POST /knock/{roomIdOrAlias} """ - PATTERNS = client_patterns("/knock/(?P[^/]*)") + PATTERNS = client_patterns("/knock/(?P[^/]*)$") CATEGORY = "Event sending requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index aaf26bac6f..87730fc512 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -708,7 +708,7 @@ class SsoRedirectServlet(RestServlet): class CasTicketServlet(RestServlet): - PATTERNS = client_patterns("/login/cas/ticket", v1=True) + PATTERNS = client_patterns("/login/cas/ticket$", v1=True) def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/openid.py b/synapse/rest/client/openid.py index e624a48ce7..b6e2c078d8 100644 --- a/synapse/rest/client/openid.py +++ b/synapse/rest/client/openid.py @@ -67,7 +67,7 @@ class IdTokenServlet(RestServlet): } """ - PATTERNS = client_patterns("/user/(?P[^/]*)/openid/request_token") + PATTERNS = client_patterns("/user/(?P[^/]*)/openid/request_token$") EXPIRES_MS = 3600 * 1000 diff --git a/synapse/rest/client/presence.py b/synapse/rest/client/presence.py index de3ffdaa0b..4565280113 100644 --- a/synapse/rest/client/presence.py +++ b/synapse/rest/client/presence.py @@ -40,7 +40,7 @@ logger = logging.getLogger(__name__) class PresenceStatusRestServlet(RestServlet): - PATTERNS = client_patterns("/presence/(?P[^/]*)/status", v1=True) + PATTERNS = client_patterns("/presence/(?P[^/]*)/status$", v1=True) CATEGORY = "Presence requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index c2ec5b3611..360a83f39c 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -58,7 +58,7 @@ def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: class ProfileRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)", v1=True) + PATTERNS = client_patterns("/profile/(?P[^/]*)$", v1=True) CATEGORY = "Event sending requests" def __init__(self, hs: "HomeServer"): @@ -92,13 +92,13 @@ class ProfileRestServlet(RestServlet): class ProfileFieldRestServlet(RestServlet): PATTERNS = [ *client_patterns( - "/profile/(?P[^/]*)/(?Pdisplayname)", v1=True + "/profile/(?P[^/]*)/(?Pdisplayname)$", v1=True ), *client_patterns( - "/profile/(?P[^/]*)/(?Pavatar_url)", v1=True + "/profile/(?P[^/]*)/(?Pavatar_url)$", v1=True ), re.compile( - r"^/_matrix/client/v3/profile/(?P[^/]*)/(?P[^/]*)" + r"^/_matrix/client/v3/profile/(?P[^/]*)/(?P[^/]*)$", ), ] @@ -112,7 +112,7 @@ class ProfileFieldRestServlet(RestServlet): if hs.config.experimental.msc4133_enabled: self.PATTERNS.append( re.compile( - r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" + r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)$" ) ) diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index ae81e80654..8355d0c571 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -338,7 +338,7 @@ class RegistrationSubmitTokenServlet(RestServlet): class UsernameAvailabilityRestServlet(RestServlet): - PATTERNS = client_patterns("/register/available") + PATTERNS = client_patterns("/register/available$") def __init__(self, hs: "HomeServer"): super().__init__() @@ -401,7 +401,7 @@ class RegistrationTokenValidityRestServlet(RestServlet): """ PATTERNS = client_patterns( - f"/register/{LoginType.REGISTRATION_TOKEN}/validity", + f"/register/{LoginType.REGISTRATION_TOKEN}/validity$", releases=("v1",), ) CATEGORY = "Registration/login requests" diff --git a/synapse/rest/client/relations.py b/synapse/rest/client/relations.py index c913bc6970..daf220545e 100644 --- a/synapse/rest/client/relations.py +++ b/synapse/rest/client/relations.py @@ -94,7 +94,7 @@ class RelationPaginationServlet(RestServlet): class ThreadsServlet(RestServlet): - PATTERNS = (re.compile("^/_matrix/client/v1/rooms/(?P[^/]*)/threads"),) + PATTERNS = (re.compile("^/_matrix/client/v1/rooms/(?P[^/]*)/threads$"),) CATEGORY = "Client API requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 951b4ff08b..c3bb82e15c 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -1505,7 +1505,7 @@ class RoomAliasListServlet(RestServlet): PATTERNS = [ re.compile( r"^/_matrix/client/unstable/org\.matrix\.msc2432" - r"/rooms/(?P[^/]*)/aliases" + r"/rooms/(?P[^/]*)/aliases$" ), ] + list(client_patterns("/rooms/(?P[^/]*)/aliases$", unstable=False)) CATEGORY = "Client API requests" diff --git a/synapse/rest/client/tags.py b/synapse/rest/client/tags.py index 5699ff35c7..f7272754e6 100644 --- a/synapse/rest/client/tags.py +++ b/synapse/rest/client/tags.py @@ -73,7 +73,7 @@ class TagServlet(RestServlet): """ PATTERNS = client_patterns( - "/user/(?P[^/]*)/rooms/(?P[^/]*)/tags/(?P[^/]*)" + "/user/(?P[^/]*)/rooms/(?P[^/]*)/tags/(?P[^/]*)$" ) CATEGORY = "Account data requests" diff --git a/synapse/rest/client/thirdparty.py b/synapse/rest/client/thirdparty.py index c17335eb48..b4be564eb0 100644 --- a/synapse/rest/client/thirdparty.py +++ b/synapse/rest/client/thirdparty.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) class ThirdPartyProtocolsServlet(RestServlet): - PATTERNS = client_patterns("/thirdparty/protocols") + PATTERNS = client_patterns("/thirdparty/protocols$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/tokenrefresh.py b/synapse/rest/client/tokenrefresh.py index 2b4f7f8953..873c6f3e4e 100644 --- a/synapse/rest/client/tokenrefresh.py +++ b/synapse/rest/client/tokenrefresh.py @@ -39,7 +39,7 @@ class TokenRefreshRestServlet(RestServlet): token. """ - PATTERNS = client_patterns("/tokenrefresh") + PATTERNS = client_patterns("/tokenrefresh$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index 1b6b001b45..c962d5f624 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) class CreateResource(RestServlet): - PATTERNS = [re.compile("/_matrix/media/v1/create")] + PATTERNS = [re.compile("/_matrix/media/v1/create$")] def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() From 44216bf2b68b3099020a13d58cbcf8a420b0c0dc Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 24 Jul 2026 13:25:26 +0100 Subject: [PATCH 049/158] Limit the size of push rules Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-fp53-rw9v-hcf9 Fixes: https://github.com/matrix-org/internal-config/issues/1071 Because this is an out-of-spec limit and could break someone's workflow on release day, I've opted to make it configurable. ----- Reviewed-on: https://github.com/element-hq/synapse-private/pull/149 --- .../configuration/config_documentation.md | 21 +- schema/synapse-config.schema.yaml | 58 +++++- synapse/config/_base.pyi | 2 + synapse/config/_util.py | 15 +- synapse/config/homeserver.py | 3 + synapse/config/push_rules.py | 63 ++++++ synapse/storage/databases/main/push_rule.py | 101 ++++++++- tests/rest/client/test_push_rule_attrs.py | 193 ++++++++++++++++++ 8 files changed, 437 insertions(+), 19 deletions(-) create mode 100644 synapse/config/push_rules.py diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 62a4676adb..8c3a35ae4e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3808,7 +3808,7 @@ This setting has the following sub-options: Defaults to `null`. -* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Fields that will be synced: +* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Fields that will be synced: * displayname * picture - only if Synapse media repository is running in the main process (i.e. not workerized) and media is stored locally Defaults to `false`. @@ -3938,6 +3938,25 @@ push: jitter_delay: 10s ``` --- +### `push_rules` + +*(object)* Options for push rules + +This setting has the following sub-options: + +* `limits` (object): Limits on the size of push rules that users can have + + This setting has the following sub-options: + + * `rule_count` (integer): This is the total number of push rules that each user can have. Power users may expect to have one push rule per room. Defaults to `10000`. + + * `rule_id_length` (integer): This is the maximum length of a push rule ID, in bytes. Push rule IDs need to be allowed to be at least as long as a room ID (which are [limited to 255 bytes per specification](https://spec.matrix.org/v1.19/appendices/#room-ids)) + It's recommended to leave this option as it is. We expect to remove this option if/when the specification standardises on a limit. Defaults to `300`. + + * `rule_size` (integer): This is the maximum size of a push rule's body, in bytes. + The exact mechanism for calculating this size is currently an implementation detail, subject to change. This limit should be treated as a coarse sanity limit rather than something to fine-tune. + It's recommended to leave this option as it is. We expect to remove this option if/when the specification standardises on a limit and a mechanism for calculating it. Defaults to `1024`. +--- ## Rooms Config options relating to rooms. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index ea5bd7c7af..691335f119 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2408,7 +2408,7 @@ properties: Enable the local on-disk media storage provider. When disabled, media is stored only in configured `media_storage_providers` and temporary files are used for processing. - + **Warning:** If this option is set to `false` and no `media_storage_providers` are configured, all media requests will return 404 errors as there will be no storage backend available. @@ -2962,7 +2962,7 @@ properties: examples: - false matrix_rtc: - type: object + type: object description: >- Options related to MatrixRTC. properties: @@ -2982,14 +2982,13 @@ properties: description: >- The base URL of the LiveKit service. Should only be used with LiveKit-based transports. example: https://matrix-rtc.example.com/livekit/jwt - description: - A list of transport types and arguments to use for MatrixRTC connections. + description: A list of transport types and arguments to use for MatrixRTC connections. default: [] default: {} examples: - transports: - - type: livekit - livekit_service_url: https://matrix-rtc.example.com/livekit/jwt + - type: livekit + livekit_service_url: https://matrix-rtc.example.com/livekit/jwt enable_registration: type: boolean description: >- @@ -4664,10 +4663,10 @@ properties: type: boolean description: >- Use this setting to keep a user's profile fields in sync with - information from the identity provider. Fields are checked on every - SSO login, and are updated if necessary. Note that enabling this - option will override user profile information, regardless of whether - users have opted-out of syncing that information when first signing + information from the identity provider. Fields are checked on every + SSO login, and are updated if necessary. Note that enabling this + option will override user profile information, regardless of whether + users have opted-out of syncing that information when first signing in. Fields that will be synced: * displayname @@ -4879,6 +4878,45 @@ properties: include_content: false group_unread_count_by_room: false jitter_delay: 10s + push_rules: + type: object + description: Options for push rules + properties: + limits: + type: object + description: Limits on the size of push rules that users can have + properties: + rule_count: + type: integer + default: 10000 + description: >- + This is the total number of push rules that each user can have. + Power users may expect to have one push rule per room. + rule_id_length: + type: integer + default: 300 + description: >- + This is the maximum length of a push rule ID, in bytes. + Push rule IDs need to be allowed to be at least as long + as a room ID (which are [limited to 255 bytes per specification](https://spec.matrix.org/v1.19/appendices/#room-ids)) + + It's recommended to leave this option as it is. + We expect to remove this option if/when the specification standardises on + a limit. + rule_size: + type: integer + default: 1024 + description: >- + This is the maximum size of a push rule's body, in bytes. + + The exact mechanism for calculating this size is currently an implementation + detail, subject to change. + This limit should be treated as a coarse sanity limit rather than something + to fine-tune. + + It's recommended to leave this option as it is. + We expect to remove this option if/when the specification standardises on + a limit and a mechanism for calculating it. encryption_enabled_by_default_for_room_type: type: string description: >- diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index 7c371d161c..f226afc2b4 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -38,6 +38,7 @@ from synapse.config import ( # noqa: F401 oidc, password_auth_providers, push, + push_rules, ratelimiting, redis, registration, @@ -103,6 +104,7 @@ class RootConfig: worker: workers.WorkerConfig authproviders: password_auth_providers.PasswordAuthProviderConfig push: push.PushConfig + push_rules: push_rules.PushRulesConfig spamchecker: spam_checker.SpamCheckerConfig room: room.RoomConfig userdirectory: user_directory.UserDirectoryConfig diff --git a/synapse/config/_util.py b/synapse/config/_util.py index e09c68ebd4..8e1dd52c29 100644 --- a/synapse/config/_util.py +++ b/synapse/config/_util.py @@ -18,12 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import Any, TypeVar +from typing import Annotated, Any, TypeVar import jsonschema -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel, BeforeValidator, StrictInt, TypeAdapter, ValidationError +from pydantic_core.core_schema import int_schema -from synapse.config._base import ConfigError +from synapse.config._base import Config, ConfigError from synapse.types import JsonDict, StrSequence @@ -97,3 +98,11 @@ def parse_and_validate_mapping( except ValidationError as e: raise ConfigError(str(e)) from e return instances + + +ConfigByteSize = Annotated[ + StrictInt, BeforeValidator(Config.parse_size), int_schema(ge=0) +] +""" +A size in bytes. Pydantic-compatible wrapper for `Config.parse_size` +""" diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 94ebe583a4..e2ae182fc3 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -19,6 +19,8 @@ # # +from synapse.config.push_rules import PushRulesConfig + from ._base import ConfigError, RootConfig from .account_validity import AccountValidityConfig from .api import ApiConfig @@ -102,6 +104,7 @@ class HomeServerConfig(RootConfig): EmailConfig, PasswordAuthProviderConfig, PushConfig, + PushRulesConfig, SpamCheckerConfig, RoomConfig, UserDirectoryConfig, diff --git a/synapse/config/push_rules.py b/synapse/config/push_rules.py new file mode 100644 index 0000000000..deb7e8dde1 --- /dev/null +++ b/synapse/config/push_rules.py @@ -0,0 +1,63 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import Annotated, Any + +from pydantic import ( + Field, + StrictInt, + ValidationError, +) + +from synapse.config._util import ConfigByteSize +from synapse.types import JsonDict +from synapse.util.pydantic_models import ParseModel + +from ._base import Config, ConfigError + + +class PushRulesLimitsConfig(ParseModel): + # Chosen arbitrarily, but with the rough rationale that a user + # might have on the order of 10k rooms and want to set a push rule override for each one. + rule_count: Annotated[StrictInt, Field(ge=0)] = 10_000 + + # Chosen arbitrarily, but with the rationale that room IDs are allowed to be up to 255 bytes + # and they are often used in rule IDs. + rule_id_length: Annotated[StrictInt, Field(ge=1)] = 300 + + # Chosen arbitrarily, but with the rationale that real-world push rules don't get + # nearly this big in practice. + # Even 512 bytes would probably have been fine, but we should leave space for the use cases + # of push rules to grow in the future. + rule_size: Annotated[ConfigByteSize, Field(ge=1)] = 1024 + + +class PushRulesConfigModel(ParseModel): + limits: PushRulesLimitsConfig = Field(default_factory=PushRulesLimitsConfig) + + +class PushRulesConfig(Config): + section = "push_rules" + + def read_config(self, config: JsonDict, **kwargs: Any) -> None: + raw_config = config.get("push_rules", {}) + + try: + parsed = PushRulesConfigModel(**raw_config) + except ValidationError as e: + raise ConfigError( + f"Could not validate configuration: {e}", + path=("push_rules",), + ) from e + + self.limits = parsed.limits diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index d361166cec..5086f4639b 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -19,6 +19,7 @@ # # import logging +from http import HTTPStatus from typing import ( TYPE_CHECKING, Any, @@ -31,7 +32,7 @@ from typing import ( from twisted.internet import defer -from synapse.api.errors import StoreError +from synapse.api.errors import Codes, StoreError, SynapseError from synapse.config.homeserver import ExperimentalConfig from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.replication.tcp.streams import PushRulesStream @@ -112,6 +113,22 @@ def _load_rules( return filtered_rules +def _push_rule_size_for_limits(*, conditions_json: str, actions_json: str) -> int: + """ + Returns the size of a push rule, as used for applying the size limit. + + We aren't tied to any particular definition, but currently this is + simply the size in bytes of the conditions and actions JSON added together, + so not rocket science, but this function provides a 'label' for it. + """ + # This is not a very predictable way of calculating the size from the + # point of view of the client, but since it's an out-of-spec limit + # entirely at our discretion, we don't really have to worry about + # the exact calculation. + # FIXME: Spec a predictable push rule size limit + return len(conditions_json.encode("utf-8")) + len(actions_json.encode("utf-8")) + + class PushRulesWorkerStore( ApplicationServiceWorkerStore, PusherWorkerStore, @@ -170,6 +187,8 @@ class PushRulesWorkerStore( self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") + self._config = hs.config.push_rules + def get_max_push_rules_stream_id(self) -> int: """Get the position of the push rules stream. @@ -409,6 +428,29 @@ class PushRulesWorkerStore( conditions_json = json_encoder.encode(conditions) actions_json = json_encoder.encode(actions) + + rule_id_len = len(rule_id.encode("utf-8")) + if rule_id_len > self._config.limits.rule_id_length: + raise SynapseError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule ID length exceeds server limit ({rule_id_len} bytes > {self._config.limits.rule_id_length} bytes).", + # FIXME: Provide a better error code. + # None of the existing options seem entirely correct, though. + Codes.UNKNOWN, + ) + + rule_body_size = _push_rule_size_for_limits( + conditions_json=conditions_json, actions_json=actions_json + ) + if rule_body_size > self._config.limits.rule_size: + raise SynapseError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule size exceeds server limit ({rule_body_size} bytes > {self._config.limits.rule_size} bytes).", + # FIXME: Provide a better error code. + # None of the existing options seem entirely correct, though. + Codes.UNKNOWN, + ) + async with self._push_rules_stream_id_gen.get_next() as stream_id: event_stream_ordering = self._stream_id_gen.get_current_token() @@ -578,13 +620,19 @@ class PushRulesWorkerStore( actions_json: str, update_stream: bool = True, ) -> None: + """Specialised version of simple_upsert_txn that picks a push_rule_id + using the _push_rule_id_gen if it needs to insert the rule. + + Preconditions: + - this worker is a push writer + - the "push_rules" table is locked + - the push rule has already been validated, + including for rule ID length and rule body size. + """ + if not self._is_push_writer: raise Exception("Not a push writer") - """Specialised version of simple_upsert_txn that picks a push_rule_id - using the _push_rule_id_gen if it needs to insert the rule. It assumes - that the "push_rules" table is locked""" - sql = ( "UPDATE push_rules" " SET priority_class = ?, priority = ?, conditions = ?, actions = ?" @@ -597,6 +645,27 @@ class PushRulesWorkerStore( ) if txn.rowcount == 0: + # About to add a new rule, so check our limits first. + txn.execute( + """ + SELECT COUNT(*) FROM push_rules + WHERE user_name = ? + """, + (user_id,), + ) + (num_push_rules,) = cast(tuple[int], txn.fetchone()) + if num_push_rules >= self._config.limits.rule_count: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"Creating a push rule would exceed the limit on the number of push rules associated with your account ({num_push_rules + 1} rules > {self._config.limits.rule_count} rules)", + # FIXME: Provide a better error code, especially for this case. + # None of the existing options seem entirely correct, though. + # `M_USER_LIMIT_EXCEEDED` comes closest but needs an `info_uri`. + # Should follow up and add a built-in error page template, similarly to + # https://github.com/element-hq/synapse/pull/18876 ? + Codes.UNKNOWN, + ) + # We didn't update a row with the given rule_id so insert one push_rule_id = self._push_rule_id_gen.get_next() @@ -839,6 +908,28 @@ class PushRulesWorkerStore( ) else: try: + # Before updating the push rule, we need to check that we won't exceed + # the size limit on push rules. + # For that, we need to fetch the `conditions` JSON. + conditions_json = self.db_pool.simple_select_one_onecol_txn( + txn, + "push_rules", + {"user_name": user_id, "rule_id": rule_id}, + "conditions", + ) + + rule_body_size = _push_rule_size_for_limits( + conditions_json=conditions_json, actions_json=actions_json + ) + if rule_body_size > self._config.limits.rule_size: + raise SynapseError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule size exceeds server limit ({rule_body_size} bytes > {self._config.limits.rule_size} bytes).", + # FIXME: Provide a better error code. + # None of the existing options seem entirely correct, though. + Codes.UNKNOWN, + ) + self.db_pool.simple_update_one_txn( txn, "push_rules", diff --git a/tests/rest/client/test_push_rule_attrs.py b/tests/rest/client/test_push_rule_attrs.py index 53c36b7a9c..f6bbc7496e 100644 --- a/tests/rest/client/test_push_rule_attrs.py +++ b/tests/rest/client/test_push_rule_attrs.py @@ -3,6 +3,7 @@ # # Copyright 2020 The Matrix.org Foundation C.I.C. # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -20,9 +21,13 @@ # from http import HTTPStatus +import canonicaljson +from parameterized.parameterized import parameterized + import synapse from synapse.api.errors import Codes from synapse.rest.client import login, push_rule, room +from synapse.types import JsonDict from tests.unittest import HomeserverTestCase @@ -508,3 +513,191 @@ class PushRuleAttributesTestCase(HomeserverTestCase): Codes.INVALID_PARAM, channel.json_body["errcode"], ) + + +class PushRuleLimitTestCase(HomeserverTestCase): + """ + Tests for server-configured limits on push rule size. + + See: https://github.com/element-hq/synapse/security/advisories/GHSA-fp53-rw9v-hcf9 + """ + + servlets = [ + synapse.rest.admin.register_servlets_for_client_rest_resource, + room.register_servlets, + login.register_servlets, + push_rule.register_servlets, + ] + hijack_auth = False + + def default_config(self) -> JsonDict: + config = super().default_config() + # Set some small limits for push rule sizes so that + # we can easily test them. + config["push_rules"] = { + "limits": { + # Size limit of each rule in bytes (canonical JSON) + "rule_size": 64, + # Size limit of the rule ID (in bytes) + "rule_id_length": 24, + # Limit on how many rules you can have + "rule_count": 2, + } + } + return config + + @parameterized.expand( + ( + ( + { + "actions": ["notify"], + "conditions": [{"kind": "event_match", "key": "a", "pattern": "a"}], + }, + 58, + True, + ), + ( + { + "actions": ["notify"], + "conditions": [ + {"kind": "event_match", "key": "a", "pattern": "abcdefg"} + ], + }, + 64, + True, + ), + ( + { + "actions": ["notify"], + "conditions": [ + {"kind": "event_match", "key": "a", "pattern": "abcdefgH"} + ], + }, + 65, + False, + ), + ) + ) + def test_limit_on_push_rule_size( + self, body: JsonDict, expected_body_num_bytes: int, expected_allowed: bool + ) -> None: + """ + Tests that the `push_rules.limits.rule_size` applies to the size of the push rule. + """ + self.register_user("alice", "pass") + token = self.login("alice", "pass") + + # Sanity check the test data: the canonical JSON size of the push rule body + # should be exactly as we expect, otherwise our test is void. + body_bytes = canonicaljson.encode_canonical_json(body) + # We exclude the size of the wrapper, as our implementation currently only + # counts the size of the actions and conditions fragments themselves. + self.assertEqual( + len(body_bytes) - len('{"actions":,"conditions":}'.encode("utf-8")), + expected_body_num_bytes, + ) + + channel = self.make_request( + "PUT", + "/pushrules/global/underride/rule1", + body, + access_token=token, + ) + if expected_allowed: + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Push rule ({body_bytes!r}) within size limit should be accepted: {channel.json_body}", + ) + else: + self.assertEqual( + channel.code, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule ({body_bytes!r}) exceeding size limit should be rejected", + ) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN) + + @parameterized.expand( + ( + ( + 18, + True, + ), + ( + 24, + True, + ), + ( + 25, + False, + ), + ) + ) + def test_limit_on_rule_id_length( + self, rule_id_length: int, expected_allowed: bool + ) -> None: + """ + Tests that the `push_rules.limits.rule_id_length` applies to the byte length + of the push rule ID. + """ + self.register_user("alice", "pass") + token = self.login("alice", "pass") + + PREFIX = "global/underride/" + rule_suffix_length = rule_id_length - len(PREFIX) + assert rule_suffix_length >= 1, "can't construct a rule ID that short" + channel = self.make_request( + "PUT", + f"/pushrules/{PREFIX}{rule_suffix_length * 'a'}", + {"conditions": [], "actions": ["notify"]}, + access_token=token, + ) + if expected_allowed: + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Push rule ID ({rule_id_length} B) within size limit should be accepted: {channel.json_body}", + ) + else: + self.assertEqual( + channel.code, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + "Push rule ID ({rule_id_length} B) exceeding size limit should be rejected", + ) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN) + + def test_limit_on_push_rule_count(self) -> None: + """ + Tests that we are allowed to create exactly the number of push rules + specified by `push_rules.limits.rule_count`, but not a single one more. + """ + self.register_user("bob", "pass") + token = self.login("bob", "pass") + + # First 2 rules are allowable + for i in range(2): + channel = self.make_request( + "PUT", + f"/pushrules/global/underride/rule{i}", + {"actions": ["notify"], "conditions": []}, + access_token=token, + ) + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Push rule within count limit should be allowed: {channel.json_body}", + ) + + # 3rd rule gets denied as it goes over the limit + channel = self.make_request( + "PUT", + "/pushrules/global/underride/rule3", + {"actions": ["notify"], "conditions": []}, + access_token=token, + ) + self.assertEqual( + channel.code, + HTTPStatus.BAD_REQUEST, + "Push rule exceeding count limit should be rejected", + ) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN) From c3adee3509d7b6a8d22ec1f5bd5dab7b154bbc93 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 24 Jul 2026 13:39:04 +0100 Subject: [PATCH 050/158] Restrict what errors we forward to clients from federation and policy servers Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq Fixes: https://github.com/matrix-org/internal-config/issues/1721 https://github.com/element-hq/synapse-private/pull/158 is the end goal, I think ----- Reviewed-on: https://github.com/element-hq/synapse-private/pull/159 --- synapse/api/errors.py | 70 ++++++++++++++++- synapse/replication/http/_base.py | 7 +- tests/handlers/test_room_policy.py | 96 ++++++++++++++++++++++- tests/rest/client/test_rooms.py | 119 ++++++++++++++++++++++++++++- 4 files changed, 285 insertions(+), 7 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0c35b4a7ba..3dfea09b84 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -823,20 +823,24 @@ class HttpResponseException(CodeMessageException): super().__init__(code, msg) self.response = response - def to_synapse_error(self) -> SynapseError: - """Make a SynapseError based on an HTTPResponseException + def unsafe_to_verbatim_synapse_error(self) -> SynapseError: + """Make a SynapseError directly based on a TRUSTED HTTPResponseException. This is useful when a proxied request has failed, and we need to decide how to map the failure onto a matrix error to send back to the client. - An attempt is made to parse the body of the http response as a matrix + An attempt is made to parse the body of the HTTP response as a Matrix error. If that succeeds, the errcode and error message from the body - are used as the errcode and error message in the new synapse error. + are copied verbatim into the new Synapse error. Otherwise, the errcode is set to M_UNKNOWN, and the error message is set to the reason code from the HTTP response. + Safety: + This must ONLY be used on errors from TRUSTED sources, + such as other Synapse workers. + Returns: The error converted to a SynapseError. """ @@ -851,10 +855,68 @@ class HttpResponseException(CodeMessageException): j = {} errcode = j.pop("errcode", Codes.UNKNOWN) + if not isinstance(errcode, str): + errcode = Codes.UNKNOWN errmsg = j.pop("error", self.msg) + if not isinstance(errmsg, str): + errmsg = self.msg return ProxiedRequestError(self.code, errmsg, errcode, j) + def to_synapse_error(self) -> SynapseError: + """Make a SynapseError directly based on a TRUSTED HTTPResponseException. + + This is useful when a proxied request has failed, and we need to + decide how to map the failure onto a matrix error to send back to the + client. + + An attempt is made to parse the body of the HTTP response as a Matrix + error. If that succeeds, the errcode and error message from the body + are copied verbatim into the new Synapse error, unless it's of + a forbidden type. + + Otherwise, the errcode is set to M_UNKNOWN, and the error message is + set to the reason code from the HTTP response. + + Safety: + This is the correct method to use when forwarding errors + from upstream requests (e.g. federation, policy servers). + + FIXME: restrict forwarded errors further + + Returns: + The error converted to a SynapseError. + """ + # try to parse the body as json, to get better errcode/msg, but + # default to M_UNKNOWN with the HTTP status as the error text + try: + j = json_decoder.decode(self.response.decode("utf-8")) + except ValueError: + j = {} + + if not isinstance(j, dict): + j = {} + + status = self.code + errcode = j.pop("errcode", Codes.UNKNOWN) + if not isinstance(errcode, str): + errcode = Codes.UNKNOWN + errmsg = j.pop("error", self.msg) + if not isinstance(errmsg, str): + errmsg = self.msg + + if errcode == Codes.UNKNOWN_TOKEN: + # We must not relay this error code back down to clients, + # because clients interpret this code to mean that they + # have been logged out. + # See: https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + errcode = Codes.UNKNOWN + + if status == HTTPStatus.UNAUTHORIZED: + status = HTTPStatus.BAD_REQUEST + + return ProxiedRequestError(status, errmsg, errcode, j) + class HomeServerNotSetupException(Exception): """ diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 87d6e80898..686c2351d2 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -344,7 +344,12 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): code=e.code, **{SERVER_NAME_LABEL: server_name}, ).inc() - raise e.to_synapse_error() + # This error is coming from another worker, so we trust it to be safe + # to relay to clients directly. + # In fact, we rely relaying verbatim at the very least to tell + # clients when they are rate-limited, + # but most likely other things too. + raise e.unsafe_to_verbatim_synapse_error() except Exception as e: _outgoing_request_counter.labels( name=cls.NAME, diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index c67ea9b0e0..6d912fd79d 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -12,17 +12,22 @@ # . # # +from http import HTTPStatus from unittest import mock import signedjson +from parameterized import parameterized from signedjson.key import encode_verify_key_base64, get_verify_key +from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from twisted.web.client import Agent from synapse.api.constants import EventTypes from synapse.api.errors import HttpResponseException, SynapseError from synapse.crypto.event_signing import compute_event_signature from synapse.events import EventBase +from synapse.federation.transport.client import TransportLayerClient from synapse.handlers.room_policy import POLICY_SERVER_KEY_ID from synapse.rest import admin from synapse.rest.client import filter, login, room, sync @@ -31,7 +36,7 @@ from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from tests import unittest -from tests.test_utils import event_injection +from tests.test_utils import FakeResponse, event_injection from tests.test_utils.event_builders import make_test_event @@ -546,3 +551,92 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): if ev["event_id"] == event_id: return ev return None + + def _mock_policy_server_response_with_http_error( + self, + status: HTTPStatus, + error_body: JsonDict, + ) -> None: + """ + Make the policy server reply to its `/sign` endpoint with an error. + + Args: + status: the HTTP status to return + error_body: the JSON error body to return + """ + + def request( + method: bytes, + uri: bytes, + headers: object = None, + bodyProducer: object = None, + ) -> "defer.Deferred": + # For our test, we don't expect any other outbound request + assert b"/_matrix/policy/v1/sign" in uri, ( + f"unexpected outbound request to {uri!r}" + ) + return defer.succeed( + FakeResponse.json( + code=status, + payload=error_body, + ) + ) + + fake_agent = mock.create_autospec(Agent, spec_set=True) + fake_agent.request.side_effect = request + self.handler._federation_client.transport_layer = TransportLayerClient(self.hs) + self.hs.get_federation_http_client().agent = fake_agent + + @parameterized.expand( + ( + ( + HTTPStatus.IM_A_TEAPOT, + {"errcode": "M_FORBIDDEN", "error": "No coffee here"}, + HTTPStatus.IM_A_TEAPOT, + {"errcode": "M_FORBIDDEN", "error": "No coffee here"}, + ), + # This case is https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + # The error is rewritten for safety. + ( + HTTPStatus.UNAUTHORIZED, + {"errcode": "M_UNKNOWN_TOKEN", "error": "unknown token"}, + HTTPStatus.BAD_REQUEST, + { + "errcode": "M_UNKNOWN", + "error": "unknown token", + }, + ), + ) + ) + def test_policy_server_error_bubbling_to_client( + self, + policy_server_error_status: HTTPStatus, + policy_server_error_body: JsonDict, + expected_client_facing_error_status: HTTPStatus, + expected_client_facing_error_body: JsonDict, + ) -> None: + """ + Tests how errors from the policy server are forwarded back to clients. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + """ + + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + + # Mock the policy server (at the HTTP level) to return + # the configured error + self._mock_policy_server_response_with_http_error( + policy_server_error_status, + policy_server_error_body, + ) + + response_body = self.helper.send_event( + self.room_id, + "m.room.message", + {"body": "honk", "msgtype": "m.text"}, + tok=self.creator_token, + expect_code=expected_client_facing_error_status, + ) + + self.assertEqual(response_body, expected_client_facing_error_body) diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 90063ccb06..ca95a3a099 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -26,12 +26,14 @@ import json from http import HTTPStatus from typing import Any, Iterable, Literal -from unittest.mock import AsyncMock, Mock, call, patch +from unittest.mock import AsyncMock, Mock, call, create_autospec, patch from urllib import parse as urlparse from parameterized import param, parameterized +from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from twisted.web.client import Agent import synapse.rest.admin from synapse.api.constants import ( @@ -67,6 +69,7 @@ from synapse.util.stringutils import random_string from tests import unittest from tests.http.server._base import make_request_with_cancellation_test from tests.storage.test_stream import PaginationTestCase +from tests.test_utils import FakeResponse from tests.test_utils.event_injection import ( create_event, inject_event, @@ -5764,3 +5767,117 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): expect_redaction=True, reason="being disruptive", ) + + +class CreateRoomRemoteInviteTestCase(unittest.FederatingHomeserverTestCase): + """ + Tests error propagation from remote invites during /createRoom. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq. + """ + + servlets = [ + room.register_servlets, + login.register_servlets, + register.register_servlets, + admin.register_servlets, + ] + + hijack_auth = False + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user_id = self.register_user("creator", "test") + self.token = self.login("creator", "test") + + def _mock_remote_invite_http_error( + self, + status: int, + error_body: JsonDict, + ) -> None: + """ + Make the remote homeserver reply to its `/invite` endpoint with an error. + + Args: + status: the HTTP status to return + error_body: the JSON error body to return + """ + federation_http_client = self.hs.get_federation_http_client() + + fake_agent = create_autospec(Agent, spec_set=True) + + def request( + method: bytes, + uri: bytes, + headers: object = None, + bodyProducer: object = None, + ) -> "defer.Deferred": + # For our test, we don't expect any other outbound request + assert b"/invite/" in uri, f"unexpected outbound request to {uri!r}" + return defer.succeed( + FakeResponse.json( + code=status, + payload=error_body, + ) + ) + + fake_agent.request.side_effect = request + federation_http_client.agent = fake_agent + + @parameterized.expand( + ( + ( + HTTPStatus.IM_A_TEAPOT, + { + "errcode": "M_FORBIDDEN", + "error": "You can't invite this user", + }, + HTTPStatus.IM_A_TEAPOT, + { + "errcode": "M_FORBIDDEN", + "error": "You can't invite this user", + }, + ), + # This case is https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + # The error is rewritten for safety. + ( + HTTPStatus.UNAUTHORIZED, + {"errcode": "M_UNKNOWN_TOKEN", "error": "unknown token"}, + HTTPStatus.BAD_REQUEST, + { + "errcode": "M_UNKNOWN", + "error": "unknown token", + }, + ), + ) + ) + def test_remote_invite_bubbles_errors( + self, + policy_server_error_status: HTTPStatus, + policy_server_error_body: JsonDict, + expected_client_facing_error_status: HTTPStatus, + expected_client_facing_error_body: JsonDict, + ) -> None: + """ + Test that, when creating a room involving a remote invite, + when the remote homeserver returns an error, we bubble it + to the client carefully. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + """ + # Mock the remote homeserver (at the HTTP level) to return the configured error + self._mock_remote_invite_http_error( + policy_server_error_status, + policy_server_error_body, + ) + + channel = self.make_request( + "POST", + "/createRoom", + {"invite": ["@alice:" + self.OTHER_SERVER_NAME]}, + access_token=self.token, + ) + + self.assertEqual( + channel.code, expected_client_facing_error_status, channel.result + ) + self.assertEqual(channel.json_body, expected_client_facing_error_body) From 0a2456fef357b6ede022fc14308a7eb8db085115 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 24 Jul 2026 14:10:23 +0100 Subject: [PATCH 051/158] Validate room name, avatar and heroes' profile fields before sending down sliding sync as trusted fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-jhcg-5392-5mjw Fixes: https://github.com/matrix-org/internal-config/issues/1751 I introduce some stricter JSON types that don't break down to `Any` — it seems these have become possible since our last attempt. (I'm pretty sure mypy wouldn't let you do this a few years ago.) Our `dict[str, Any]` type is such a footgun. I'd like to spread this out further, but will do so after the security release. I then use these stricter JSON types on everything the sliding sync handler pulls out of `event.content` and therefore get forced into a bare minimum level of validation, by the type checker. ----- Reviewed-on: https://github.com/element-hq/synapse-private/pull/151 --- synapse/handlers/sliding_sync/__init__.py | 22 +- synapse/types/__init__.py | 42 +++ .../client/sliding_sync/test_rooms_meta.py | 243 +++++++++++++++++- 3 files changed, 302 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 10ca3ddea0..afb93e42b4 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -57,6 +57,7 @@ from synapse.types import ( StrCollection, StreamKeyType, StreamToken, + StrictJsonMapping, ) from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.handlers.sliding_sync import ( @@ -953,7 +954,10 @@ class SlidingSyncHandler: ) name_event = name_states.get((EventTypes.Name, "")) if name_event is not None: - room_name = name_event.content.get("name") + name_event_content: StrictJsonMapping = name_event.content + unchecked_room_name = name_event_content.get("name") + if isinstance(unchecked_room_name, str): + room_name = unchecked_room_name # We only need the room summary for calculating heroes, however if we do # fetch it then we can use it to calculate `joined_count` and @@ -1356,18 +1360,28 @@ class SlidingSyncHandler: room_avatar: str | None = None avatar_event = room_state.get((EventTypes.RoomAvatar, "")) if avatar_event is not None: - room_avatar = avatar_event.content.get("url") + room_avatar_content: StrictJsonMapping = avatar_event.content + unchecked_room_avatar = room_avatar_content.get("url") + if isinstance(unchecked_room_avatar, str): + room_avatar = unchecked_room_avatar # Assemble heroes: extract the info from the state we just fetched heroes: list[SlidingSyncResult.RoomResult.StrippedHero] = [] for hero_user_id in hero_user_ids: member_event = hero_membership_state.get((EventTypes.Member, hero_user_id)) if member_event is not None: + member_event_content: StrictJsonMapping = member_event.content + unchecked_display_name = member_event_content.get("displayname") + unchecked_avatar_url = member_event_content.get("avatar_url") heroes.append( SlidingSyncResult.RoomResult.StrippedHero( user_id=hero_user_id, - display_name=member_event.content.get("displayname"), - avatar_url=member_event.content.get("avatar_url"), + display_name=unchecked_display_name + if isinstance(unchecked_display_name, str) + else None, + avatar_url=unchecked_avatar_url + if isinstance(unchecked_avatar_url, str) + else None, ) ) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index a6fc806701..b42893e39d 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -37,6 +37,7 @@ from typing import ( MutableMapping, NoReturn, Optional, + Sequence, TypedDict, TypeVar, Union, @@ -102,6 +103,47 @@ JsonMapping = Mapping[str, Any] # A JSON-serialisable object. JsonSerializable = object +StrictJsonValue = Union[ + None, + bool, + int, + float, + str, + "StrictJsonList", + "StrictJsonDict", + "StrictJsonSequence", + "StrictJsonMapping", +] +""" +Type that represents any valid JSON value, recursively. +Does not fall back to `Any` at deeper levels, which makes it more safe than `JsonValue`. + +Can also represent immutable mapping and tuple types. +(Not sure if we would be better splitting them out.) +""" + +StrictJsonList = list["StrictJsonValue"] +""" +Type that represents a list of any valid JSON value. +""" + +StrictJsonDict = dict[str, "StrictJsonValue"] +""" +Type that represents a dict with string keys (as per JSON) and values of any +valid JSON type. +Does not fall back to `Any` at deeper levels, which makes it more safe than `JsonDict`. +""" + +StrictJsonSequence = Sequence["StrictJsonValue"] +""" +Like `StrictJsonList` but using a `Sequence` as the collection type. +""" + +StrictJsonMapping = Mapping[str, "StrictJsonValue"] +""" +Like `StrictJsonDict` but using `Mapping` as the collection type. +""" + # Collection[str] that does not include str itself; str being a Sequence[str] # is very misleading and results in bugs. # diff --git a/tests/rest/client/sliding_sync/test_rooms_meta.py b/tests/rest/client/sliding_sync/test_rooms_meta.py index b1b771ef84..93fdfa59f9 100644 --- a/tests/rest/client/sliding_sync/test_rooms_meta.py +++ b/tests/rest/client/sliding_sync/test_rooms_meta.py @@ -13,6 +13,7 @@ # import logging from typing import Any +from unittest.mock import patch from parameterized import parameterized, parameterized_class @@ -26,7 +27,11 @@ from synapse.server import HomeServer from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase -from tests.test_utils.event_injection import create_event +from tests.test_utils.event_injection import ( + create_event, + inject_event, + inject_member_event, +) logger = logging.getLogger(__name__) @@ -1407,3 +1412,239 @@ class SlidingSyncRoomsMetaTestCase(SlidingSyncBase): } } response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_non_string_name(self, non_string_name: object) -> None: + """ + Test that when the room name is not a string, it gets + treated the same as if there is no room name set; + the `name` field is omitted and `heroes` are populated instead. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # For heroes to be emitted, we need a second user + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + self.helper.join(room_id, user2_id, tok=user2_tok) + + # Set the room name to a non-string + # Need to patch out our client-sent event checks to do this. + # (We don't apply these same out-of-spec checks to events + # received through federation. + # Could have instead set up the test to receive the event over federation.) + with patch("synapse.events.validator.EventValidator.validate_new"): + self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user1_id, + type=EventTypes.Name, + state_key="", + content={"name": non_string_name}, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + + # The name should be omitted (non-string value treated as unset). + self.assertNotIn( + "name", + response_body["rooms"][room_id], + response_body["rooms"][room_id], + ) + + # Since there is no name, heroes should be populated. + self.assertEqual( + response_body["rooms"][room_id]["heroes"], + [{"displayname": "user2", "user_id": "@user2:test"}], + ) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_non_string_avatar(self, non_string_avatar: str) -> None: + """ + Test that when the room avatar is not a string, it gets + treated the same as if there is no room avatar set; + the `avatar` field is omitted. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + + # Set the room avatar to a dict (non-string) instead of a URL string. + # Need to patch out our client-sent event checks to do this + # (We don't apply these same out-of-spec checks to events + # received through federation. + # Could have instead set up the test to receive the event over federation.) + with patch("synapse.events.validator.EventValidator.validate_new"): + self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user1_id, + type=EventTypes.RoomAvatar, + state_key="", + content={"url": non_string_avatar}, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + + # The avatar should be omitted (non-string value treated as unset). + self.assertNotIn( + "avatar", + response_body["rooms"][room_id], + response_body["rooms"][room_id], + ) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_heroes_non_string_displayname( + self, non_string_name: str + ) -> None: + """ + Test that when a hero's displayname is not a string, it gets + treated the same as if there is no displayname set: + the `displayname` field is omitted from the hero entry. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + + # Create a room with no name so heroes are populated. + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + + # Inject a membership event for user2 with a non-string displayname. + self.get_success( + inject_member_event( + self.hs, + room_id, + sender=user2_id, + target=user2_id, + membership=Membership.JOIN, + extra_content={ + "displayname": non_string_name, + "avatar_url": "mxc://example.org/a-real-mxc", + }, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + self.assertNotIn( + "name", response_body["rooms"][room_id], response_body["rooms"][room_id] + ) + + # user2 should be in the heroes list, but without a displayname + self.assertEqual( + response_body["rooms"][room_id]["heroes"], + [ + { + "avatar_url": "mxc://example.org/a-real-mxc", + "user_id": "@user2:test", + } + ], + ) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_heroes_non_string_avatar_url( + self, non_string_avatar: str + ) -> None: + """ + Test that when a hero's avatar URL is not a string, it gets + treated the same as if there is no avatar URL set: + the `avatar_url` field is omitted from the hero entry. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room with no name so heroes are populated. + room_id = self.helper.create_room_as( + user2_id, + tok=user2_tok, + ) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Inject a membership event for user2 with a non-string avatar_url. + self.get_success( + inject_member_event( + self.hs, + room_id, + sender=user2_id, + target=user2_id, + membership=Membership.JOIN, + extra_content={ + "displayname": "second user", + "avatar_url": non_string_avatar, + }, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + self.assertNotIn("name", response_body["rooms"][room_id]) + + # user2 should be in the heroes list, but without an avatar + self.assertEqual( + response_body["rooms"][room_id]["heroes"], + [{"displayname": "second user", "user_id": "@user2:test"}], + ) From 0c709bbb78dc9872a1de366217fd8a40f8306d13 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 28 Jul 2026 14:02:12 +0100 Subject: [PATCH 052/158] 1.157.2 --- CHANGES.md | 30 ++++++++++++++++++++++++++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d122a74e19..336ed7addf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,33 @@ +# Synapse 1.157.2 (2026-07-28) + +This security release addresses several vulnerabilities. + +Please upgrade when you can, particularly if your homeserver participates in open federation +and/or has untrusted local users. + +## Security Fixes + +High severity: + +- Fix [ELEMENTSEC-2026-1071](https://github.com/element-hq/synapse/security/advisories/GHSA-fp53-rw9v-hcf9) +- Fix [ELEMENTSEC-2024-1520](https://github.com/element-hq/synapse/security/advisories/GHSA-rgv2-84w7-5j9p) +- Fix [ELEMENTSEC-2026-1717](https://github.com/element-hq/synapse/security/advisories/GHSA-27p5-4f45-gx76) +- Fix [ELEMENTSEC-2026-1721](https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq) +- Fix [ELEMENTSEC-2026-1729](https://github.com/element-hq/synapse/security/advisories/GHSA-cjh7-rcpx-xpf8) +- Fix [ELEMENTSEC-2026-1740](https://github.com/element-hq/synapse/security/advisories/GHSA-6wjm-9p2x-gvpm) + +Moderate severity: + +- Fix [ELEMENTSEC-2026-1714](https://github.com/element-hq/synapse/security/advisories/GHSA-qcjr-46gf-7f4r) +- Fix [ELEMENTSEC-2026-1718](https://github.com/element-hq/synapse/security/advisories/GHSA-r66v-qhwx-8rg4) +- Fix [ELEMENTSEC-2026-1751](https://github.com/element-hq/synapse/security/advisories/GHSA-jhcg-5392-5mjw) + +Low severity: + +- Fix [ELEMENTSEC-2026-1703](https://github.com/element-hq/synapse/security/advisories/GHSA-vh4c-pqh4-w3wq) +- Fix [ELEMENTSEC-2026-1760](https://github.com/element-hq/synapse/security/advisories/GHSA-hgcg-p9gx-fq5f) + + # Synapse 1.157.1 (2026-07-22) ## Bugfixes diff --git a/debian/changelog b/debian/changelog index 6a8d4663a4..38a6625e8b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.157.2) stable; urgency=medium + + * New Synapse release 1.157.2. + + -- Synapse Packaging team Tue, 28 Jul 2026 14:01:25 +0100 + matrix-synapse-py3 (1.157.1) stable; urgency=medium * New synapse release 1.157.1. diff --git a/pyproject.toml b/pyproject.toml index 64e2838085..4dd059a45c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.157.1" +version = "1.157.2" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 5d55a05f8e2f782943b511d5edac0e8fd8fe7bcb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:02:43 +0000 Subject: [PATCH 053/158] Bump regex from 1.12.4 to 1.13.0 (#20013) Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25ceb4d0c0..4c68ebcb18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -992,9 +992,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" dependencies = [ "aho-corasick", "memchr", From 1a1af7b622f219ba0f2501709298caa230ad3912 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 28 Jul 2026 15:44:26 -0500 Subject: [PATCH 054/158] Link to changelog instead of duplicating content in the tag/release (#19984) This means we can point to a single source of truth instead of duplicating the content to the tag and GitHub release. Less to manage and worry about when you make some updates to the changelog (maintenance burden). We also get to avoid the content sitting in the vendor lock-in GitHub releases. We point to `https://github.com/element-hq/synapse/blob/{repo.active_branch.name}/CHANGES.md` as it will have the relevant changelog entry at the top and won't change as we archive releases on `develop`. Even for RC releases after the main release goes out, the entry will still be towards the top. We could try to get the heading anchor for the specific section but I thought that it wasn't necessary (nice but more complex). --- changelog.d/19984.misc | 1 + scripts-dev/release.py | 20 ++++++-------------- 2 files changed, 7 insertions(+), 14 deletions(-) create mode 100644 changelog.d/19984.misc diff --git a/changelog.d/19984.misc b/changelog.d/19984.misc new file mode 100644 index 0000000000..7083017a58 --- /dev/null +++ b/changelog.d/19984.misc @@ -0,0 +1 @@ +Link to changelog instead of duplicating content in the tag/release. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index d1ba2e5822..37fd560123 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -373,19 +373,10 @@ def _tag(gh_token: str | None) -> None: ) click.get_current_context().abort() - # Get the appropriate changelogs and tag. - changes = get_changes_for_version(current_version) + # We simply point to the changelog instead of duplicating the content into the git tag/release + tag_message = f"Changelog: https://github.com/element-hq/synapse/blob/{repo.active_branch.name}/CHANGES.md" - click.echo_via_pager(changes) - if click.confirm("Edit text?", default=False): - edited_changes = click.edit(changes, require_save=False) - # This assert is for mypy's benefit. click's docs are a little unclear, but - # when `require_save=False`, not saving the temp file in the editor returns - # the original string. - assert edited_changes is not None - changes = edited_changes - - repo.create_tag(tag_name, message=changes, sign=True) + repo.create_tag(tag_name, message=tag_message, sign=True) if not click.confirm("Push tag to GitHub?", default=True): print("") @@ -420,7 +411,7 @@ def _tag(gh_token: str | None) -> None: release = gh_repo.create_git_release( tag=tag_name, name=tag_name, - message=changes, + message=tag_message, draft=True, prerelease=current_version.is_prerelease, ) @@ -745,6 +736,7 @@ def announce(_gh_token: str | None) -> None: def _announce() -> None: """Generate markdown to announce the release.""" + synapse_repo = get_repo_and_check_clean_checkout() current_version = get_package_version() tag_name = f"v{current_version}" is_rc = "rc" in tag_name @@ -764,7 +756,7 @@ Hi everyone. Synapse {current_version} has just been released. ) release_text += f""" -[notes](https://github.com/element-hq/synapse/releases/tag/{tag_name}) | \ +[notes](https://github.com/element-hq/synapse/blob/{synapse_repo.active_branch.name}/CHANGES.md) | \ [docker](https://hub.docker.com/r/matrixdotorg/synapse/tags?name={tag_name}) | \ [debs](https://packages.matrix.org/debian/) | \ [pypi](https://pypi.org/project/matrix-synapse/{current_version}/)""" From 41c41d2c1219365ebcebd93f26e18ae41ccc917a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:30:01 +0000 Subject: [PATCH 055/158] Bump quinn-proto from 0.11.14 to 0.11.16 (#20010) Signed-off-by: dependabot[bot] --- Cargo.lock | 87 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4c68ebcb18..22ddde611a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,6 +103,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -137,6 +148,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -312,12 +332,24 @@ name = "getrandom" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "r-efi 6.0.0", + "rand_core 0.10.1", "wasm-bindgen", ] @@ -913,14 +945,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.3", + "getrandom 0.4.3", "lru-slab", - "rand", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -961,6 +994,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.9.4" @@ -968,7 +1007,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -978,7 +1028,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", ] [[package]] @@ -990,6 +1040,21 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "regex" version = "1.13.0" @@ -1249,7 +1314,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1260,7 +1325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1548,7 +1613,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand", + "rand 0.9.4", "web-time", ] From e3715b24fbf2c9399df54cb9e4e5282b91a94597 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:30:21 +0000 Subject: [PATCH 056/158] Bump the patches group with 2 updates (#20012) Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22ddde611a..de9bdb900b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,9 +78,9 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", From fba14358a7dd47c43166ce3f7139b04a44cf14d5 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 Jul 2026 10:32:41 -0500 Subject: [PATCH 057/158] Fix `RemoteJoinHelper` signing events with the default room version (room version mismatch) (#20015) Fix `RemoteJoinHelper` signing events with mismatched room version compared to the `room_version` arg. The default room version on `develop` is `11` but the `RemoteJoinHelper` `room_version` arg defaults to `10` (room version mismatch). This mismatch wasn't present where this fix was developed (https://github.com/element-hq/synapse-private/pull/136) as the default room version was only recently bumped to `11` via https://github.com/element-hq/synapse/pull/18680 (not even in a release yet). Fixes the CI being broken on `develop` :x:: ``` [ERROR] Traceback (most recent call last): File "/home/runner/work/synapse/synapse/tests/federation/test_federation_join_upgraded_room.py", line 298, in test_no_transfer_when_tombstone_does_not_match join_helper.join(local_user_id, local_user_tok) File "/home/runner/work/synapse/synapse/tests/federation/_remote_join.py", line 350, in join self._test_case.helper.join(remote_room_id, local_user_id, tok=local_user_tok) File "/home/runner/work/synapse/synapse/tests/rest/client/utils.py", line 195, in join return self.change_membership( File "/home/runner/work/synapse/synapse/tests/rest/client/utils.py", line 333, in change_membership assert channel.code == expect_code, ( builtins.AssertionError: Expected: 200, got: 400, PUT /_matrix/client/r0/rooms/!remote-room:other.example.com/state/m.room.member/@user1:test?access_token=syt_dXNlcjE_JSEarRndiAGqtydnrdLn_33qlLD -> resp: b'{"errcode":"M_UNKNOWN","error":"No create event in state"}' tests.federation.test_federation_join_upgraded_room.FederationJoinUpgradedRoomTestCase.test_no_transfer_when_tombstone_does_not_match ``` These tests were originally introduced https://github.com/element-hq/synapse-private/pull/136 (developed private as this was part of the Synapse security release) and introduced into the public codebase via https://github.com/element-hq/synapse/commit/cbc6934821aab314506c5957223c60c74eeda091 --- changelog.d/20015.misc | 1 + tests/federation/_remote_join.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 changelog.d/20015.misc diff --git a/changelog.d/20015.misc b/changelog.d/20015.misc new file mode 100644 index 0000000000..2ef49ff017 --- /dev/null +++ b/changelog.d/20015.misc @@ -0,0 +1 @@ +Fix `RemoteJoinHelper` test helper signing events with the default room version (room version mismatch). diff --git a/tests/federation/_remote_join.py b/tests/federation/_remote_join.py index 94fa44e200..feb9c528c9 100644 --- a/tests/federation/_remote_join.py +++ b/tests/federation/_remote_join.py @@ -21,7 +21,8 @@ from unittest.mock import Mock import attr from synapse.api.constants import EventContentFields, EventTypes, Membership -from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion +from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.events import EventBase from synapse.events.utils import strip_event from synapse.federation.transport.client import SendJoinResponse @@ -87,7 +88,7 @@ class RemoteJoinHelper: federation_http_client: Mock, *, remote_creator_user_id: str | None = None, - room_version: RoomVersion = RoomVersions.V10, + room_version: RoomVersion = KNOWN_ROOM_VERSIONS[DEFAULT_ROOM_VERSION], create_content: JsonDict | None = None, state_events: list[RemoteStateEvent] | None = None, ) -> None: @@ -127,6 +128,7 @@ class RemoteJoinHelper: room_create_event = make_test_event( test_case.add_hashes_and_signatures_from_other_server( create_event_dict, + room_version=room_version, ), room_version=room_version, ) @@ -146,7 +148,8 @@ class RemoteJoinHelper: "content": {"membership": Membership.JOIN}, "auth_events": [room_create_event.event_id], "prev_events": [room_create_event.event_id], - } + }, + room_version=room_version, ), room_version=room_version, ) @@ -172,7 +175,8 @@ class RemoteJoinHelper: creator_membership_event.event_id, ], "prev_events": [prev_event.event_id], - } + }, + room_version=room_version, ), room_version=room_version, ) @@ -217,7 +221,8 @@ class RemoteJoinHelper: if extra_events else creator_membership_event.event_id ], - } + }, + room_version=room_version, ), room_version=room_version, ) @@ -315,7 +320,9 @@ class RemoteJoinHelper: ): # As the remote server, sign the join event before returning it. join_membership_event_signed = make_test_event( - self._test_case.add_hashes_and_signatures_from_other_server(data), + self._test_case.add_hashes_and_signatures_from_other_server( + data, room_version=room_version + ), room_version=room_version, ) return SendJoinResponse( From 897f68b3c2ca0a4f07943f48f65c5a898e2448e8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 Jul 2026 13:30:39 -0500 Subject: [PATCH 058/158] Fix `assertIncludes` printing `None` at the end of the message (#20020) --- changelog.d/20020.misc | 1 + tests/unittest.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20020.misc diff --git a/changelog.d/20020.misc b/changelog.d/20020.misc new file mode 100644 index 0000000000..aa77313581 --- /dev/null +++ b/changelog.d/20020.misc @@ -0,0 +1 @@ +Fix `assertIncludes` printing `None` at the end of the message. diff --git a/tests/unittest.py b/tests/unittest.py index 3d130c1d77..152fb341cb 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -317,7 +317,11 @@ class TestCase(unittest.TestCase): ) diff_message = f"{first_message}\n{expected_string}\n{actual_string}" - self.fail(f"{diff_message}\n{message}") + extra_message = "" + if message is not None: + extra_message = "f\n{message}" + + self.fail(f"{diff_message}{extra_message}") def DEBUG(target: TV) -> TV: From 36664cac4fdec38993c68078cd9a82452b56dca7 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 30 Jul 2026 12:32:27 +0100 Subject: [PATCH 059/158] Add Module API hooks that notify modules when events are delivered over federation. (#20019) Closes: https://github.com/element-hq/synapse/issues/19904 --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/20019.feature | 1 + docs/SUMMARY.md | 1 + docs/modules/federation_callbacks.md | 39 ++ synapse/federation/federation_server.py | 85 +++- .../federation/sender/transaction_manager.py | 43 +- synapse/module_api/__init__.py | 17 + synapse/module_api/callbacks/__init__.py | 3 + synapse/module_api/callbacks/federation.py | 207 ++++++++++ tests/module_api/test_federation_callbacks.py | 378 ++++++++++++++++++ 9 files changed, 754 insertions(+), 20 deletions(-) create mode 100644 changelog.d/20019.feature create mode 100644 docs/modules/federation_callbacks.md create mode 100644 synapse/module_api/callbacks/federation.py create mode 100644 tests/module_api/test_federation_callbacks.py diff --git a/changelog.d/20019.feature b/changelog.d/20019.feature new file mode 100644 index 0000000000..2744fb4a8d --- /dev/null +++ b/changelog.d/20019.feature @@ -0,0 +1 @@ +Add Synapse Module API hook that notifies modules when events are delivered over federation (`register_federation_callbacks(...)`).. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 980f51d078..709e1a01e2 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -50,6 +50,7 @@ - [Background update controller callbacks](modules/background_update_controller_callbacks.md) - [Account data callbacks](modules/account_data_callbacks.md) - [Add extra fields to client events unsigned section callbacks](modules/add_extra_fields_to_client_events_unsigned.md) + - [Federation callbacks](modules/federation_callbacks.md) - [Media repository callbacks](modules/media_repository_callbacks.md) - [Ratelimit callbacks](modules/ratelimit_callbacks.md) - [Porting a legacy module to the new interface](modules/porting_legacy_module.md) diff --git a/docs/modules/federation_callbacks.md b/docs/modules/federation_callbacks.md new file mode 100644 index 0000000000..09154e5640 --- /dev/null +++ b/docs/modules/federation_callbacks.md @@ -0,0 +1,39 @@ +# Federation callbacks + +Federation callbacks can be registered using the module API's `register_federation_callbacks` method. + +## Callbacks + +The available federation callbacks are: + +### `on_event_delivered_over_federation` + +_First introduced in Synapse v1.158.0_ + +```python +async def on_event_delivered_over_federation( + event: FederationEventDeliveryEvent, +) -> None: +``` + +Called when an event has been delivered over federation. +See `FederationEventDeliveryEvent` for detailed information available on the event. + +Note that depending on the specific method, delivery may not have +actually been acknowledged by the other homeserver. +See `FederatedEventDeliveryMethod` for details on which cases imply acknowledgment. + +Modules should anticipate more methods being added to the `FederatedEventDeliveryMethod` enum +over time (it is non-exhaustive). + +Only methods that deliver full, signed PDUs are included in this mechanism. +Some notable examples of excluded endpoints: +- `/send_knock` is excluded as it only returns unsigned 'stripped state'. +- `/timestamp_to_event` is excluded as it only returns event IDs, not events themselves. + +If multiple modules implement this callback, Synapse runs them all in order. +Exceptions are logged and otherwise ignored. + +Performance note: +- Registering this hook causes a performance (caching) optimisation on the + Federation `/state` endpoint to be bypassed. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 83566050f6..2379b2ed2c 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -81,6 +81,7 @@ from synapse.logging.opentracing import ( ) from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.module_api.callbacks.federation import FederatedEventDeliveryMethod from synapse.replication.http.federation import ( ReplicationFederationSendEduRestServlet, ) @@ -142,6 +143,7 @@ class FederationServer(FederationBase): self.server_name = hs.hostname self.handler = hs.get_federation_handler() self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker + self._federation_callbacks = hs.get_module_api_callbacks().federation self._federation_event_handler = hs.get_federation_event_handler() self.state = hs.get_state_handler() self._event_auth_handler = hs.get_event_auth_handler() @@ -244,6 +246,10 @@ class FederationServer(FederationBase): res = self._transaction_dict_from_pdus(pdus) + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, pdus, FederatedEventDeliveryMethod.BACKFILL + ) + return 200, res async def on_timestamp_to_event_request( @@ -655,14 +661,27 @@ class FederationServer(FederationBase): # - but that's non-trivial to get right, and anyway somewhat defeats # the point of the linearizer. async with self._server_linearizer.queue((origin, room_id)): - resp = await self._state_resp_cache.wrap( - (room_id, event_id), - self._on_context_state_request_compute, - room_id, - event_id, - ) - - return 200, resp + if not self._federation_callbacks.interested_in_events_delivered_over_federation(): + # In the usual case where no module is interested in tracking event deliveries, + # use the response cache. + resp = await self._state_resp_cache.wrap( + (room_id, event_id), + self._on_context_state_request_compute, + room_id, + event_id, + ) + return 200, resp + else: + # When a module is interested in tracking event deliveries, + # we can't use the response cache that returns pre-serialised + # events, as we wouldn't have the raw events to track. + resp, events = await self._on_context_state_request_compute_with_events( + room_id, event_id + ) + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, events, FederatedEventDeliveryMethod.STATE + ) + return 200, resp @trace @tag_args @@ -697,6 +716,28 @@ class FederationServer(FederationBase): async def _on_context_state_request_compute( self, room_id: str, event_id: str ) -> dict[str, list]: + """ + Respond to a `/state` request, returning just the response. + + This separation exists because we don't want to hold on to the underlying + events in the response cache, just the serialised JSON. + """ + resp, _ = await self._on_context_state_request_compute_with_events( + room_id, event_id + ) + return resp + + async def _on_context_state_request_compute_with_events( + self, room_id: str, event_id: str + ) -> tuple[dict[str, list], list[EventBase]]: + """ + Respond to a `/state` request. + + Returns: + Tuple of: + 1. the `/state` response + 2. list of the events used to build that response + """ pdus: Collection[EventBase] event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) pdus = await self.store.get_events_as_list(event_ids) @@ -705,10 +746,13 @@ class FederationServer(FederationBase): room_id, [pdu.event_id for pdu in pdus] ) - return { - "pdus": serialize_and_filter_pdus(pdus), - "auth_chain": serialize_and_filter_pdus(auth_chain), - } + return ( + { + "pdus": serialize_and_filter_pdus(pdus), + "auth_chain": serialize_and_filter_pdus(auth_chain), + }, + [*pdus, *auth_chain], + ) async def on_pdu_request( self, origin: str, event_id: str @@ -716,6 +760,9 @@ class FederationServer(FederationBase): pdu = await self.handler.get_persisted_pdu(origin, event_id) if pdu: + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, [pdu], FederatedEventDeliveryMethod.EVENT + ) return 200, self._transaction_dict_from_pdus([pdu]) else: return 404, "" @@ -893,6 +940,12 @@ class FederationServer(FederationBase): if servers_in_room is not None: resp["servers_in_room"] = list(servers_in_room) + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, + [event, *state_events, *auth_chain_events], + FederatedEventDeliveryMethod.SEND_JOIN, + ) + return resp async def on_make_leave_request( @@ -1131,6 +1184,10 @@ class FederationServer(FederationBase): time_now = self._clock.time_msec() auth_pdus = await self.handler.on_event_auth(event_id, room_id) res = {"auth_chain": serialize_and_filter_pdus(auth_pdus, time_now)} + + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, auth_pdus, FederatedEventDeliveryMethod.EVENT_AUTH + ) return 200, res async def on_query_client_keys( @@ -1213,6 +1270,10 @@ class FederationServer(FederationBase): else: logger.debug("Returning %d events", len(missing_events)) + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, missing_events, FederatedEventDeliveryMethod.GET_MISSING_EVENTS + ) + time_now = self._clock.time_msec() return {"events": serialize_and_filter_pdus(missing_events, time_now)} diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index 99aa05ebd6..9a7fb8a3b1 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -18,7 +18,7 @@ # # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Mapping from prometheus_client import Gauge @@ -35,6 +35,7 @@ from synapse.logging.opentracing import ( whitelisted_homeserver, ) from synapse.metrics import SERVER_NAME_LABEL +from synapse.module_api.callbacks.federation import FederatedEventDeliveryMethod from synapse.types import JsonDict from synapse.util.json import json_decoder from synapse.util.metrics import measure_func @@ -64,6 +65,7 @@ class TransactionManager: self._store = hs.get_datastores().main self._transaction_actions = TransactionActions(self._store) self._transport_layer = hs.get_federation_transport_client() + self._federation_callbacks = hs.get_module_api_callbacks().federation self._federation_metrics_domains = ( hs.config.federation.federation_metrics_domains @@ -192,14 +194,39 @@ class TransactionManager: logger.info("TX [%s] {%s} got 200 response", destination, txn_id) - for e_id, r in response.get("pdus", {}).items(): - if "error" in r: - logger.warning( - "TX [%s] {%s} Remote returned error for %s: %s", + pdu_responses = response.get("pdus", {}) + if not isinstance(pdu_responses, Mapping): + logger.warning( + "TX [%s] {%s} Remote returned invalid type for `pdus`", + destination, + txn_id, + ) + else: + for event_id, pdu_response in pdu_responses.items(): + if not isinstance(pdu_response, Mapping) or "error" in pdu_response: + logger.warning( + "TX [%s] {%s} Remote returned error for %s: %s", + destination, + txn_id, + event_id, + pdu_response, + ) + + # If modules have requested to be notified about delivered events, + # build and send that notification. + if self._federation_callbacks.interested_in_events_delivered_over_federation(): + # A PDU is considered acknowledged when the remote echoes the event_id back to + # us, without an error in the PDU response dict. + acknowledged_pdu_ids = { + event_id + for event_id, pdu_response in response.get("pdus", {}).items() + if isinstance(pdu_response, Mapping) + and "error" not in pdu_response + } + await self._federation_callbacks.notify_on_event_delivered_over_federation( destination, - txn_id, - e_id, - r, + [p for p in pdus if p.event_id in acknowledged_pdu_ids], + FederatedEventDeliveryMethod.SEND, ) if pdus and destination in self._federation_metrics_domains: diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 1131253028..48963b8d83 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -90,6 +90,9 @@ from synapse.module_api.callbacks.account_validity_callbacks import ( ON_USER_LOGIN_CALLBACK, ON_USER_REGISTRATION_CALLBACK, ) +from synapse.module_api.callbacks.federation import ( + ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK, +) from synapse.module_api.callbacks.media_repository_callbacks import ( GET_MEDIA_CONFIG_FOR_USER_CALLBACK, GET_MEDIA_UPLOAD_LIMITS_FOR_USER_CALLBACK, @@ -631,6 +634,20 @@ class ModuleApi: add_field_to_unsigned_callback ) + def register_federation_callbacks( + self, + *, + on_event_delivered_over_federation: ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK + | None = None, + ) -> None: + """Registers callbacks for federation. + + Added in Synapse v1.158.0.""" + if on_event_delivered_over_federation is not None: + self._callbacks.federation.register_callbacks( + on_event_delivered_over_federation=on_event_delivered_over_federation + ) + ######################################################################### # The following methods can be called by the module at any point in time. diff --git a/synapse/module_api/callbacks/__init__.py b/synapse/module_api/callbacks/__init__.py index 16ef7a4b47..6ce429dd1c 100644 --- a/synapse/module_api/callbacks/__init__.py +++ b/synapse/module_api/callbacks/__init__.py @@ -21,6 +21,8 @@ from typing import TYPE_CHECKING +from synapse.module_api.callbacks.federation import FederationModuleApiCallbacks + if TYPE_CHECKING: from synapse.server import HomeServer @@ -44,6 +46,7 @@ from synapse.module_api.callbacks.third_party_event_rules_callbacks import ( class ModuleApiCallbacks: def __init__(self, hs: "HomeServer") -> None: self.account_validity = AccountValidityModuleApiCallbacks() + self.federation = FederationModuleApiCallbacks() self.media_repository = MediaRepositoryModuleApiCallbacks(hs) self.ratelimit = RatelimitModuleApiCallbacks(hs) self.spam_checker = SpamCheckerModuleApiCallbacks(hs) diff --git a/synapse/module_api/callbacks/federation.py b/synapse/module_api/callbacks/federation.py new file mode 100644 index 0000000000..ce879b764b --- /dev/null +++ b/synapse/module_api/callbacks/federation.py @@ -0,0 +1,207 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import logging +from enum import Enum +from typing import Awaitable, Callable, Collection + +import attr + +from synapse.events import EventBase + +logger = logging.getLogger(__name__) + + +class FederatedEventDeliveryMethod(str, Enum): + """ + Method by which an event was 'delivered' to another server. + + Note that depending on the specific method, delivery may not have + actually been acknowledged by the other homeserver. + + Modules should anticipate more methods being added to this enum + over time (it is non-exhaustive). + + Only methods that deliver full, signed PDUs are included in this mechanism. + Some notable examples of excluded endpoints: + - `/send_knock` is excluded as it only returns unsigned 'stripped state'. + - `/timestamp_to_event` is excluded as it only returns event IDs, not events themselves. + """ + + SEND = "/send" + """ + The events were pushed over [`/send`](https://spec.matrix.org/v1.19/server-server-api/#put_matrixfederationv1sendtxnid). + + When a callback is triggered with this method, the events have been acknowledged + without error. + """ + + BACKFILL = "/backfill" + """ + The events were pulled over [`/backfill`](https://spec.matrix.org/v1.19/server-server-api/#get_matrixfederationv1backfillroomid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + GET_MISSING_EVENTS = "/get_missing_events" + """ + The events were pulled over [`/get_missing_events`](https://spec.matrix.org/v1.19/server-server-api/#post_matrixfederationv1get_missing_eventsroomid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + EVENT = "/event" + """ + The event was pulled over [`/event`](https://spec.matrix.org/v1.19/server-server-api/#get_matrixfederationv1eventeventid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + EVENT_AUTH = "/event_auth" + """ + The events were pulled over [`/event_auth`](https://spec.matrix.org/v1.19/server-server-api/#get_matrixfederationv1event_authroomideventid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + STATE = "/state" + """ + The events were pulled over [`/state`](https://spec.matrix.org/v1.19/server-server-api/#get_matrixfederationv1stateroomid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + SEND_JOIN = "/send_join" + """ + The events were pulled over [`/send_join`](https://spec.matrix.org/v1.19/server-server-api/#put_matrixfederationv2send_joinroomideventid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + +@attr.s(frozen=True, slots=True, auto_attribs=True) +class FederationEventDeliveryEvent: + """ + Represents the delivery of some events. + + Note that depending on `method`, + delivery may not be acknowledged. + """ + + server_name: str + """ + The server name of the destination the events were delivered to. + """ + + events: Collection[EventBase] + """ + The events that were delivered. + + Modules should not rely on this being the exhaustive list of all events that + were delivered in a single request; + delivery hooks may be triggered in multiple batches. + """ + + method: FederatedEventDeliveryMethod + """ + How the events were delivered to the server. + """ + + +ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK = Callable[ + [FederationEventDeliveryEvent], Awaitable[None] +] + + +class FederationModuleApiCallbacks: + """ + Module API callbacks for generic federation events. + """ + + def __init__(self) -> None: + self._on_event_delivered_over_federation_callbacks: list[ + ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK + ] = [] + + def interested_in_events_delivered_over_federation(self) -> bool: + """ + Whether any `on_event_delivered_over_federation` callbacks are registered. + """ + return len(self._on_event_delivered_over_federation_callbacks) > 0 + + def register_callbacks( + self, + on_event_delivered_over_federation: ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK + | None = None, + ) -> None: + """ + Register callbacks from module for each hook. + + on_event_delivered_over_federation: + Callback fired when an event is delivered over federation. + See `FederationEventDeliveryEvent` for details. + + Performance note: + Registering this hook causes a performance (caching) optimisation on the + Federation `/state` endpoint to be bypassed. + """ + if on_event_delivered_over_federation is not None: + self._on_event_delivered_over_federation_callbacks.append( + on_event_delivered_over_federation + ) + + async def notify_on_event_delivered_over_federation( + self, + server_name: str, + events: Collection[EventBase], + method: FederatedEventDeliveryMethod, + ) -> None: + """Fire the registered callbacks to notify modules that some events were + delivered to another homeserver over federation. + + Does nothing if no callbacks are registered or if there are no events to + report. A callback that raises is logged and does not interrupt the others. + """ + if not events or not self._on_event_delivered_over_federation_callbacks: + return + + delivery = FederationEventDeliveryEvent( + server_name=server_name, + events=events, + method=method, + ) + for callback in self._on_event_delivered_over_federation_callbacks: + try: + await callback(delivery) + except Exception: + logger.exception( + "Error running on_event_delivered_over_federation callback" + ) diff --git a/tests/module_api/test_federation_callbacks.py b/tests/module_api/test_federation_callbacks.py new file mode 100644 index 0000000000..b8a34a0866 --- /dev/null +++ b/tests/module_api/test_federation_callbacks.py @@ -0,0 +1,378 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +from http import HTTPStatus +from typing import Callable +from unittest.mock import AsyncMock, Mock + +from twisted.internet.testing import MemoryReactor + +from synapse.api.constants import EventTypes +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.config.server import DEFAULT_ROOM_VERSION +from synapse.federation.federation_base import event_from_pdu_json +from synapse.federation.units import Transaction +from synapse.module_api.callbacks.federation import ( + FederatedEventDeliveryMethod, + FederationEventDeliveryEvent, +) +from synapse.rest import admin +from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock + +from tests import unittest + + +class FederationDeliveryCallbackTests(unittest.FederatingHomeserverTestCase): + """ + Tests for `on_event_delivered_over_federation` module callbacks. + """ + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + # Mock out the calls over federation. + self.fed_transport_client = Mock(spec=["send_transaction"]) + self.fed_transport_client.send_transaction = AsyncMock(return_value={}) + + hs = self.setup_test_homeserver( + federation_transport_client=self.fed_transport_client, + ) + + return hs + + def default_config(self) -> JsonDict: + # By default, federation sending is disabled in tests. + # Re-enable it for the main process. + config = super().default_config() + config["federation_sender_instances"] = None + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + # Record every delivery the module callback is told about. + self._deliveries: list[FederationEventDeliveryEvent] = [] + + async def record(delivery: FederationEventDeliveryEvent) -> None: + self._deliveries.append(delivery) + + hs.get_module_api().register_federation_callbacks( + on_event_delivered_over_federation=record + ) + + # Create a public room with the remote server joined + self.creator = self.register_user("creator", "pass") + self.creator_tok = self.login("creator", "pass") + self.room_id = self.helper.create_room_as( + self.creator, tok=self.creator_tok, is_public=True + ) + self.remote_user = f"@remote:{self.OTHER_SERVER_NAME}" + self.inject_room_member(self.room_id, self.remote_user, "join") + + def _assert_only_delivery( + self, + method: FederatedEventDeliveryMethod, + ) -> FederationEventDeliveryEvent: + """ + Assert that exactly one delivery, with the given `method`, is currently recorded + (since the tracker was last cleared) and return it. + + This clears the tracker. + """ + self.assertEqual( + len(self._deliveries), + 1, + f"expected exactly one delivery; saw {self._deliveries!r}", + ) + + delivery = self._deliveries[0] + self._deliveries.clear() + + self.assertEqual(delivery.method, method, delivery) + + return delivery + + def test_backfill(self) -> None: + """ + Tests that the callback is triggered for incoming `/backfill` requests. + """ + ( + message_event_id1, + message_event_id2, + message_event_id3, + ) = self.helper.send_messages(self.room_id, 3, tok=self.creator_tok) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/backfill/{self.room_id}" + f"?v={message_event_id3}&limit=3", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.BACKFILL) + self.assertEqual(delivery.server_name, self.OTHER_SERVER_NAME) + self.assertEqual( + {e.event_id for e in delivery.events}, + {message_event_id1, message_event_id2, message_event_id3}, + ) + + def test_event(self) -> None: + """ + Tests that the callback is triggered for incoming `/event` requests. + """ + (message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/v1/event/{message_event_id}" + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.EVENT) + self.assertIncludes( + {e.event_id for e in delivery.events}, + {message_event_id}, + exact=True, + ) + + def test_event_auth(self) -> None: + """ + Tests that the callback is triggered for incoming `/event_auth` requests. + """ + (message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/event_auth/{self.room_id}/{message_event_id}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery( + FederatedEventDeliveryMethod.EVENT_AUTH + ) + + state_key_pairs_included = {(e.type, e.state_key) for e in delivery.events} + self.assertEqual( + state_key_pairs_included, + { + (EventTypes.Create, ""), + (EventTypes.PowerLevels, ""), + (EventTypes.Member, self.creator), + }, + ) + + def test_state(self) -> None: + """ + Tests that the callback is triggered for incoming `/state` requests. + """ + (message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/state/{self.room_id}?event_id={message_event_id}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.STATE) + + # Check that we got notified about delivery for all the expected state events + # included in a `/state` response (including `pdus` and `auth_chain`) + state_key_pairs_included = {(e.type, e.state_key) for e in delivery.events} + self.assertEqual( + state_key_pairs_included, + { + (EventTypes.Create, ""), + (EventTypes.JoinRules, ""), + (EventTypes.PowerLevels, ""), + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, self.creator), + (EventTypes.Member, self.remote_user), + }, + ) + + def test_get_missing_events(self) -> None: + """ + Tests that the callback is triggered for incoming `/get_missing_events` requests. + """ + ( + message_event_id1, + message_event_id2, + message_event_id3, + ) = self.helper.send_messages(self.room_id, 3, tok=self.creator_tok) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_id}", + { + "earliest_events": [message_event_id1], + "latest_events": [message_event_id3], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery( + FederatedEventDeliveryMethod.GET_MISSING_EVENTS + ) + self.assertIncludes( + {e.event_id for e in delivery.events}, + {message_event_id2}, + exact=True, + ) + + def test_send_join(self) -> None: + """ + Tests that the callback is triggered for incoming `/send_join` requests, + including both the state events and the newly-created join event. + """ + joining_user = f"@joiner:{self.OTHER_SERVER_NAME}" + make_join = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{self.room_id}/{joining_user}" + f"?ver={DEFAULT_ROOM_VERSION}", + ) + self.assertEqual(make_join.code, HTTPStatus.OK, make_join.json_body) + + join_event_dict = make_join.json_body["event"] + self.add_hashes_and_signatures_from_other_server( + join_event_dict, KNOWN_ROOM_VERSIONS[DEFAULT_ROOM_VERSION] + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/send_join/{self.room_id}/x", + content=join_event_dict, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.SEND_JOIN) + + # Check that we got notified about delivery for all the expected state events + # included in a `/send_join` response (the join itself, the room state + # and the auth chain events) + state_key_pairs_included = {(e.type, e.state_key) for e in delivery.events} + self.assertIncludes( + state_key_pairs_included, + { + (EventTypes.Create, ""), + (EventTypes.JoinRules, ""), + (EventTypes.PowerLevels, ""), + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, self.creator), + (EventTypes.Member, self.remote_user), + (EventTypes.Member, joining_user), + }, + exact=True, + ) + + def test_send_outbound_transaction(self) -> None: + """ + Tests that the callback is triggered for outgoing `/send` transactions + when the remote acknowledges the PDU. + """ + + async def _acknowledge_pdus( + transaction: Transaction, + json_data_cb: Callable[[], JsonDict], + ) -> JsonDict: + """ + Acknowledge the PDUs. + """ + body = json_data_cb() + pdu_responses: JsonDict = {} + for pdu_json in body.get("pdus", []): + # We have to construct the event to calculate its event ID + event = event_from_pdu_json( + pdu_json, KNOWN_ROOM_VERSIONS[DEFAULT_ROOM_VERSION] + ) + # Empty dict means 'OK' + pdu_responses[event.event_id] = {} + return {"pdus": pdu_responses} + + self.fed_transport_client.send_transaction.side_effect = _acknowledge_pdus + + # After sending, the event propagates to the federation transmission queue + # and gets fired as a `/send` request + (message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.SEND) + self.assertEqual(delivery.server_name, self.OTHER_SERVER_NAME) + self.assertIncludes( + {e.event_id for e in delivery.events}, {message_event_id}, exact=True + ) + + def test_send_outbound_excludes_rejected_pdus(self) -> None: + """ + Tests that the event is NOT triggered for outgoing `/send` transactions + when the remote marks the PDU as failed. + """ + + async def _error_pdus( + transaction: Transaction, + json_data_cb: Callable[[], JsonDict], + ) -> JsonDict: + """ + Return an error for the PDUs. + """ + body = json_data_cb() + pdu_responses: JsonDict = {} + for pdu_json in body.get("pdus", []): + # We have to construct the event to calculate its event ID + event = event_from_pdu_json( + pdu_json, KNOWN_ROOM_VERSIONS[DEFAULT_ROOM_VERSION] + ) + pdu_responses[event.event_id] = {"error": "failed"} + return {"pdus": pdu_responses} + + self.fed_transport_client.send_transaction.side_effect = _error_pdus + + # After sending, the event propagates to the federation transmission queue + # and gets fired as a `/send` request + (_message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + self.assertIncludes(set(self._deliveries), set(), exact=True) From ade9176934b2f9c5530a314864d1cda939a2bd6a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 30 Jul 2026 09:54:20 -0500 Subject: [PATCH 060/158] 1.158.0rc1 --- CHANGES.md | 32 +++++++++++++++++++++++++++++++ changelog.d/18680.feature | 1 - changelog.d/18831.feature | 1 - changelog.d/18876.feature | 1 - changelog.d/19898.bugfix | 3 --- changelog.d/19962.misc | 1 - changelog.d/19963.doc | 1 - changelog.d/19966.bugfix | 1 - changelog.d/19968.misc | 1 - changelog.d/19984.misc | 1 - changelog.d/20009.bugfix | 1 - changelog.d/20015.misc | 1 - changelog.d/20019.feature | 1 - changelog.d/20020.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 17 files changed, 40 insertions(+), 17 deletions(-) delete mode 100644 changelog.d/18680.feature delete mode 100644 changelog.d/18831.feature delete mode 100644 changelog.d/18876.feature delete mode 100644 changelog.d/19898.bugfix delete mode 100644 changelog.d/19962.misc delete mode 100644 changelog.d/19963.doc delete mode 100644 changelog.d/19966.bugfix delete mode 100644 changelog.d/19968.misc delete mode 100644 changelog.d/19984.misc delete mode 100644 changelog.d/20009.bugfix delete mode 100644 changelog.d/20015.misc delete mode 100644 changelog.d/20019.feature delete mode 100644 changelog.d/20020.misc diff --git a/CHANGES.md b/CHANGES.md index 336ed7addf..0407a98eee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,35 @@ +# Synapse 1.158.0rc1 (2026-07-30) + +## Features + +- Change default room version to 11, implementing [MSC4239](https://github.com/matrix-org/matrix-spec-proposals/pull/4239) as part of Matrix v1.14. ([\#18680](https://github.com/element-hq/synapse/issues/18680)) +- Add animation support to the media thumbnailer, gated behind the `animated` query parameter on the thumbnail endpoint (defaults to off). ([\#18831](https://github.com/element-hq/synapse/issues/18831)) +- Return `M_USER_LIMIT_EXCEEDED` error code for media upload limits from [MSC4335](https://github.com/matrix-org/matrix-spec-proposals/pull/4335). ([\#18876](https://github.com/element-hq/synapse/issues/18876)) +- Add Synapse Module API hook that notifies modules when events are delivered over federation (`register_federation_callbacks(...)`).. ([\#20019](https://github.com/element-hq/synapse/issues/20019)) + +## Bugfixes + +- Fix third-party (3pid) invites over federation failing intermittently in version 12 rooms, whose room IDs no longer encode a server name. ([#19898](https://github.com/element-hq/synapse/issues/19898)) + + Fix `/createRoom` intermittently failing with a 500 error in version 12 rooms when the same user creates several rooms at once, due to colliding room IDs. ([\#19898](https://github.com/element-hq/synapse/issues/19898)) +- Fix server key cache invalidations being silently dropped on workers. ([\#19966](https://github.com/element-hq/synapse/issues/19966)) +- Fix `HomeServer.shutdown()` not being able to cleanly shutdown the homeserver (caused by Rust code referencing Python `DatabasePool`). ([\#20009](https://github.com/element-hq/synapse/issues/20009)) + +## Improved Documentation + +- Clarify the usage of the `guests` parameter when using the List user API with the Matrix Authentication Service integration enabled. ([\#19963](https://github.com/element-hq/synapse/issues/19963)) + +## Internal Changes + +- Update release script JSON schema find/replace task to be compatible with macOS. ([\#19962](https://github.com/element-hq/synapse/issues/19962)) +- Remove alert silencing deploy step from release script instructions as it's no longer necessary. ([\#19968](https://github.com/element-hq/synapse/issues/19968)) +- Link to changelog instead of duplicating content in the tag/release. ([\#19984](https://github.com/element-hq/synapse/issues/19984)) +- Fix `RemoteJoinHelper` test helper signing events with the default room version (room version mismatch). ([\#20015](https://github.com/element-hq/synapse/issues/20015)) +- Fix `assertIncludes` printing `None` at the end of the message. ([\#20020](https://github.com/element-hq/synapse/issues/20020)) + + + + # Synapse 1.157.2 (2026-07-28) This security release addresses several vulnerabilities. diff --git a/changelog.d/18680.feature b/changelog.d/18680.feature deleted file mode 100644 index e4280d4678..0000000000 --- a/changelog.d/18680.feature +++ /dev/null @@ -1 +0,0 @@ -Change default room version to 11, implementing [MSC4239](https://github.com/matrix-org/matrix-spec-proposals/pull/4239) as part of Matrix v1.14. diff --git a/changelog.d/18831.feature b/changelog.d/18831.feature deleted file mode 100644 index 022105c0b5..0000000000 --- a/changelog.d/18831.feature +++ /dev/null @@ -1 +0,0 @@ -Add animation support to the media thumbnailer, gated behind the `animated` query parameter on the thumbnail endpoint (defaults to off). diff --git a/changelog.d/18876.feature b/changelog.d/18876.feature deleted file mode 100644 index d62540f2f6..0000000000 --- a/changelog.d/18876.feature +++ /dev/null @@ -1 +0,0 @@ -Return `M_USER_LIMIT_EXCEEDED` error code for media upload limits from [MSC4335](https://github.com/matrix-org/matrix-spec-proposals/pull/4335). diff --git a/changelog.d/19898.bugfix b/changelog.d/19898.bugfix deleted file mode 100644 index 0d2a6d5063..0000000000 --- a/changelog.d/19898.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Fix third-party (3pid) invites over federation failing intermittently in version 12 rooms, whose room IDs no longer encode a server name. ([#19898](https://github.com/element-hq/synapse/issues/19898)) - -Fix `/createRoom` intermittently failing with a 500 error in version 12 rooms when the same user creates several rooms at once, due to colliding room IDs. \ No newline at end of file diff --git a/changelog.d/19962.misc b/changelog.d/19962.misc deleted file mode 100644 index 2f47794a67..0000000000 --- a/changelog.d/19962.misc +++ /dev/null @@ -1 +0,0 @@ -Update release script JSON schema find/replace task to be compatible with macOS. diff --git a/changelog.d/19963.doc b/changelog.d/19963.doc deleted file mode 100644 index de11804be5..0000000000 --- a/changelog.d/19963.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify the usage of the `guests` parameter when using the List user API with the Matrix Authentication Service integration enabled. diff --git a/changelog.d/19966.bugfix b/changelog.d/19966.bugfix deleted file mode 100644 index 5940159f56..0000000000 --- a/changelog.d/19966.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix server key cache invalidations being silently dropped on workers. diff --git a/changelog.d/19968.misc b/changelog.d/19968.misc deleted file mode 100644 index 1526938614..0000000000 --- a/changelog.d/19968.misc +++ /dev/null @@ -1 +0,0 @@ -Remove alert silencing deploy step from release script instructions as it's no longer necessary. diff --git a/changelog.d/19984.misc b/changelog.d/19984.misc deleted file mode 100644 index 7083017a58..0000000000 --- a/changelog.d/19984.misc +++ /dev/null @@ -1 +0,0 @@ -Link to changelog instead of duplicating content in the tag/release. diff --git a/changelog.d/20009.bugfix b/changelog.d/20009.bugfix deleted file mode 100644 index 474afcef09..0000000000 --- a/changelog.d/20009.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `HomeServer.shutdown()` not being able to cleanly shutdown the homeserver (caused by Rust code referencing Python `DatabasePool`). diff --git a/changelog.d/20015.misc b/changelog.d/20015.misc deleted file mode 100644 index 2ef49ff017..0000000000 --- a/changelog.d/20015.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `RemoteJoinHelper` test helper signing events with the default room version (room version mismatch). diff --git a/changelog.d/20019.feature b/changelog.d/20019.feature deleted file mode 100644 index 2744fb4a8d..0000000000 --- a/changelog.d/20019.feature +++ /dev/null @@ -1 +0,0 @@ -Add Synapse Module API hook that notifies modules when events are delivered over federation (`register_federation_callbacks(...)`).. diff --git a/changelog.d/20020.misc b/changelog.d/20020.misc deleted file mode 100644 index aa77313581..0000000000 --- a/changelog.d/20020.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `assertIncludes` printing `None` at the end of the message. diff --git a/debian/changelog b/debian/changelog index 38a6625e8b..16527b136f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.158.0~rc1) stable; urgency=medium + + * New synapse release 1.158.0rc1. + + -- Synapse Packaging team Thu, 30 Jul 2026 14:53:30 +0000 + matrix-synapse-py3 (1.157.2) stable; urgency=medium * New Synapse release 1.157.2. diff --git a/pyproject.toml b/pyproject.toml index 4dd059a45c..5f452adeab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.157.2" +version = "1.158.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index b284716c76..c2153f3f2a 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.157/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.158/synapse-config.schema.json type: object properties: modules: From 446bc852de8ebc34b98f88e37efbc5e1fe26e0a7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 30 Jul 2026 10:52:13 -0500 Subject: [PATCH 061/158] Split #19898 changelog across two points --- CHANGES.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0407a98eee..c244b050dd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,8 +10,7 @@ ## Bugfixes - Fix third-party (3pid) invites over federation failing intermittently in version 12 rooms, whose room IDs no longer encode a server name. ([#19898](https://github.com/element-hq/synapse/issues/19898)) - - Fix `/createRoom` intermittently failing with a 500 error in version 12 rooms when the same user creates several rooms at once, due to colliding room IDs. ([\#19898](https://github.com/element-hq/synapse/issues/19898)) +- Fix `/createRoom` intermittently failing with a 500 error in version 12 rooms when the same user creates several rooms at once, due to colliding room IDs. ([\#19898](https://github.com/element-hq/synapse/issues/19898)) - Fix server key cache invalidations being silently dropped on workers. ([\#19966](https://github.com/element-hq/synapse/issues/19966)) - Fix `HomeServer.shutdown()` not being able to cleanly shutdown the homeserver (caused by Rust code referencing Python `DatabasePool`). ([\#20009](https://github.com/element-hq/synapse/issues/20009)) From 305e590bea8b1bfc18b4fe39d47e749e8f7c2bc0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 30 Jul 2026 10:55:30 -0500 Subject: [PATCH 062/158] Link and use canonical API name --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c244b050dd..210a219c61 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,7 +16,7 @@ ## Improved Documentation -- Clarify the usage of the `guests` parameter when using the List user API with the Matrix Authentication Service integration enabled. ([\#19963](https://github.com/element-hq/synapse/issues/19963)) +- Clarify the usage of the `guests` parameter when using the [List Accounts (V2) admin API](https://element-hq.github.io/synapse/develop/admin_api/user_admin_api.html#list-accounts-v2) with the Matrix Authentication Service integration enabled. ([\#19963](https://github.com/element-hq/synapse/issues/19963)) ## Internal Changes From e4644ae2b9cf9cc7ff760feae408e183a9232ae6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 30 Jul 2026 11:03:39 -0500 Subject: [PATCH 063/158] Remove duplicated punctuation --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 210a219c61..5eabc424a7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,7 @@ - Change default room version to 11, implementing [MSC4239](https://github.com/matrix-org/matrix-spec-proposals/pull/4239) as part of Matrix v1.14. ([\#18680](https://github.com/element-hq/synapse/issues/18680)) - Add animation support to the media thumbnailer, gated behind the `animated` query parameter on the thumbnail endpoint (defaults to off). ([\#18831](https://github.com/element-hq/synapse/issues/18831)) - Return `M_USER_LIMIT_EXCEEDED` error code for media upload limits from [MSC4335](https://github.com/matrix-org/matrix-spec-proposals/pull/4335). ([\#18876](https://github.com/element-hq/synapse/issues/18876)) -- Add Synapse Module API hook that notifies modules when events are delivered over federation (`register_federation_callbacks(...)`).. ([\#20019](https://github.com/element-hq/synapse/issues/20019)) +- Add Synapse Module API hook that notifies modules when events are delivered over federation (`register_federation_callbacks(...)`). ([\#20019](https://github.com/element-hq/synapse/issues/20019)) ## Bugfixes From f61950d54d815b1c11f447bc740a2a2e4f59b522 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 31 Jul 2026 13:58:20 -0500 Subject: [PATCH 064/158] Fix release announcement to link to correct release branch of changelog (#20023) Without this change, it was linking to `develop` as we were on the `develop` branch because of the previous `merge-back` step at this point in the release process. Follow-up to https://github.com/element-hq/synapse/pull/19984 as this was an oversight and I assumed we would still be on the `release-v1.158` branch by that point. --- changelog.d/20023.misc | 1 + scripts-dev/release.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20023.misc diff --git a/changelog.d/20023.misc b/changelog.d/20023.misc new file mode 100644 index 0000000000..17dd927536 --- /dev/null +++ b/changelog.d/20023.misc @@ -0,0 +1 @@ +Fix release script announcement to link to correct release branch of changelog. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 37fd560123..f78c2c0ab0 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -736,8 +736,8 @@ def announce(_gh_token: str | None) -> None: def _announce() -> None: """Generate markdown to announce the release.""" - synapse_repo = get_repo_and_check_clean_checkout() current_version = get_package_version() + release_branch_name = get_release_branch_name(current_version) tag_name = f"v{current_version}" is_rc = "rc" in tag_name @@ -756,7 +756,7 @@ Hi everyone. Synapse {current_version} has just been released. ) release_text += f""" -[notes](https://github.com/element-hq/synapse/blob/{synapse_repo.active_branch.name}/CHANGES.md) | \ +[notes](https://github.com/element-hq/synapse/blob/{release_branch_name}/CHANGES.md) | \ [docker](https://hub.docker.com/r/matrixdotorg/synapse/tags?name={tag_name}) | \ [debs](https://packages.matrix.org/debian/) | \ [pypi](https://pypi.org/project/matrix-synapse/{current_version}/)""" From 415a869f1f53bd4bf22a69e74081875dc9c17735 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Mon, 3 Aug 2026 13:47:26 -0500 Subject: [PATCH 065/158] fix RemoteJoinHelper to handle v12 rooms (#20021) While preparing for the default room version change to "12", the new tests for [FederationJoinUpgradedRoomTestCase](https://github.com/element-hq/synapse/blob/36664cac4fdec38993c68078cd9a82452b56dca7/tests/federation/test_federation_join_upgraded_room.py#L44) were unable to handle creation events inside of the manufactured `auth_events` sections. All of these tests fail. Related to https://github.com/element-hq/synapse/pull/20015 as another recent fix to `RemoteJoinHelper` ### Testing strategy To test, I recommend changing the `DEFAULT_ROOM_VERSION` in [the `server.py` config file](https://github.com/element-hq/synapse/blob/36664cac4fdec38993c68078cd9a82452b56dca7/synapse/config/server.py#L179) to `"12"`, then running `poetry run trial tests.federation.test_federation_join_upgraded_room.FederationJoinUpgradedRoomTestCase` for these selective tests. Alternatively, creating the `default_config()` for the `TestCase` in question to set the default room version, but that is more work. --- changelog.d/20021.misc | 1 + tests/federation/_remote_join.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 changelog.d/20021.misc diff --git a/changelog.d/20021.misc b/changelog.d/20021.misc new file mode 100644 index 0000000000..9291f5eea0 --- /dev/null +++ b/changelog.d/20021.misc @@ -0,0 +1 @@ +Fix `RemoteJoinHelper` test helper to handle room version "12" rooms. Contributed by @famedly @jason-famedly. diff --git a/tests/federation/_remote_join.py b/tests/federation/_remote_join.py index feb9c528c9..8a4e438466 100644 --- a/tests/federation/_remote_join.py +++ b/tests/federation/_remote_join.py @@ -146,7 +146,9 @@ class RemoteJoinHelper: "type": EventTypes.Member, "state_key": remote_creator_user_id, "content": {"membership": Membership.JOIN}, - "auth_events": [room_create_event.event_id], + "auth_events": [room_create_event.event_id] + if not room_version.msc4291_room_ids_as_hashes + else [], "prev_events": [room_create_event.event_id], }, room_version=room_version, @@ -173,7 +175,9 @@ class RemoteJoinHelper: "auth_events": [ room_create_event.event_id, creator_membership_event.event_id, - ], + ] + if not room_version.msc4291_room_ids_as_hashes + else [creator_membership_event.event_id], "prev_events": [prev_event.event_id], }, room_version=room_version, @@ -215,7 +219,9 @@ class RemoteJoinHelper: "auth_events": [ room_create_event.event_id, creator_membership_event.event_id, - ], + ] + if not room_version.msc4291_room_ids_as_hashes + else [creator_membership_event.event_id], "prev_events": [ extra_events[-1].event_id if extra_events @@ -254,7 +260,9 @@ class RemoteJoinHelper: "auth_events": [ room_create_event.event_id, invite_membership_event.event_id, - ], + ] + if not room_version.msc4291_room_ids_as_hashes + else [invite_membership_event.event_id], "prev_events": [invite_membership_event.event_id], }, room_version=room_version, From 2c8bf9a3ab0132b430c7e0f859257b79b8b6cb40 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Aug 2026 09:08:19 -0500 Subject: [PATCH 066/158] Drop support for Ubuntu 25.10 'Questing Quokka', add support for Ubuntu 26.04 'Resolute Raccoon' (#20039) Fix https://github.com/element-hq/synapse/issues/20031 ### Dev notes - https://endoflife.date/ubuntu - https://endoflife.date/debian - https://endoflife.date/python Finding Python version on distros: - https://packages.debian.org/trixie/python3 - https://packages.ubuntu.com/resolute/python3 --- changelog.d/20039.misc | 1 + changelog.d/20039.removal | 1 + docs/upgrade.md | 8 ++++++++ scripts-dev/build_debian_packages.py | 10 +++++----- 4 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20039.misc create mode 100644 changelog.d/20039.removal diff --git a/changelog.d/20039.misc b/changelog.d/20039.misc new file mode 100644 index 0000000000..d8900d1223 --- /dev/null +++ b/changelog.d/20039.misc @@ -0,0 +1 @@ +Add package build targets for Ubuntu 26.04 'Resolute Raccoon'. diff --git a/changelog.d/20039.removal b/changelog.d/20039.removal new file mode 100644 index 0000000000..3a83c77275 --- /dev/null +++ b/changelog.d/20039.removal @@ -0,0 +1 @@ +Remove package build targets for Ubuntu 25.10 'Questing Quokka' (end-of-life 2026-07-01). diff --git a/docs/upgrade.md b/docs/upgrade.md index 372e9d147b..2f65d86cbe 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,14 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.158.0 + +## Drop support for Ubuntu 25.10 'Questing Quokka', add support for Ubuntu 26.04 'Resolute Raccoon' + +Ubuntu 25.10 'Questing Quokka' [is end-of-life as of +2026-07-01](https://endoflife.date/ubuntu). This release drops support for Ubuntu 25.10, +and in its place adds support for Ubuntu 26.04 'Resolute Raccoon'. + # Upgrading to v1.157.0 ## MSC3861 Auth Delegation must be migrated to stable Matrix Authentication Service integration diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index cdde81ee2c..bd5c890711 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -27,12 +27,12 @@ from typing import Collection, Sequence # to remove references to the distibution across Synapse (search for "bookworm" for # example) DISTS = ( - "debian:bookworm", # (EOL 2026-06) (our EOL forced by Python 3.11 is 2027-10-24) + "debian:bookworm", # (EOL 2028-06-30) (our EOL forced by Python 3.11 is 2027-10-31) "debian:sid", # (rolling distro, no EOL) - "ubuntu:jammy", # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04) - "ubuntu:noble", # 24.04 LTS (EOL 2029-06) - "ubuntu:questing", # 25.10 (EOL 2026-07) - "debian:trixie", # (EOL not specified yet) + "ubuntu:jammy", # 22.04 LTS (EOL 2027-04-01) (our EOL forced by Python 3.10 is 2026-10-31) + "ubuntu:noble", # 24.04 LTS (EOL 2029-05-31) (our EOL forced by Python 3.12 is 2028-10-31) + "debian:trixie", # (EOL 2030-06-30) (our EOL forced by Python 3.13 is 2029-10-31) + "ubuntu:resolute", # 26.04 (EOL 2031-04-30) (our EOL forced by Python 3.14 is 2030-10-31) ) DESC = """\ From 5b29f8160ee7651e2ac5b8145f50214569da8362 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Aug 2026 09:08:19 -0500 Subject: [PATCH 067/158] Drop support for Ubuntu 25.10 'Questing Quokka', add support for Ubuntu 26.04 'Resolute Raccoon' (#20039) Fix https://github.com/element-hq/synapse/issues/20031 ### Dev notes - https://endoflife.date/ubuntu - https://endoflife.date/debian - https://endoflife.date/python Finding Python version on distros: - https://packages.debian.org/trixie/python3 - https://packages.ubuntu.com/resolute/python3 --- changelog.d/20039.misc | 1 + changelog.d/20039.removal | 1 + docs/upgrade.md | 8 ++++++++ scripts-dev/build_debian_packages.py | 10 +++++----- 4 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20039.misc create mode 100644 changelog.d/20039.removal diff --git a/changelog.d/20039.misc b/changelog.d/20039.misc new file mode 100644 index 0000000000..d8900d1223 --- /dev/null +++ b/changelog.d/20039.misc @@ -0,0 +1 @@ +Add package build targets for Ubuntu 26.04 'Resolute Raccoon'. diff --git a/changelog.d/20039.removal b/changelog.d/20039.removal new file mode 100644 index 0000000000..3a83c77275 --- /dev/null +++ b/changelog.d/20039.removal @@ -0,0 +1 @@ +Remove package build targets for Ubuntu 25.10 'Questing Quokka' (end-of-life 2026-07-01). diff --git a/docs/upgrade.md b/docs/upgrade.md index 372e9d147b..2f65d86cbe 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,14 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.158.0 + +## Drop support for Ubuntu 25.10 'Questing Quokka', add support for Ubuntu 26.04 'Resolute Raccoon' + +Ubuntu 25.10 'Questing Quokka' [is end-of-life as of +2026-07-01](https://endoflife.date/ubuntu). This release drops support for Ubuntu 25.10, +and in its place adds support for Ubuntu 26.04 'Resolute Raccoon'. + # Upgrading to v1.157.0 ## MSC3861 Auth Delegation must be migrated to stable Matrix Authentication Service integration diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index cdde81ee2c..bd5c890711 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -27,12 +27,12 @@ from typing import Collection, Sequence # to remove references to the distibution across Synapse (search for "bookworm" for # example) DISTS = ( - "debian:bookworm", # (EOL 2026-06) (our EOL forced by Python 3.11 is 2027-10-24) + "debian:bookworm", # (EOL 2028-06-30) (our EOL forced by Python 3.11 is 2027-10-31) "debian:sid", # (rolling distro, no EOL) - "ubuntu:jammy", # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04) - "ubuntu:noble", # 24.04 LTS (EOL 2029-06) - "ubuntu:questing", # 25.10 (EOL 2026-07) - "debian:trixie", # (EOL not specified yet) + "ubuntu:jammy", # 22.04 LTS (EOL 2027-04-01) (our EOL forced by Python 3.10 is 2026-10-31) + "ubuntu:noble", # 24.04 LTS (EOL 2029-05-31) (our EOL forced by Python 3.12 is 2028-10-31) + "debian:trixie", # (EOL 2030-06-30) (our EOL forced by Python 3.13 is 2029-10-31) + "ubuntu:resolute", # 26.04 (EOL 2031-04-30) (our EOL forced by Python 3.14 is 2030-10-31) ) DESC = """\ From 7a3e98b6f77ee3a5fe4dbeb934b0a0c1721e6afe Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Aug 2026 12:53:51 -0500 Subject: [PATCH 068/158] 1.158.0 --- CHANGES.md | 13 +++++++++++++ changelog.d/20039.misc | 1 - changelog.d/20039.removal | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 5 files changed, 20 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/20039.misc delete mode 100644 changelog.d/20039.removal diff --git a/CHANGES.md b/CHANGES.md index 5eabc424a7..bd8b40bab3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,16 @@ +# Synapse 1.158.0 (2026-08-04) + +## Deprecations and Removals + +- Remove package build targets for Ubuntu 25.10 'Questing Quokka' (end-of-life 2026-07-01). ([\#20039](https://github.com/element-hq/synapse/issues/20039)) + +## Internal Changes + +- Add package build targets for Ubuntu 26.04 'Resolute Raccoon'. ([\#20039](https://github.com/element-hq/synapse/issues/20039)) + + + + # Synapse 1.158.0rc1 (2026-07-30) ## Features diff --git a/changelog.d/20039.misc b/changelog.d/20039.misc deleted file mode 100644 index d8900d1223..0000000000 --- a/changelog.d/20039.misc +++ /dev/null @@ -1 +0,0 @@ -Add package build targets for Ubuntu 26.04 'Resolute Raccoon'. diff --git a/changelog.d/20039.removal b/changelog.d/20039.removal deleted file mode 100644 index 3a83c77275..0000000000 --- a/changelog.d/20039.removal +++ /dev/null @@ -1 +0,0 @@ -Remove package build targets for Ubuntu 25.10 'Questing Quokka' (end-of-life 2026-07-01). diff --git a/debian/changelog b/debian/changelog index 16527b136f..a92e6d7ce6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.158.0) stable; urgency=medium + + * New synapse release 1.158.0. + + -- Synapse Packaging team Tue, 04 Aug 2026 17:52:05 +0000 + matrix-synapse-py3 (1.158.0~rc1) stable; urgency=medium * New synapse release 1.158.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 5f452adeab..a7f8b124e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.158.0rc1" +version = "1.158.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 3cb6c3484daa59c56bbc3d8df295e011c9f0d2cf Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 5 Aug 2026 11:11:30 -0500 Subject: [PATCH 069/158] Document how to capture a JSON snapshot of a Grafana dashboard (#20048) To make it easy to share to people and aid in debugging when they run into performance issues. Unfortunately, the Grafana UI doesn't make the export or import steps easy so it involves some manual Grafana API calls. As first explored in https://github.com/element-hq/synapse-rust-apps/pull/397 --- changelog.d/20048.misc | 1 + contrib/grafana/README.md | 57 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 changelog.d/20048.misc diff --git a/changelog.d/20048.misc b/changelog.d/20048.misc new file mode 100644 index 0000000000..07ee0f045a --- /dev/null +++ b/changelog.d/20048.misc @@ -0,0 +1 @@ +Document how to capture a JSON snapshot of a Grafana dashboard to aid in debugging. diff --git a/contrib/grafana/README.md b/contrib/grafana/README.md index 0bbd57439e..4ccfb9b60b 100644 --- a/contrib/grafana/README.md +++ b/contrib/grafana/README.md @@ -4,3 +4,60 @@ 1. Have your Prometheus scrape your Synapse. https://element-hq.github.io/synapse/latest/metrics-howto.html 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/ 3. Set up required recording rules. [contrib/prometheus](../prometheus) + + +## Sharing a JSON snapshot of a Grafana dashboard + +To aid in debugging, you can share the dashboard with others by creating a snapshot of +the Grafana dashboard and exporting it as JSON. The snapshot will contain all of the +current values of the metrics visible on the dashboard. + +**To capture the JSON snapshot:** + + 1. Visit the Grafana dashboard in your browser + 1. Expand all of the sections on the dashboard and let all of the panels load in (the + snapshot only captures what's loaded on your page) + 1. Use the Grafana UI to capture the snapshot: **Share** (drop down arrow) -> **Share + Snapshot** -> **Publish Snapshot** + - If you run into `413` (`Content Too Large`) errors, you're probably just running + into the upload limit set on your reverse proxy (like nginx) in front of your + Grafana instance. Just increase it and try again. + - You may also run into `400` (`Bad Request`) which appear as `bad request data` + in the Grafana UI if the snapshot is larger than 100 MB. Grafana introduced a + [100 MB + limit](https://github.com/grafana/grafana/blob/555d6dde60b0f49acd453c7293b1cd518fda3592/pkg/web/binding.go#L13-L14) + as part of their [2026 June security + releases](https://github.com/grafana/grafana/pull/125789). There is no + workaround on this app limit, so you will have to reduce the time window, + number of panels shown, etc. + 1. Grab the snapshot ID from the link generated in the last step or find it from the + list of snapshots on https://localhost:3000/dashboard/snapshots + 1. To export the JSON, you have to use the [API for getting a + snapshot](https://grafana.com/docs/grafana/latest/developer-resources/api-reference/http-api/api-legacy/snapshot/#get-snapshot-by-key) + (update the snapshot ID in the command below): + ```shell + curl --request GET \ + --header 'Content-Type: application/json' \ + --output ~/Downloads/2026-08-16-synapse-myhomeserver.com.json \ + http://admin:admin@localhost:3000/api/snapshots/nerimdSEDz530rM6CiwkEFi09A1841yF + ``` + 1. If you're trying to upload to GitHub, keep in mind that GitHub has a 25MB limit for + attachments on issues. As an alternative, you could create a [GitHub + Gist](https://gist.github.com/). If the snapshot is too big to upload via the GitHub + UI, you can create a blank/empty gist and add it via git (gists are git repos). + 1. Once you have the JSON file, you can delete the snapshot from your Grafana instance + to free up space (from the snaphots page, + https://localhost:3000/dashboard/snapshots). The JSON file will still be valid and + can be shared with others. + +**To import the JSON snapshot into Grafana**, you have to use the [API for creating a snapshot](https://grafana.com/docs/grafana/latest/developer-resources/api-reference/http-api/snapshot/#create-new-snapshot) (passing in the whole JSON). + + 1. Import example: + ```shell + cat ~/Downloads/2026-08-16-synapse-myhomeserver.com.json \ + | jq '. += {"name": "2026-08-16-synapse-myhomeserver.com"}' \ + | curl --request POST \ + --header 'Content-Type: application/json' \ + --data @- http://admin:admin@localhost:3000/api/snapshots + ``` + 1. Then you can find the snapshot on https://localhost:3000/dashboard/snapshots to view it. From 320265f09bfb8f82e926b17cbbc60ebe1b4a3edc Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 6 Aug 2026 10:15:21 +0100 Subject: [PATCH 070/158] Remove broken `DROP` statements for SQLite in `make_full_schema` script. (#20028) Broken out of #20027 because I'd like to have it land first. > It seems a lot of time in our trial tests goes towards setting up the database. (The same is probably true of Complement too) > > We haven't done a full schema for about 20 schema versions, so no surprise! > > As a result, I want to produce a full schema soon. When running `make_full_schema.sh`, these drop statements now cause the error: ``` Parse error near line 2: table event_search_content may not be dropped Parse error near line 3: table event_search_segments may not be dropped Parse error near line 4: table event_search_segdir may not be dropped Parse error near line 5: table event_search_docsize may not be dropped Parse error near line 6: table event_search_stat may not be dropped Parse error near line 7: table user_directory_search_content may not be dropped Parse error near line 8: table user_directory_search_segments may not be dropped Parse error near line 9: table user_directory_search_segdir may not be dropped Parse error near line 10: table user_directory_search_docsize may not be dropped Parse error near line 11: table user_directory_search_stat may not be dropped ``` It seems SQLite has cracked down on code that edits the internal tables. Because SQLite dumps the schema with `CREATE TABLE IF NOT EXISTS` for these virtual tables, it's harmless to leave them in the schema dump. --------- Signed-off-by: Olivier 'reivilibre' --- changelog.d/20028.misc | 1 + scripts-dev/make_full_schema.sh | 18 ++++-------------- 2 files changed, 5 insertions(+), 14 deletions(-) create mode 100644 changelog.d/20028.misc diff --git a/changelog.d/20028.misc b/changelog.d/20028.misc new file mode 100644 index 0000000000..ad3738f00e --- /dev/null +++ b/changelog.d/20028.misc @@ -0,0 +1 @@ +Remove broken `DROP` statements for SQLite in `make_full_schema` script. \ No newline at end of file diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index 473f54772a..196869ce8f 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -232,20 +232,10 @@ psql "$POSTGRES_MAIN_DB_NAME" -w <<< "$DROP_COMMON_TABLES" psql "$POSTGRES_STATE_DB_NAME" -w <<< "$DROP_COMMON_TABLES" # For Reasons(TM), SQLite's `.schema` also dumps out "shadow tables", the implementation -# details behind full text search tables. Omit these from the dumps. - -sqlite3 "$SQLITE_MAIN_DB" <<< " -DROP TABLE event_search_content; -DROP TABLE event_search_segments; -DROP TABLE event_search_segdir; -DROP TABLE event_search_docsize; -DROP TABLE event_search_stat; -DROP TABLE user_directory_search_content; -DROP TABLE user_directory_search_segments; -DROP TABLE user_directory_search_segdir; -DROP TABLE user_directory_search_docsize; -DROP TABLE user_directory_search_stat; -" +# details behind full text search tables. +# Previously we omitted these from the dumps by dropping them beforehand, +# but nowadays it seems to be forbidden to drop those. +# The emitted dump adds `IF NOT EXISTS` text for them, so it's harmless. echo "Dumping SQLite3 schema..." From 98a0cae3c3c9311686019a6739b5ca07fb10c76b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 10:09:27 +0000 Subject: [PATCH 071/158] Bump the patches group with 6 updates (#20042) Signed-off-by: dependabot[bot] --- Cargo.lock | 107 +++++++++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de9bdb900b..50daff2dd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" @@ -25,13 +25,13 @@ checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -186,7 +186,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -218,9 +218,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -243,15 +243,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -260,38 +260,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -896,7 +896,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -909,7 +909,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1057,9 +1057,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1069,9 +1069,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -1254,9 +1254,9 @@ checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1264,29 +1264,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -1390,6 +1390,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "synapse" version = "0.1.0" @@ -1440,7 +1451,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1466,7 +1477,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1698,7 +1709,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -1733,7 +1744,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1912,7 +1923,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "synstructure", ] @@ -1933,7 +1944,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1953,7 +1964,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "synstructure", ] @@ -1993,7 +2004,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] From 701baedbea72616be1469cc228cb3d97ba96c80a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 10:09:50 +0000 Subject: [PATCH 072/158] Bump actions/checkout from 7.0.0 to 7.0.1 in the minor-and-patches group (#20043) Signed-off-by: dependabot[bot] --- .github/workflows/complement_tests.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/docs-pr.yaml | 4 +- .github/workflows/docs.yaml | 2 +- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 8 ++-- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/push_complement_image.yml | 6 +-- .github/workflows/release-artifacts.yml | 8 ++-- .github/workflows/schema.yaml | 4 +- .github/workflows/tests.yml | 44 ++++++++++----------- .github/workflows/triage_labelled.yml | 2 +- .github/workflows/twisted_trunk.yml | 8 ++-- 13 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index 629a48dcc7..cb0c5ba1ea 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout synapse codebase - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: path: synapse diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b087e7f8cf..5f7ed77d15 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,7 +31,7 @@ jobs: uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index a1b355e74f..51d0f6e11b 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -13,7 +13,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -50,7 +50,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index f8479a8c51..3908003504 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,7 @@ jobs: needs: - pre steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index aaf6b2ba65..22ef26e78c 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 0b39c5c372..61f10d5335 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -42,7 +42,7 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -77,7 +77,7 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -151,7 +151,7 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -201,7 +201,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 856e51e7ab..51bea8f763 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -16,7 +16,7 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index ac74cf9cf8..257b1a668d 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,17 +33,17 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: github.event_name == 'push' with: ref: master diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 2482662726..87812566ea 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,7 +27,7 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -61,7 +61,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: path: src @@ -129,7 +129,7 @@ jobs: os: "ubuntu-24.04-arm" steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: @@ -167,7 +167,7 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index c4d46a1058..3aad64a601 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -14,7 +14,7 @@ jobs: name: Ensure Synapse config schema is valid runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -40,7 +40,7 @@ jobs: name: Ensure generated documentation is up-to-date runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cd967f6ade..d2fa1ca1c4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -106,7 +106,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -126,7 +126,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -136,7 +136,7 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -149,7 +149,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -171,7 +171,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -207,7 +207,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -216,7 +216,7 @@ jobs: if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 @@ -234,7 +234,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -253,7 +253,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -271,7 +271,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -307,7 +307,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -326,7 +326,7 @@ jobs: if: ${{ needs.changes.outputs.golangci == 'true' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: @@ -344,7 +344,7 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -393,7 +393,7 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -414,7 +414,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -470,7 +470,7 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -533,7 +533,7 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -583,7 +583,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers @@ -630,7 +630,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: sudo apt-get -qq install xmlsec1 postgresql-client - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -673,7 +673,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -726,7 +726,7 @@ jobs: - changes steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -747,7 +747,7 @@ jobs: - changes steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index f6880fef1f..f852cfcf11 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -22,7 +22,7 @@ jobs: # This field is case-sensitive. TARGET_STATUS: Needs info steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Only clone the script file we care about, instead of the whole repo. sparse-checkout: .ci/scripts/triage_labelled_issue.sh diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 80984ceeb3..88b279ffce 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust @@ -115,7 +115,7 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -172,7 +172,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 556154848c54ccca9c1cff5bfa4a0baad8d752aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 10:11:56 +0000 Subject: [PATCH 073/158] Bump gitpython from 3.1.54 to 3.1.57 (#20046) Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3b37350eaf..329e21b740 100644 --- a/poetry.lock +++ b/poetry.lock @@ -583,14 +583,14 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.54" +version = "3.1.57" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.54-py3-none-any.whl", hash = "sha256:b90d7b3d9bc0238681d24369130826f0dcdb0ceaa45db67cf1d4ffa4c302dedf"}, - {file = "gitpython-3.1.54.tar.gz", hash = "sha256:53f2085e24a2cda300eed7c3fc5f1559ae289634b725e98acaf4791940247aa0"}, + {file = "gitpython-3.1.57-py3-none-any.whl", hash = "sha256:4ccf7d73c10f5c9e76043fbb2675ac5a1b3ff5b41e648f56bcbed5f63792ecaf"}, + {file = "gitpython-3.1.57.tar.gz", hash = "sha256:c493ec57c0ef6b19743798b6a5af859c71814b524e7e6f97baa2f8e658961488"}, ] [package.dependencies] @@ -598,7 +598,7 @@ gitdb = ">=4.0.1,<5" [package.extras] doc = ["sphinx (>=7.4.7,<8)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] +test = ["basedpyright (==1.39.9) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] name = "hiredis" From c5f02a93137ccee0dedb7e8db0a99620d6c293e3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:35:07 +0100 Subject: [PATCH 074/158] Introduce support for MSC4429: Profile Updates for Legacy Sync (#19556) Implements support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). Paired with https://github.com/matrix-org/complement/pull/849 and https://github.com/matrix-org/sytest/tree/anoa/msc4429 Tracking issue for removing unstable identifiers in Synapse: https://github.com/element-hq/synapse/issues/19891 Further improvements tracked in: https://github.com/element-hq/synapse/issues/19981 --------- Co-authored-by: Half-Shot Co-authored-by: Jason Robinson Co-authored-by: Olivier 'reivilibre' --- changelog.d/19556.feature | 2 + .../conf/workers-shared-extra.yaml.j2 | 2 + .../configuration/config_documentation.md | 11 + rust/src/config/mod.rs | 1 + rust/src/handlers/versions.rs | 4 + schema/synapse-config.schema.yaml | 15 + scripts-dev/check_schema_delta.py | 24 +- scripts-dev/complement.sh | 1 + scripts-dev/mypy_synapse_plugin.py | 5 + synapse/_scripts/synapse_port_db.py | 4 + synapse/api/constants.py | 51 + synapse/api/filtering.py | 17 + synapse/config/server.py | 6 + synapse/config/workers.py | 21 +- synapse/handlers/deactivate_account.py | 4 +- synapse/handlers/profile.py | 230 ++- synapse/handlers/room_member.py | 1 - synapse/handlers/sso.py | 18 +- synapse/handlers/sync.py | 358 ++++ synapse/module_api/__init__.py | 5 +- synapse/notifier.py | 1 + synapse/replication/http/__init__.py | 2 + synapse/replication/http/profile.py | 150 ++ synapse/replication/tcp/client.py | 18 + synapse/replication/tcp/handler.py | 7 + synapse/replication/tcp/streams/__init__.py | 3 + synapse/replication/tcp/streams/_base.py | 77 +- synapse/rest/admin/users.py | 26 +- synapse/rest/client/profile.py | 51 +- synapse/rest/client/sync.py | 10 + synapse/rest/synapse/mas/users.py | 43 +- synapse/storage/_base.py | 15 +- synapse/storage/databases/main/events.py | 126 ++ synapse/storage/databases/main/profile.py | 828 +++++++-- synapse/storage/databases/main/roommember.py | 121 +- synapse/storage/schema/__init__.py | 1 + .../main/delta/94/07_profile_updates.sql | 60 + .../94/07_profile_updates_seq.sql.postgres | 18 + synapse/streams/events.py | 3 + synapse/types/__init__.py | 17 +- tests/handlers/test_deactivate_account.py | 24 +- tests/handlers/test_profile.py | 676 +++++++- tests/handlers/test_register.py | 10 +- tests/handlers/test_sync.py | 1543 ++++++++++++++++- tests/rest/admin/test_room.py | 4 +- tests/rest/admin/test_user.py | 37 +- tests/rest/client/test_account.py | 30 +- tests/rest/client/test_rooms.py | 4 +- tests/rest/synapse/mas/test_users.py | 14 +- tests/storage/test_main.py | 9 +- tests/storage/test_profile.py | 31 +- 51 files changed, 4380 insertions(+), 359 deletions(-) create mode 100644 changelog.d/19556.feature create mode 100644 synapse/replication/http/profile.py create mode 100644 synapse/storage/schema/main/delta/94/07_profile_updates.sql create mode 100644 synapse/storage/schema/main/delta/94/07_profile_updates_seq.sql.postgres diff --git a/changelog.d/19556.feature b/changelog.d/19556.feature new file mode 100644 index 0000000000..bcb6c5c983 --- /dev/null +++ b/changelog.d/19556.feature @@ -0,0 +1,2 @@ +Add optional support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). +Currently defaults to not enabled, and is limited to local users only for the sync results. \ No newline at end of file diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index e829292aca..64a36522fa 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -15,6 +15,8 @@ enable_registration_without_verification: true bcrypt_rounds: 4 url_preview_enabled: true url_preview_ip_range_blacklist: [] +# MSC4429 Profile updates down legacy /sync +include_profile_updates_in_sync: true ## Registration ## diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 510bc5f8c9..3fb961d6f8 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -336,6 +336,17 @@ Example configuration: include_profile_data_on_invite: false ``` --- +### `include_profile_updates_in_sync` + +*(boolean)* Use this option to include updates of other users' profiles in sync responses, for users who share rooms. +Requires an [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) compatible client, and is currently limited to legacy sync and local users only. +This feature is under development and should be used with caution on busy servers or servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring profile information doesn't leak across rooms. Defaults to `false`. + +Example configuration: +```yaml +include_profile_updates_in_sync: true +``` +--- ### `allow_public_rooms_without_auth` *(boolean)* If set to true, removes the need for authentication to access the server's public rooms directory through the client API, meaning that anyone can query the room directory. Defaults to `false`. diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index d79d12a83a..1c97373c27 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -48,6 +48,7 @@ pub struct AuthConfig { #[derive(FromPyObject, Clone)] pub struct ServerConfig { pub msc4140_enabled: bool, + pub include_profile_updates_in_sync: bool, } #[derive(FromPyObject, Clone)] diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 25da9d23fc..5d35b052bd 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -257,6 +257,9 @@ pub struct UnstableFeatureMap { /// MSC4380: Invite blocking #[serde(rename = "org.matrix.msc4380.stable")] msc4380: bool, + /// MSC4429: Profile updates for legacy /sync. + #[serde(rename = "org.matrix.msc4429")] + msc4429: bool, /// MSC4445: Sync timeline order #[serde(rename = "org.matrix.msc4445.initial_sync_timeline_topological_ordering")] msc4445_initial_sync_timeline_topological_ordering: bool, @@ -316,6 +319,7 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4169: config.experimental.msc4169_enabled, msc4354: config.experimental.msc4354_enabled, msc4380: true, + msc4429: config.server.include_profile_updates_in_sync, msc4445_initial_sync_timeline_topological_ordering: true, msc4491_enabled: config.experimental.msc4491_enabled, msc4143_enabled: config.experimental.msc4143_enabled, diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index c2153f3f2a..6e7880a900 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -275,6 +275,21 @@ properties: default: true examples: - false + include_profile_updates_in_sync: + type: boolean + description: >- + Use this option to include updates of other users' profiles in sync responses, + for users who share rooms. + + Requires an [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) + compatible client, and is currently limited to legacy sync and local users only. + + This feature is under development and should be used with caution on busy servers or + servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring + profile information doesn't leak across rooms. + default: false + examples: + - true allow_public_rooms_without_auth: type: boolean description: diff --git a/scripts-dev/check_schema_delta.py b/scripts-dev/check_schema_delta.py index 12ed5d258c..7f2500ec05 100755 --- a/scripts-dev/check_schema_delta.py +++ b/scripts-dev/check_schema_delta.py @@ -14,6 +14,11 @@ import sqlglot.expressions SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") +# Keep this in sync with synapse.storage.engines._base. The CI job for this +# script deliberately installs only its lightweight parsing dependencies, so we +# avoid importing Synapse here. +AUTO_INCREMENT_PRIMARY_KEYPLACEHOLDER = "$%AUTO_INCREMENT_PRIMARY_KEY%$" + # The base branch we want to check against. We use the main development branch # on the assumption that is what we are developing against. DEVELOP_BRANCH = "develop" @@ -81,7 +86,7 @@ def main(force_colors: bool) -> None: bad_delta_files = [] changed_delta_files = [] for diff in diffs: - if diff.b_path is None: + if diff.deleted_file or diff.b_path is None: # We don't lint deleted files. continue @@ -196,6 +201,10 @@ def check_schema_delta(delta_files: list[str], force_colors: bool) -> bool: ) return True + delta_contents = _replace_auto_increment_primary_key_placeholder( + delta_contents, sql_lang + ) + statements = sqlglot.parse(delta_contents, read=sql_lang) for statement in statements: @@ -244,5 +253,18 @@ def check_schema_delta(delta_files: list[str], force_colors: bool) -> bool: return success +def _replace_auto_increment_primary_key_placeholder( + delta_contents: str, sql_lang: str +) -> str: + """Replace Synapse's auto-increment PK placeholder with parseable SQL.""" + + if sql_lang == "sqlite": + replacement = "INTEGER PRIMARY KEY AUTOINCREMENT" + else: + replacement = "BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY" + + return delta_contents.replace(AUTO_INCREMENT_PRIMARY_KEYPLACEHOLDER, replacement) + + if __name__ == "__main__": main() diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index cca87d42a9..d8e553f446 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -286,6 +286,7 @@ main() { ./tests/msc4155 ./tests/msc4306 ./tests/msc4222 + ./tests/msc4429 ) # Export the list of test packages as a space-separated environment variable, so other diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 7fe4d6cd86..ac1fee7f67 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -45,6 +45,7 @@ from mypy.types import ( AnyType, CallableType, Instance, + LiteralType, NoneType, Options, TupleType, @@ -813,6 +814,10 @@ def is_cacheable( if isinstance(rt, AnyType): return True, ("may be mutable" if verbose else None) + elif isinstance(rt, LiteralType): + # Literal[True] etc + return True, None + elif isinstance(rt, Instance): if ( rt.type.fullname in IMMUTABLE_VALUE_TYPES diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 0b8a289d92..f4f598d27c 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -917,6 +917,10 @@ class Porter: "quarantined_media_id_seq", [("quarantined_media_changes", "stream_id")], ) + await self._setup_sequence( + "profile_updates_sequence", + [("profile_updates", "stream_id")], + ) # Step 3. Get tables. self.progress.set_state("Fetching tables") diff --git a/synapse/api/constants.py b/synapse/api/constants.py index acac057334..041a7f284a 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -410,6 +410,57 @@ class ProfileFields: AVATAR_URL: Final = "avatar_url" +class ProfileUpdateAction(str, enum.Enum): + """ + Enum representing the action of a row in the profile updates stream tables. + The action determines whether a profile field update has occurred, or whether + something else has happened that the sync code should know about, for example + a user joining or leaving a room. + """ + + JOINED_ROOM = "joined_room" + """ + This profile update row action represents a user joining a room. + + When gathering an incremental sync non-lazy response for profile updates, + we always include the full profile of users who have joined a room the syncing + user is a member of, where full profile means all the current profile values the + client asked for, regardless of whether they have changed recently. This ensures + that clients have profiles re-populated for any users who have recently left + shared rooms. + + A scenario example would be as follows: + + * Alice leaves a room with Bob + * Bob's client clears all profile fields from Alice + * Alice joins a room with Bob + * Bob's client does an incremental non-lazy sync + + At the end of the flow Bob should receive all the profile fields the client + is interested in, not just the potential diff, which non-lazy incremental sync + normally includes. This update action currently has no meaning for sync responses + that are not incremental and non-lazy. + """ + LEFT_ROOM = "left_room" + """ + This profile update row action represents a user leaving a room. + + Clients will want to know when they no longer share rooms with a user. This + profile action row allows the sync code to deliver a `null` response for those + profiles, so clients can clear their cache containing the users profile data + they are no longer interested in. + """ + UPDATE = "update" + """ + This profile update row action represents a user updating a profile field. + + Depending on the type of sync (initial/incremental, lazy/non-lazy), either the + diff of profile field updates or all the current profile fields are included + in the sync response. In the latter case the profile update action row signifies + a change, but the client may still get fields that have not changed. + """ + + class StickyEventField(TypedDict): """ Dict content of the `sticky` part of an event. diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 9b47c20437..cbae9133c6 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -123,6 +123,13 @@ USER_FILTER_SCHEMA = { "filter": FILTER_SCHEMA, "room_filter": ROOM_FILTER_SCHEMA, "room_event_filter": ROOM_EVENT_FILTER_SCHEMA, + "profile_fields_filter": { + "type": "object", + "properties": { + "ids": {"type": "array", "items": {"type": "string"}}, + }, + "additionalProperties": True, + }, }, "properties": { "presence": {"$ref": "#/definitions/filter"}, @@ -130,6 +137,9 @@ USER_FILTER_SCHEMA = { "room": {"$ref": "#/definitions/room_filter"}, "event_format": {"type": "string", "enum": ["client", "federation"]}, "event_fields": {"type": "array", "items": {"type": "string"}}, + "org.matrix.msc4429.profile_fields": { + "$ref": "#/definitions/profile_fields_filter" + }, }, "additionalProperties": True, # Allow new fields for forward compatibility } @@ -217,6 +227,13 @@ class FilterCollection: self.event_fields = filter_json.get("event_fields", []) self.event_format = filter_json.get("event_format", "client") + self.profile_fields: set[str] = set() + if hs.config.server.include_profile_updates_in_sync: + profile_fields_filter = filter_json.get("org.matrix.msc4429.profile_fields") + + if isinstance(profile_fields_filter, Mapping): + self.profile_fields = set(profile_fields_filter.get("ids", [])) + def __repr__(self) -> str: return "" % (json.dumps(self._filter_json),) diff --git a/synapse/config/server.py b/synapse/config/server.py index f071040c6e..42d43ea7f6 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -585,6 +585,12 @@ class ServerConfig(Config): " 'allow_public_rooms_over_federation' is set." ) + # Whether to support MSC4429 profile updates down legacy /sync + self.include_profile_updates_in_sync = config.get( + "include_profile_updates_in_sync", + False, + ) + # Check if the legacy "restrict_public_rooms_to_local_users" flag is set. This # flag is now obsolete but we need to check it for backward-compatibility. if config.get("restrict_public_rooms_to_local_users", False): diff --git a/synapse/config/workers.py b/synapse/config/workers.py index fb7378bfc8..c92534b799 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -127,9 +127,10 @@ class WriterLocations: """Specifies the instances that write various streams. Attributes: - events: The instances that write to the event, backfill and `sticky_events` streams. - (`sticky_events` is written to during event persistence so must be handled by the - same stream writers.) + events: The instances that write to the `event`, `backfill`, `sticky_events` and + `profile_updates` streams. + (`sticky_events` and `profile_updates` are written to during event + persistence so must be handled by the same stream writers.) typing: The instances that write to the typing stream. Currently can only be a single instance. to_device: The instances that write to the to_device stream. Currently @@ -142,6 +143,8 @@ class WriterLocations: push_rules: The instances that write to the push stream. Currently can only be a single instance. device_lists: The instances that write to the device list stream. + thread_subscriptions: The instances that write to the thread subscriptions + stream. quarantined_media_changes: The instances that write to the quarantined media changes stream. """ @@ -179,7 +182,7 @@ class WriterLocations: converter=_instance_to_list_converter, ) thread_subscriptions: list[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) quarantined_media_changes: list[str] = attr.ib( @@ -361,8 +364,7 @@ class WorkerConfig(Config): writers = config.get("stream_writers") or {} self.writers = WriterLocations(**writers) - # Check that the configured writers for events and typing also appears in - # `instance_map`. + # Check that the configured writers also appear in `instance_map`. for stream in ( "events", "typing", @@ -371,6 +373,8 @@ class WorkerConfig(Config): "receipts", "presence", "push_rules", + "device_lists", + "thread_subscriptions", ): instances = _instance_to_list_converter(getattr(self.writers, stream)) for instance in instances: @@ -421,6 +425,11 @@ class WorkerConfig(Config): "Must specify at least one instance to handle `device_lists` messages." ) + if len(self.writers.thread_subscriptions) == 0: + raise ConfigError( + "Must specify at least one instance to handle `thread_subscriptions` messages." + ) + self.events_shard_config = RoutableShardedWorkerHandlingConfig( self.writers.events ) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 9ec00d55ad..34596ade16 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -173,7 +173,9 @@ class DeactivateAccountHandler: # in rooms, but these cases behave like message history, following # https://spec.matrix.org/v1.17/client-server-api/#post_matrixclientv3accountdeactivate await self._profile_handler.delete_profile_upon_deactivation( - user, requester, by_admin + target_user=user, + requester=requester, + by_admin=by_admin, ) logger.info("Marking %s as erased", user_id) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index c3886795b6..5f2cec2366 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -34,6 +34,10 @@ from synapse.api.errors import ( StoreError, SynapseError, ) +from synapse.replication.http.profile import ( + ReplicationProfileDeleteField, + ReplicationProfileSetField, +) from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.storage.roommember import ProfileInfo from synapse.types import ( @@ -42,6 +46,7 @@ from synapse.types import ( JsonValue, Requester, ScheduledTask, + StreamKeyType, TaskStatus, UserID, create_requester, @@ -75,6 +80,7 @@ class ProfileHandler: self.clock = hs.get_clock() # nb must be called this for @cached self.store = hs.get_datastores().main self.hs = hs + self._notifier = hs.get_notifier() self.federation = hs.get_federation_client() hs.get_federation_registry().register_query_handler( @@ -99,6 +105,17 @@ class ProfileHandler: ) self._worker_locks = hs.get_worker_locks_handler() + # Profile updates stream + self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._is_events_writer = ( + hs.get_instance_name() in hs.config.worker.writers.events + ) + self._delete_profile_field_client = ReplicationProfileDeleteField.make_client( + self.hs + ) + self._set_profile_field_client = ReplicationProfileSetField.make_client(self.hs) + self._profile_updates_writer_instance = self.hs.config.worker.writers.events[0] + async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDict: """ Get a user's profile as a JSON dictionary. @@ -191,13 +208,13 @@ class ProfileHandler: async def set_displayname( self, + *, target_user: UserID, requester: Requester, new_displayname: str, - *, by_admin: bool = False, propagate: bool = True, - ) -> None: + ) -> int | None: """Set the displayname of a user Preconditions: @@ -207,12 +224,17 @@ class ProfileHandler: updates into rooms, which could cause rooms to be accidentally joined after the deactivated user has left them. + FIXME: This precondition seems to lack a test. + Args: target_user: the user whose displayname is to be changed. requester: The user attempting to make this change. new_displayname: The displayname to give this user. by_admin: Whether this change was made by an administrator. propagate: Whether this change also applies to the user's membership events. + + Returns: + Stream ID of the profile updates stream row that was just inserted. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -252,7 +274,11 @@ class ProfileHandler: authenticated_entity=requester.authenticated_entity, ) - await self.store.set_profile_displayname(target_user, displayname_to_set) + stream_id = await self.store.set_profile_field( + target_user, + ProfileFields.DISPLAYNAME, + displayname_to_set, + ) profile = await self.store.get_profileinfo(target_user) @@ -267,6 +293,8 @@ class ProfileHandler: if propagate: await self._update_join_states(requester, target_user) + return stream_id + async def get_avatar_url(self, target_user: UserID) -> str | None: """ Fetch a user's avatar URL from their profile. @@ -302,13 +330,13 @@ class ProfileHandler: async def set_avatar_url( self, + *, target_user: UserID, requester: Requester, new_avatar_url: str, - *, by_admin: bool = False, propagate: bool = True, - ) -> None: + ) -> int | None: """Set a new avatar URL for a user. Preconditions: @@ -318,12 +346,17 @@ class ProfileHandler: updates into rooms, which could cause rooms to be accidentally joined after the deactivated user has left them. + FIXME: This precondition seems to lack a test. + Args: target_user: the user whose avatar URL is to be changed. requester: The user attempting to make this change. new_avatar_url: The avatar URL to give this user. by_admin: Whether this change was made by an administrator. propagate: Whether this change also applies to the user's membership events. + + Returns: + Stream ID of the profile updates stream row that was just inserted. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -361,7 +394,11 @@ class ProfileHandler: target_user, authenticated_entity=requester.authenticated_entity ) - await self.store.set_profile_avatar_url(target_user, avatar_url_to_set) + stream_id = await self.store.set_profile_field( + target_user, + ProfileFields.AVATAR_URL, + avatar_url_to_set, + ) profile = await self.store.get_profileinfo(target_user) @@ -376,6 +413,8 @@ class ProfileHandler: if propagate: await self._update_join_states(requester, target_user) + return stream_id + async def delete_profile_upon_deactivation( self, target_user: UserID, @@ -394,6 +433,9 @@ class ProfileHandler: **leave** the room on the user's behalf, so there's no point sending new join events into rooms to propagate the profile deletion. See the `users_pending_deactivation` table and the associated user parter loop. + + Profile update streams are NOT updated in any way; this happens when the + event persister processes the room leave events triggered elsewhere as above. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -406,7 +448,10 @@ class ProfileHandler: # have it. raise AuthError(400, "Cannot remove another user's profile") - await self.store.delete_profile(target_user) + # Record the profile delete + await self.store.delete_profile( + user_id=target_user, + ) await self._third_party_rules.on_profile_update( target_user.to_string(), @@ -415,6 +460,50 @@ class ProfileHandler: deactivation=True, ) + async def dispatch_set_profile_field( + self, + *, + target_user: UserID, + requester: Requester, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + by_admin: bool = False, + propagate: bool = True, + ) -> None: + """ + Dispatch setting a profile field value. This either happens in the same + instance, if configured for profile updates, or via replication in the + right instance. + + Args: + target_user: the user whose profile field is to be changed. + requester: The user attempting to make this change. + field_name: The field name to update. + new_value: New value for the profile field. + by_admin: Whether this change was made by an administrator. + propagate: Whether this change also applies to the user's membership events. + """ + if self._is_events_writer: + await self.set_field( + target_user=target_user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=by_admin, + propagate=propagate, + ) + else: + # Offload to the right worker via http replication + await self._set_profile_field_client( + instance_name=self._profile_updates_writer_instance, + user_id=target_user.to_string(), + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=by_admin, + propagate=propagate, + ) + @cached() async def check_avatar_size_and_mime_type(self, mxc: str) -> bool: """Check that the size and content type of the avatar at the given MXC URI are @@ -492,7 +581,7 @@ class ProfileHandler: async def get_profile_field( self, target_user: UserID, field_name: str - ) -> JsonValue: + ) -> JsonValue | dict[str, JsonValue]: """ Fetch a user's profile from the database for local users and over federation for remote users. @@ -530,15 +619,68 @@ class ProfileHandler: return result.get(field_name) - async def set_profile_field( + async def set_field( self, + *, target_user: UserID, requester: Requester, field_name: str, - new_value: JsonValue, - *, + new_value: JsonValue | dict[str, JsonValue], by_admin: bool = False, + propagate: bool = True, ) -> None: + """Wrapper function for setting any profile field for a user.""" + if field_name == ProfileFields.DISPLAYNAME: + if not isinstance(new_value, str): + raise SynapseError( + 400, "'displayname' must be a string", errcode=Codes.INVALID_PARAM + ) + stream_id = await self.set_displayname( + target_user=target_user, + requester=requester, + new_displayname=new_value, + by_admin=by_admin, + propagate=propagate, + ) + elif field_name == ProfileFields.AVATAR_URL: + if not isinstance(new_value, str): + raise SynapseError( + 400, "'avatar_url' must be a string", errcode=Codes.INVALID_PARAM + ) + stream_id = await self.set_avatar_url( + target_user=target_user, + requester=requester, + new_avatar_url=new_value, + by_admin=by_admin, + propagate=propagate, + ) + else: + stream_id = await self.set_profile_field( + target_user=target_user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=by_admin, + ) + + if stream_id is not None: + room_ids = await self.store.get_rooms_for_user(target_user.to_string()) + if room_ids: + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + rooms=room_ids, + ) + + async def set_profile_field( + self, + *, + target_user: UserID, + requester: Requester, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + by_admin: bool = False, + ) -> int | None: """Set a new profile field for a user. Preconditions: @@ -546,6 +688,8 @@ class ProfileHandler: notify modules about the change whilst claiming it is not related to user deactivation. + FIXME: This precondition seems to lack a test. + Args: target_user: the user whose profile is to be changed. requester: The user attempting to make this change. @@ -559,7 +703,11 @@ class ProfileHandler: if not by_admin and target_user != requester.user: raise AuthError(403, "Cannot set another user's profile") - await self.store.set_profile_field(target_user, field_name, new_value) + stream_id = await self.store.set_profile_field( + target_user, + field_name, + new_value, + ) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) @@ -567,6 +715,48 @@ class ProfileHandler: target_user.to_string(), profile, by_admin, deactivation=False ) + return stream_id + + async def dispatch_delete_profile_field( + self, + *, + target_user: UserID, + requester: Requester, + field_name: str, + by_admin: bool = False, + ) -> None: + """ + Dispatch deleting a profile field value. This either happens in the same + instance, if configured for profile updates, or via replication in the + right instance. + + To delete a displayname / avatar_uri, use the `dispatch_set_profile_field` + method, using an empty string as the value. + + Args: + target_user: the user whose profile field is to be changed. + requester: The user attempting to make this change. + field_name: The field name to update. + by_admin: Whether this change was made by an administrator. + """ + assert field_name not in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL) + if self._is_events_writer: + await self.delete_profile_field( + target_user=target_user, + requester=requester, + field_name=field_name, + by_admin=by_admin, + ) + else: + # Offload to the right worker via http replication + await self._delete_profile_field_client( + instance_name=self._profile_updates_writer_instance, + user_id=target_user.to_string(), + requester=requester, + field_name=field_name, + by_admin=by_admin, + ) + async def delete_profile_field( self, target_user: UserID, @@ -582,6 +772,8 @@ class ProfileHandler: notify modules about the change whilst claiming it is not related to user deactivation. + FIXME: This precondition seems to lack a test. + Args: target_user: the user whose profile is to be changed. requester: The user attempting to make this change. @@ -594,7 +786,10 @@ class ProfileHandler: if not by_admin and target_user != requester.user: raise AuthError(400, "Cannot set another user's profile") - await self.store.delete_profile_field(target_user, field_name) + stream_id = await self.store.delete_profile_field( + target_user, + field_name, + ) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) @@ -602,6 +797,15 @@ class ProfileHandler: target_user.to_string(), profile, by_admin, deactivation=False ) + if stream_id: + room_ids = await self.store.get_rooms_for_user(target_user.to_string()) + if room_ids: + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + rooms=room_ids, + ) + async def on_profile_query(self, args: JsonDict) -> JsonDict: """Handles federation profile query requests.""" diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 5152d0b522..4f3a63f8bc 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1536,7 +1536,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): prev_member_event_id = prev_state_ids.get( (EventTypes.Member, event.state_key), None ) - if prev_member_event_id: prev_member_event = await self.store.get_event(prev_member_event_id) if prev_member_event.membership == Membership.JOIN: diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index bb5ca329e0..f9d9475711 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -530,10 +530,11 @@ class SsoHandler: user_id, authenticated_entity=user_id, ) - await self._profile_handler.set_displayname( - user_id_obj, - requester, - attributes.display_name, + await self._profile_handler.dispatch_set_profile_field( + target_user=user_id_obj, + requester=requester, + field_name=ProfileFields.DISPLAYNAME, + new_value=attributes.display_name, by_admin=True, ) if attributes.picture: @@ -842,10 +843,11 @@ class SsoHandler: ) # save it as user avatar - await self._profile_handler.set_avatar_url( - uid, - create_requester(uid), - str(avatar_mxc_url), + await self._profile_handler.dispatch_set_profile_field( + target_user=uid, + requester=create_requester(uid), + field_name=ProfileFields.AVATAR_URL, + new_value=str(avatar_mxc_url), ) logger.info("successfully saved the user avatar") diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index a05d6c6e59..943105415a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -18,8 +18,11 @@ # [This file includes modifications made by New Vector Limited] # # +import hashlib import itertools +import json import logging +import os from typing import ( TYPE_CHECKING, AbstractSet, @@ -37,6 +40,7 @@ from synapse.api.constants import ( EventContentFields, EventTypes, Membership, + ProfileUpdateAction, StickyEvent, ) from synapse.api.filtering import FilterCollection @@ -64,6 +68,7 @@ from synapse.types import ( DeviceListUpdates, JsonDict, JsonMapping, + JsonValue, MultiWriterStreamToken, MutableStateMap, Requester, @@ -104,10 +109,25 @@ non_empty_sync_counter = Counter( # client for no more than 30 minutes. LAZY_LOADED_MEMBERS_CACHE_MAX_AGE = 30 * 60 * 1000 +# Store the cache that tracks which lazy-loaded profile fields have been sent to a given +# client for no more than 30 minutes. +LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE = 30 * 60 * 1000 + # Remember the last 100 members we sent to a client for the purposes of # avoiding redundantly sending the same lazy-loaded members to the client LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE = 100 +# Remember the last 100 profile field updates we sent to a client for the purposes of +# avoiding redundantly sending the same lazy-loaded full profiles to the client +LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_SIZE = 100 + +# The digest size for the lazy loaded profile fields cache. +LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_SIZE = 16 + +# A random key generated on server startup, for the lazy loaded profile fields cache. +# Since this is a per-process cache, we don't care if the key is different per process. +LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_KEY = os.urandom(32) + SyncRequestKey = tuple[Any, ...] @@ -224,6 +244,7 @@ class SyncResult: next_batch: Token for the next sync presence: List of presence events for the user. account_data: List of account_data events for the user. + profile_updates: Map of user_id to profile field updates for that user. joined: JoinedSyncResult for each joined room. invited: InvitedSyncResult for each invited room. knocked: KnockedSyncResult for each knocked on room. @@ -239,6 +260,8 @@ class SyncResult: next_batch: StreamToken presence: list[UserPresenceState] account_data: list[JsonDict] + # user ID -> {profile field -> value | null if unset } + profile_updates: dict[str, dict[str, JsonValue | dict[str, JsonValue]] | None] joined: list[JoinedSyncResult] invited: list[InvitedSyncResult] knocked: list[KnockedSyncResult] @@ -260,6 +283,7 @@ class SyncResult: or self.knocked or self.archived or self.account_data + or self.profile_updates or self.to_device or self.device_lists ) @@ -275,6 +299,7 @@ class SyncResult: next_batch=next_batch, presence=[], account_data=[], + profile_updates={}, joined=[], invited=[], knocked=[], @@ -291,6 +316,7 @@ class SyncHandler: self.server_name = hs.hostname self.hs_config = hs.config self.store = hs.get_datastores().main + self._is_mine_id = hs.is_mine_id self.notifier = hs.get_notifier() self.presence_handler = hs.get_presence_handler() self._relations_handler = hs.get_relations_handler() @@ -329,6 +355,29 @@ class SyncHandler: max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE, ) + # ExpiringCache((User, Device)) + # -> LruCache( + # blake2b(Other User ID + Field Name) -> blake2b(Field value) + # ) + self.lazy_loaded_profile_fields_cache: ExpiringCache[ + tuple[str, str | None], LruCache[bytes, bytes] + ] = ExpiringCache( + cache_name="lazy_loaded_profile_fields_cache", + server_name=self.server_name, + hs=hs, + clock=self.clock, + max_len=0, + expiry_ms=LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE, + ) + """This cache contains fields and values we have sent to clients as profile + updates, for a particular user + device combo. The cache entry is a blake2b hash + of the user + field name, with the value being a blake2b hash of the field value. + If the field value changes for a particular user, the hash will change + and the cache will be missed. + + We don't manually remove entries from this cache, though it may be ignored + in cases where the sync must send the field down to the client. + """ self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync @@ -1023,6 +1072,8 @@ class SyncHandler: def get_lazy_loaded_members_cache( self, cache_key: tuple[str, str | None] ) -> LruCache[str, str]: + # FIXME: This cache may be subject to losing members in the case that + # a sync is interrupted and retried, see https://github.com/element-hq/synapse/issues/19978 cache: LruCache[str, str] | None = self.lazy_loaded_members_cache.get(cache_key) if cache is None: logger.debug("creating LruCache for %r", cache_key) @@ -1036,6 +1087,35 @@ class SyncHandler: logger.debug("found LruCache for %r", cache_key) return cache + def get_lazy_loaded_profile_fields_cache( + self, cache_key: tuple[str, str | None] + ) -> LruCache[bytes, bytes]: + """This cache contains fields and values we have sent to clients as profile + updates, for a particular user + device combo. The cache entry is a blake2b hash + of the user + field name, with the value being a blake2b hash of the field value. + If the field value changes for a particular user, the hash will change + and the cache will be missed. + + We don't manually remove entries from this cache, though it may be ignored + in cases where the sync must send the field down to the client. + """ + # FIXME: This cache may be subject to losing field updates in the case that + # a sync is interrupted and retried, see https://github.com/element-hq/synapse/issues/19978 + cache: LruCache[bytes, bytes] | None = ( + self.lazy_loaded_profile_fields_cache.get(cache_key) + ) + if cache is None: + logger.debug("creating LruCache for %r", cache_key) + cache = LruCache( + max_size=LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_SIZE, + clock=self.clock, + server_name=self.server_name, + ) + self.lazy_loaded_profile_fields_cache[cache_key] = cache + else: + logger.debug("found LruCache for %r", cache_key) + return cache + async def compute_state_delta( self, room_id: str, @@ -1863,10 +1943,18 @@ class SyncHandler: } ) + # Note, this needs to be after we collect `joined`, `invited`, `knocked` and + # `archived` sync results since we want to utilize the work we did to collect + # events in those responses as a basis for which users to include profiles + # for when lazy loading. + if self.hs_config.server.include_profile_updates_in_sync: + await self._generate_sync_entry_for_profile_updates(sync_result_builder) + logger.debug("Sync response calculation complete") return SyncResult( presence=sync_result_builder.presence, account_data=sync_result_builder.account_data, + profile_updates=sync_result_builder.profile_updates, joined=sync_result_builder.joined, invited=sync_result_builder.invited, knocked=sync_result_builder.knocked, @@ -2131,6 +2219,272 @@ class SyncHandler: sync_result_builder.account_data = account_data_for_user + async def _generate_initial_sync_entry_for_profile_updates( + self, + *, + user_id: str, + sync_result_builder: "SyncResultBuilder", + profile_fields: set[str], + include_users: set[str] | None, + ) -> None: + """ + Build an initial sync entry for profile updates and attach it to the + given `sync_result_builder`. + + Note: Currently, only profile updates of local users are generated. + + Args: + user_id: The Matrix ID of the user to generate the sync entry for. + sync_result_builder: + profile_fields: The list of field IDs to filter for. + include_users: List of users profiles to include in the sync response, + for when we have calculated a list of users in our lazy loading + sync and want to only return those. + """ + # Currently, limited to only local profiles, so filter remote servers out + user_ids = await self.store.get_local_users_who_share_room_with_user(user_id) + # Ensure we're in the list even if we don't belong to any rooms + user_ids.add(user_id) + if include_users: + # Filter down to selected included users + user_ids = {user_id for user_id in user_ids if user_id in include_users} + + if not user_ids: + return + + profile_data_by_user = await self.store.get_profile_data_for_users(user_ids) + + # Serialise the profile updates into the sync response format. + profile_updates: dict[ + str, dict[str, JsonValue | dict[str, JsonValue]] | None + ] = {} + for other_user_id in user_ids: + profile_data = profile_data_by_user.get(other_user_id) + if profile_data is None: + # Don't generate anything for users with no profile data + # in initial sync. + continue + + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} + for field_name in profile_fields: + if field_name in profile_data.keys(): + per_user_updates[field_name] = profile_data[field_name] + + if per_user_updates: + profile_updates[other_user_id] = per_user_updates + + if profile_updates: + sync_result_builder.profile_updates = profile_updates + + async def _generate_sync_entry_for_profile_updates( + self, sync_result_builder: "SyncResultBuilder" + ) -> None: + """ + Build a sync entry for profile updates and attach it to the given + `sync_result_builder`. + + Currently only local profiles updates will be included in the sync response. + + Args: + sync_result_builder: + """ + sync_config = sync_result_builder.sync_config + profile_fields = sync_config.filter_collection.profile_fields + if not profile_fields: + return + + user_id = sync_config.user.to_string() + since_token = sync_result_builder.since_token + now_token = sync_result_builder.now_token + + sync_config = sync_result_builder.sync_config + lazy_load_members = sync_config.filter_collection.lazy_load_members() + include_users = None + if lazy_load_members: + # Collect members from the existing `sync_result_builder` data. + # Ensure we filter out any remove users until we support profile + # updates for federated users. + include_users = set() + # invited + for invited in sync_result_builder.invited: + if self._is_mine_id(invited.invite.sender): + include_users.add(invited.invite.sender) + # joined + for joined in sync_result_builder.joined: + for timeline_event in joined.timeline.events: + if self._is_mine_id(timeline_event.event.sender): + include_users.add(timeline_event.event.sender) + # knocked + for knocked in sync_result_builder.knocked: + if self._is_mine_id(knocked.knock.sender): + include_users.add(knocked.knock.sender) + # archived + for archived in sync_result_builder.archived: + for timeline_event in archived.timeline.events: + if self._is_mine_id(timeline_event.event.sender): + include_users.add(timeline_event.event.sender) + + if since_token is None: + await self._generate_initial_sync_entry_for_profile_updates( + user_id=user_id, + sync_result_builder=sync_result_builder, + profile_fields=profile_fields, + include_users=include_users, + ) + return + + updates = await self.store.get_profile_updates_for_user_and_fields( + from_id=since_token.profile_updates_key, + to_id=now_token.profile_updates_key, + user_id=user_id, + field_names=profile_fields, + ) + + left_room_user_ids = { + update.user_id + for update in updates + if update.action == ProfileUpdateAction.LEFT_ROOM.value + } + joined_room_user_ids = { + update.user_id + for update in updates + if update.action == ProfileUpdateAction.JOINED_ROOM.value + } + users = set() + updated_users = { + update.user_id + for update in updates + if update.action == ProfileUpdateAction.UPDATE.value + } + # Add any users in the timeline, if we collected them due to lazy loading + if include_users: + users.update(include_users) + # Add users with updates + users.update(updated_users) + # Add any newly joined users + users.update(joined_room_user_ids) + + if not users and not left_room_user_ids: + return + + # Serialise the profile updates into the sync response format. + # user ID -> {profile field -> value | null if unset } + profile_updates: dict[ + str, dict[str, JsonValue | dict[str, JsonValue]] | None + ] = {} + + # Process field updates and users who have events in the sync response + if users: + updated_user_fields: dict[str, set[str]] = {} + # Set fields from updates + for update in updates: + if ( + # Skip the update if there is no field update (a joined or left room action), + update.action != ProfileUpdateAction.UPDATE + or update.affected_fields is None + # or if the client isn't interested in any of the fields + or update.affected_fields.isdisjoint(profile_fields) + # or we're not interested in this user. + or update.user_id not in users + ): + continue + updated_user_fields.setdefault(update.user_id, set()).update( + # Add any fields that were affected and that we're interested in + update.affected_fields & profile_fields + ) + + # Note: there's a small race condition here where a profile update may + # occur between fetching `now_token` above and reaching this step. In + # that case, the profile information will be newer than `now_token`. + # This is fine, as users will generally always want the latest profile + # information. However, it does mean that on the next sync, the same + # profile update will come down a second time. + # + # Hopefully clients can just filter these out. + profile_data_by_user = await self.store.get_profile_data_for_users(users) + + # Note, we've already collected field updates above via `updates`, + # outside of events in the timeline when lazy loading. When lazy loading, + # we're already always sending the fields that have changed, regardless + # of the lazy loading cache. + for other_user_id in users: + profile_data = profile_data_by_user.get(other_user_id) + if profile_data is None: + # No profile data for this user, just return a blank dictionary + # in incremental sync, telling the clients to remove all profile + # information for this user. + profile_updates[other_user_id] = None + continue + + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} + if include_users and other_user_id in include_users: + # Include all the fields the client asked for, as this user + # has events in a lazy loaded sync response, except for + # fields we've recently sent in a previous lazy loaded sync response + fields = set(profile_data.keys()).intersection(profile_fields) + for field_name in fields: + cache_key = ( + sync_config.user.to_string(), + sync_config.device_id, + ) + cache = self.get_lazy_loaded_profile_fields_cache(cache_key) + # Only send this users field if we haven't recently sent it. + # Our cache contains previously set values as pairs of + # blake2b(other_used_id + field_name) -> blake2b(value), + # which ensures if the value changes, we'll miss the cache, + # thus sending the field update to the syncing user. + cache_value = hashlib.blake2b( + f"{other_user_id}-{field_name}".encode("utf8"), + key=LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_KEY, + digest_size=LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_SIZE, + ).digest() + value_hash = hashlib.blake2b( + json.dumps( + [ + profile_data.get(field_name), + ], + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf8"), + key=LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_KEY, + digest_size=LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_SIZE, + ).digest() + if cache.get(cache_value) != value_hash: + per_user_updates[field_name] = profile_data.get(field_name) + # Update our cache to indicate this user/field combo + # has been recently sent. + cache.set( + cache_value, + value_hash, + ) + else: + # Include only the diff, unless the user recently joined, + # then send all the fields the client asked for. + # We don't use a cache here as for non-lazy sync we always + # send changes and/or fields the client asked for, if relevant + # as above joined condition. + fields = ( + profile_fields + if other_user_id in joined_room_user_ids + else set(updated_user_fields.get(other_user_id, [])) + ) + fields = set(profile_data.keys()).intersection(fields) + for field_name in fields: + per_user_updates[field_name] = profile_data[field_name] + + if per_user_updates: + profile_updates[other_user_id] = per_user_updates + + # Process left rooms + if left_room_user_ids: + for other_user_id in left_room_user_ids: + # Return an empty dictionary to the client + profile_updates[other_user_id] = None + + if profile_updates: + sync_result_builder.profile_updates = profile_updates + async def _generate_sync_entry_for_presence( self, sync_result_builder: "SyncResultBuilder", @@ -3147,6 +3501,7 @@ class SyncResultBuilder: # The following mirror the fields in a sync response presence account_data + profile_updates joined invited knocked @@ -3165,6 +3520,9 @@ class SyncResultBuilder: presence: list[UserPresenceState] = attr.Factory(list) account_data: list[JsonDict] = attr.Factory(list) + profile_updates: dict[str, dict[str, JsonValue | dict[str, JsonValue]] | None] = ( + attr.Factory(dict) + ) joined: list[JoinedSyncResult] = attr.Factory(list) invited: list[InvitedSyncResult] = attr.Factory(list) knocked: list[KnockedSyncResult] = attr.Factory(list) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 48963b8d83..1be0e9fd91 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -2042,10 +2042,11 @@ class ModuleApi: deactivation, ) - await self._hs.get_profile_handler().set_displayname( + await self._hs.get_profile_handler().dispatch_set_profile_field( target_user=user_id, requester=requester, - new_displayname=new_displayname, + field_name=ProfileFields.DISPLAYNAME, + new_value=new_displayname, by_admin=True, ) diff --git a/synapse/notifier.py b/synapse/notifier.py index 6a057ac09f..e24d0ef5a2 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -528,6 +528,7 @@ class Notifier: StreamKeyType.UN_PARTIAL_STATED_ROOMS, StreamKeyType.THREAD_SUBSCRIPTIONS, StreamKeyType.STICKY_EVENTS, + StreamKeyType.PROFILE_UPDATES, ], new_token: int, users: Collection[str | UserID] | None = None, diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index 68cc6ce1fc..d934ef8067 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -30,6 +30,7 @@ from synapse.replication.http import ( login, membership, presence, + profile, push, register, send_events, @@ -59,6 +60,7 @@ class ReplicationRestResource(JsonResource): push.register_servlets(hs, self) state.register_servlets(hs, self) devices.register_servlets(hs, self) + profile.register_servlets(hs, self) # The following can't currently be instantiated on workers. if hs.config.worker.worker_app is None: diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py new file mode 100644 index 0000000000..dddc36477f --- /dev/null +++ b/synapse/replication/http/profile.py @@ -0,0 +1,150 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +import logging +from typing import TYPE_CHECKING + +from twisted.web.server import Request + +from synapse.http.server import HttpServer +from synapse.replication.http._base import ReplicationEndpoint +from synapse.synapse_rust.types import Requester +from synapse.types import JsonDict, JsonValue, UserID, create_requester + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ReplicationProfileSetField(ReplicationEndpoint): + """Update a profile field for a user. + + The POST looks like: + + POST /_synapse/replication/profile_set_field/ + + { + "requester": "@admin:hs", + "field_name": "displayname", + "new_value": "Alice", + "by_admin": true, + "propagate": false + } + + 200 OK + + {} + """ + + NAME = "profile_set_field" + PATH_ARGS = ("user_id",) + METHOD = "POST" + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self._profile_handler = hs.get_profile_handler() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + user_id: str, + requester: Requester, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + by_admin: bool, + propagate: bool, + ) -> JsonDict: + return { + "requester": requester.user.to_string(), + "field_name": field_name, + "new_value": new_value, + "by_admin": by_admin, + "propagate": propagate, + } + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> tuple[int, JsonDict]: + await self._profile_handler.set_field( + target_user=UserID.from_string(user_id), + requester=create_requester(content["requester"]), + field_name=content["field_name"], + new_value=content["new_value"], + by_admin=content["by_admin"], + propagate=content["propagate"], + ) + + return (200, {}) + + +class ReplicationProfileDeleteField(ReplicationEndpoint): + """Delete a profile field for a user. + + The POST looks like: + + POST /_synapse/replication/profile_delete_field/ + + { + "requester": "@admin:hs", + "field_name": "displayname", + "by_admin": true + } + + 200 OK + + {} + """ + + NAME = "profile_delete_field" + PATH_ARGS = ("user_id",) + METHOD = "POST" + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self._profile_handler = hs.get_profile_handler() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + user_id: str, + requester: Requester, + field_name: str, + by_admin: bool, + ) -> JsonDict: + return { + "requester": requester.user.to_string(), + "field_name": field_name, + "by_admin": by_admin, + } + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> tuple[int, JsonDict]: + await self._profile_handler.delete_profile_field( + target_user=UserID.from_string(user_id), + requester=create_requester(content["requester"]), + field_name=content["field_name"], + by_admin=content["by_admin"], + ) + + return (200, {}) + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + ReplicationProfileSetField(hs).register(http_server) + ReplicationProfileDeleteField(hs).register(http_server) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index bc7e46d4c9..c0896b83e7 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -44,6 +44,7 @@ from synapse.replication.tcp.streams import ( UnPartialStatedRoomStream, ) from synapse.replication.tcp.streams._base import ( + ProfileUpdatesStream, StickyEventsStream, ThreadSubscriptionsStream, ) @@ -265,6 +266,23 @@ class ReplicationDataHandler: token, users=[row.user_id for row in rows], ) + elif stream_name == ProfileUpdatesStream.NAME: + updated_user_ids = {row.user_id for row in rows} + if updated_user_ids: + room_ids: set[str] = set() + # Get all the rooms of the updated users, dict of + # User ID -> [Room ID] + users_and_rooms = await self.store.get_rooms_for_users(updated_user_ids) + # Loop through each user's room IDs and add to our set of rooms + for user_room_ids in users_and_rooms.values(): + room_ids.update(user_room_ids) + + if room_ids: + self.notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + token, + rooms=room_ids, + ) elif stream_name == StickyEventsStream.NAME: self.notifier.on_new_event( StreamKeyType.STICKY_EVENTS, diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index ad9fed72dd..1d0586abe0 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -67,6 +67,7 @@ from synapse.replication.tcp.streams import ( ) from synapse.replication.tcp.streams._base import ( DeviceListsStream, + ProfileUpdatesStream, StickyEventsStream, ThreadSubscriptionsStream, ) @@ -218,6 +219,12 @@ class ReplicationCommandHandler: continue + if isinstance(stream, ProfileUpdatesStream): + if hs.get_instance_name() in hs.config.worker.writers.events: + self._streams_to_replicate.append(stream) + + continue + if isinstance(stream, StickyEventsStream): if hs.get_instance_name() in hs.config.worker.writers.events: self._streams_to_replicate.append(stream) diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index e41573cf68..e657822da7 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -37,6 +37,7 @@ from synapse.replication.tcp.streams._base import ( DeviceListsStream, PresenceFederationStream, PresenceStream, + ProfileUpdatesStream, PushersStream, PushRulesStream, QuarantinedMediaStream, @@ -70,6 +71,7 @@ STREAMS_MAP = { ToDeviceStream, FederationStream, AccountDataStream, + ProfileUpdatesStream, StickyEventsStream, ThreadSubscriptionsStream, UnPartialStatedRoomStream, @@ -94,6 +96,7 @@ __all__ = [ "ToDeviceStream", "FederationStream", "AccountDataStream", + "ProfileUpdatesStream", "StickyEventsStream", "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index a73f767add..3d7b0ec3d1 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -31,8 +31,9 @@ from typing import ( import attr -from synapse.api.constants import AccountDataTypes +from synapse.api.constants import AccountDataTypes, ProfileUpdateAction from synapse.replication.http.streams import ReplicationGetStreamUpdates +from synapse.types import UserID if TYPE_CHECKING: from synapse.server import HomeServer @@ -765,6 +766,80 @@ class ThreadSubscriptionsStream(_StreamFromIdGen): return rows, rows[-1][0], len(updates) == limit +def _convert_affected_fields( + wire: list[str] | frozenset[str] | None, +) -> frozenset[str] | None: + return ( + frozenset(wire) + if wire is not None and not isinstance(wire, frozenset) + else None + ) + + +@attr.s(slots=True, auto_attribs=True) +class ProfileUpdatesStreamRow: + """Profile update stream row detailing what the profile update changes.""" + + user_id: UserID + """The full user ID with the profile update.""" + action: ProfileUpdateAction + """The action, either 'update' for a field update, 'left_room' if the user left + a room or `joined_room` if the user joined a room, see ProfileUpdateAction enum. + """ + affected_fields: frozenset[str] | None = attr.ib( + # Convert list back to frozenset from wire format + converter=_convert_affected_fields + ) + """Names of the profile fields that were added, updated or removed, see https://spec.matrix.org/unstable/client-server-api/#profiles. + This is None if `action` is not `update`. + """ + + +class ProfileUpdatesStream(_StreamFromIdGen): + """Stream to inform users about profile updates.""" + + # FIXME: See issue https://github.com/element-hq/synapse/issues/19981 + # for concerns around the current implementation of the profile + # updates stream. + + NAME = "profile_updates" + ROW_TYPE = ProfileUpdatesStreamRow + + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + super().__init__( + hs.get_instance_name(), + self._update_function, + self.store._profile_updates_id_gen, + ) + + async def _update_function( + self, instance_name: str, from_token: int, to_token: int, limit: int + ) -> StreamUpdateResult: + updates = await self.store.get_updated_profile_updates( + from_id=from_token, to_id=to_token, limit=limit + ) + rows = [ + ( + stream_id, + # These are the args to `ProfileUpdatesStreamRow` + ( + user_id, + action, + # Must convert `field_names` to a list for transport over the wire + # It will be reconstructed as a frozenset on the other end + list(field_names) if field_names is not None else None, + ), + ) + for stream_id, user_id, action, field_names in updates + ] + + if not rows: + return [], to_token, False + + return rows, rows[-1][0], len(updates) == limit + + @attr.s(slots=True, auto_attribs=True) class StickyEventsStreamRow: """Stream to inform workers about changes to sticky events.""" diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 53d1f2d366..ddb4483789 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -28,7 +28,7 @@ from typing import TYPE_CHECKING import attr from pydantic import StrictBool, StrictInt, StrictStr -from synapse.api.constants import Direction +from synapse.api.constants import Direction, ProfileFields from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.http.servlet import ( RestServlet, @@ -366,8 +366,12 @@ class UserRestServletV2(UserRestServletV2Get): if user: # modify user if "displayname" in body: - await self.profile_handler.set_displayname( - target_user, requester, body["displayname"], by_admin=True + await self.profile_handler.dispatch_set_profile_field( + target_user=target_user, + requester=requester, + field_name=ProfileFields.DISPLAYNAME, + new_value=body["displayname"], + by_admin=True, ) if threepids is not None: @@ -415,8 +419,12 @@ class UserRestServletV2(UserRestServletV2Get): ) if "avatar_url" in body: - await self.profile_handler.set_avatar_url( - target_user, requester, body["avatar_url"], by_admin=True + await self.profile_handler.dispatch_set_profile_field( + target_user=target_user, + requester=requester, + field_name=ProfileFields.AVATAR_URL, + new_value=body["avatar_url"], + by_admin=True, ) if "admin" in body: @@ -523,8 +531,12 @@ class UserRestServletV2(UserRestServletV2Get): ) if "avatar_url" in body and isinstance(body["avatar_url"], str): - await self.profile_handler.set_avatar_url( - target_user, requester, body["avatar_url"], by_admin=True + await self.profile_handler.dispatch_set_profile_field( + target_user=target_user, + requester=requester, + field_name=ProfileFields.AVATAR_URL, + new_value=body["avatar_url"], + by_admin=True, ) user_info_dict = await self.admin_handler.get_user(target_user) diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 360a83f39c..4431aa2b2c 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -146,7 +146,9 @@ class ProfileFieldRestServlet(RestServlet): await self.profile_handler.check_profile_query_allowed(user, requester_user) if field_name == ProfileFields.DISPLAYNAME: - field_value: JsonValue = await self.profile_handler.get_displayname(user) + field_value: ( + JsonValue | dict[str, JsonValue] + ) = await self.profile_handler.get_displayname(user) elif field_name == ProfileFields.AVATAR_URL: field_value = await self.profile_handler.get_avatar_url(user) else: @@ -204,18 +206,14 @@ class ProfileFieldRestServlet(RestServlet): Codes.USER_ACCOUNT_SUSPENDED, ) - if field_name == ProfileFields.DISPLAYNAME: - await self.profile_handler.set_displayname( - user, requester, new_value, by_admin=is_admin, propagate=propagate - ) - elif field_name == ProfileFields.AVATAR_URL: - await self.profile_handler.set_avatar_url( - user, requester, new_value, by_admin=is_admin, propagate=propagate - ) - else: - await self.profile_handler.set_profile_field( - user, requester, field_name, new_value, by_admin=is_admin - ) + await self.profile_handler.dispatch_set_profile_field( + target_user=user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=is_admin, + propagate=propagate, + ) return 200, {} @@ -261,17 +259,21 @@ class ProfileFieldRestServlet(RestServlet): Codes.USER_ACCOUNT_SUSPENDED, ) - if field_name == ProfileFields.DISPLAYNAME: - await self.profile_handler.set_displayname( - user, requester, "", by_admin=is_admin, propagate=propagate - ) - elif field_name == ProfileFields.AVATAR_URL: - await self.profile_handler.set_avatar_url( - user, requester, "", by_admin=is_admin, propagate=propagate + if field_name in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL): + await self.profile_handler.dispatch_set_profile_field( + target_user=user, + requester=requester, + field_name=field_name, + new_value="", + by_admin=is_admin, + propagate=propagate, ) else: - await self.profile_handler.delete_profile_field( - user, requester, field_name, by_admin=is_admin + await self.profile_handler.dispatch_delete_profile_field( + target_user=user, + requester=requester, + field_name=field_name, + by_admin=is_admin, ) return 200, {} @@ -284,8 +286,9 @@ class UnstableProfileFieldRestServlet(ProfileFieldRestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - # The specific field endpoint *must* appear before the generic profile endpoint. ProfileFieldRestServlet(hs).register(http_server) - ProfileRestServlet(hs).register(http_server) + if hs.config.experimental.msc4133_enabled: UnstableProfileFieldRestServlet(hs).register(http_server) + + ProfileRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 962317dedb..4e437c5e98 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -123,6 +123,7 @@ class SyncRestServlet(RestServlet): self._event_serializer = hs.get_event_client_serializer() self._msc2654_enabled = hs.config.experimental.msc2654_enabled self._msc3773_enabled = hs.config.experimental.msc3773_enabled + self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, @@ -351,6 +352,15 @@ class SyncRestServlet(RestServlet): if sync_result.to_device: response["to_device"] = {"events": sync_result.to_device} + if self._msc4429_enabled and sync_result.profile_updates: + # FIXME: See issue https://github.com/element-hq/synapse/issues/19981 + # for concerns around the current implementation of the profile + # updates stream. + response["org.matrix.msc4429.users"] = { + user_id: {"profile_updates": updates} + for user_id, updates in sync_result.profile_updates.items() + } + if sync_result.device_lists.changed: response["device_lists"]["changed"] = list(sync_result.device_lists.changed) if sync_result.device_lists.left: diff --git a/synapse/rest/synapse/mas/users.py b/synapse/rest/synapse/mas/users.py index 01db41bcfa..cc5717b103 100644 --- a/synapse/rest/synapse/mas/users.py +++ b/synapse/rest/synapse/mas/users.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any, TypedDict from pydantic import StrictBool, StrictStr, model_validator +from synapse.api.constants import ProfileFields from synapse.api.errors import NotFoundError, SynapseError from synapse.http.servlet import ( parse_and_validate_json_object_from_request, @@ -162,18 +163,18 @@ class MasProvisionUserResource(MasBaseResource): ) else: created = False + new_displayname = None if body.unset_displayname: - await self.profile_handler.set_displayname( - target_user=user_id, - requester=requester, - new_displayname="", - by_admin=True, - ) + new_displayname = "" elif body.set_displayname is not None: - await self.profile_handler.set_displayname( + new_displayname = body.set_displayname + + if new_displayname is not None: + await self.profile_handler.dispatch_set_profile_field( target_user=user_id, requester=requester, - new_displayname=body.set_displayname, + field_name=ProfileFields.DISPLAYNAME, + new_value=new_displayname, by_admin=True, ) @@ -221,18 +222,18 @@ class MasProvisionUserResource(MasBaseResource): if body.locked is not None: await self.store.set_user_locked_status(user_id.to_string(), body.locked) + new_avatar_url_value = None if body.unset_avatar_url: - await self.profile_handler.set_avatar_url( - target_user=user_id, - requester=requester, - new_avatar_url="", - by_admin=True, - ) + new_avatar_url_value = "" elif body.set_avatar_url is not None: - await self.profile_handler.set_avatar_url( + new_avatar_url_value = body.set_avatar_url + + if new_avatar_url_value is not None: + await self.profile_handler.dispatch_set_profile_field( target_user=user_id, requester=requester, - new_avatar_url=body.set_avatar_url, + field_name=ProfileFields.AVATAR_URL, + new_value=new_avatar_url_value, by_admin=True, ) @@ -380,10 +381,11 @@ class MasSetDisplayNameResource(MasBaseResource): requester = create_requester(user_id=user_id) - await self.profile_handler.set_displayname( + await self.profile_handler.dispatch_set_profile_field( target_user=requester.user, requester=requester, - new_displayname=body.displayname, + field_name=ProfileFields.DISPLAYNAME, + new_value=body.displayname, by_admin=True, ) @@ -424,10 +426,11 @@ class MasUnsetDisplayNameResource(MasBaseResource): requester = create_requester(user_id=user_id) - await self.profile_handler.set_displayname( + await self.profile_handler.dispatch_set_profile_field( target_user=requester.user, requester=requester, - new_displayname="", + field_name=ProfileFields.DISPLAYNAME, + new_value="", by_admin=True, ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8eeea20967..1df7f70b71 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -218,12 +218,18 @@ class SQLBaseStore(metaclass=ABCMeta): self.external_cached_functions[cache_name] = func -def db_to_json(db_content: memoryview | bytes | bytearray | str) -> Any: +def db_to_json( + db_content: memoryview | bytes | bytearray | str | dict[str, Any] | list[Any], +) -> Any: """ Take some data from a database row and return a JSON-decoded object. Args: db_content: The JSON-encoded contents from the database. + Supports TEXT columns, as well as JSON/JSONB columns containing lists or objects. + Note that psycopg will decode JSON/JSONB automatically but SQLite doesn't have + such a data type (and returns the text verbatim), so this function can help + paper over the difference. Returns: The object decoded from JSON. @@ -238,6 +244,13 @@ def db_to_json(db_content: memoryview | bytes | bytearray | str) -> Any: if isinstance(db_content, (bytes, bytearray)): db_content = db_content.decode("utf8") + if isinstance(db_content, (dict, list)): + # psycopg2 has already decoded this JSON or JSONB value + # Maybe we should be splitting this case out to a separate helper where + # we expect JSON/JSONB columns and switch behaviour based on + # the database driver + return db_content + try: return json_decoder.decode(db_content) except Exception: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index d92bbeeae3..8211612e2d 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -42,6 +42,7 @@ from synapse.api.constants import ( EventContentFields, EventTypes, Membership, + ProfileUpdateAction, RelationTypes, ) from synapse.api.errors import PartialStateConflictError @@ -76,6 +77,7 @@ from synapse.types import ( MutableStateMap, StateMap, StrCollection, + UserID, ) from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.state import StateFilter @@ -267,6 +269,7 @@ class PersistEventsStore: self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() self._msc4354_enabled = hs.config.experimental.msc4354_enabled + self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync self._ephemeral_messages_enabled = hs.config.server.enable_ephemeral_messages self.is_mine_id = hs.is_mine_id @@ -2118,6 +2121,129 @@ class PersistEventsStore: txn, {m for m in members_to_cache_bust if not self.hs.is_mine_id(m)} ) + if self._msc4429_enabled: + # Handle changes to the profile updates stream. + # We've already done a bunch of work calculating the changes needed + # for the sliding sync tables, so we may as well re-use that information + # here to avoid parsing the state delta again, and handling various + # edge cases. + # FIXME: See issue https://github.com/element-hq/synapse/issues/19981 + # for concerns around the current implementation of the profile + # updates stream. + profile_update_additions = { + c.user_id + for c in sliding_sync_table_changes.to_insert_membership_snapshots + if self.hs.is_mine_id(c.user_id) + # FIXME: Ideally we would filter out JOIN -> JOIN. See note below. + and c.membership == Membership.JOIN + } + profile_update_leaves = { + c.user_id + for c in sliding_sync_table_changes.to_insert_membership_snapshots + if self.hs.is_mine_id(c.user_id) + # Any transition from JOIN to something else counts as a leave here. + # Even the 'invalid' transitions might effectively happen due to + # state resolution. + and c.membership != Membership.JOIN + } | ( + # We also need to consider users that get fully state reset out of the room. + # These should be treated as 'leave' + set(sliding_sync_table_changes.to_delete_membership_snapshots) + ) + + if profile_update_additions: + # Write the profile updates for additions to the room, from either + # a join, knock, invite, etc. + # FIXME this will add rows also when a display name changes due to + # the facts that `sliding_sync_table_changes` contains a JOIN + # membership event in that case. We should aim to filter these + # unnecessary rows out, as we're also generating an UPDATE profile + # update action row for the actual display name change itself. + # See https://github.com/element-hq/synapse/issues/19981 + self.store.record_profile_updates_for_user_joined_room_txn( + txn=txn, + room_id=room_id, + joined_users=profile_update_additions, + ) + if profile_update_leaves: + # Write the profile updates for LEAVE events + for user_id in profile_update_leaves: + self._record_profile_updates_for_user_left_room_txn( + txn=txn, + user_id=UserID.from_string(user_id), + room_id=room_id, + ) + + def _record_profile_updates_for_user_left_room_txn( + self, + txn: LoggingTransaction, + user_id: UserID, + room_id: str, + ) -> None: + """ + Record updates into the profile updates stream for when a user leaves a room. + + If this was the last shared room with a set of users, clear all old rows from + the `profile_updates_per_user` table relating to those users, to avoid exposing + any profile field changes past the point of not being in any common rooms with + the user. + + Currently, updates are only recorded for local users. + + Note, this method lives here in the events store file due to the profile + store not having access to the membership store (which the events store does), + which we need to re-use the `do_users_share_a_room_txn` method there. + + Args: + user_id: The user who left the room. + room_id: The room that was left. + """ + # Get the local members of the room + room_members = self.db_pool.simple_select_onecol_txn( + txn=txn, + table="local_current_membership", + retcol="user_id", + keyvalues={ + "membership": Membership.JOIN, + "room_id": room_id, + }, + ) + # For each user check if we still share rooms + users_sharing_rooms = self.store.do_users_share_a_room_txn( + txn=txn, + user_id=user_id.to_string(), + other_user_ids=set(room_members), + ) + users_no_longer_sharing_rooms = set(room_members) - set( + users_sharing_rooms.keys() + ) + + # First clear the previous rows from the table + user_clause, user_args = make_in_list_sql_clause( + txn.database_engine, + "user_id", + users_no_longer_sharing_rooms, + ) + txn.execute( + f""" + DELETE FROM profile_updates_per_user + WHERE {user_clause} + AND stream_id IN ( + SELECT stream_id FROM profile_updates WHERE user_id = ? + ) + """, + (*user_args, user_id.to_string()), + ) + + # Now record the "left room" action in the stream + self.store.record_profile_updates_txn( + txn=txn, + user_id=user_id, + action=ProfileUpdateAction.LEFT_ROOM, + field_names=[], + target_users=users_no_longer_sharing_rooms, + ) + @classmethod def _get_relevant_sliding_sync_current_state_event_ids_txn( cls, txn: LoggingTransaction, room_id: str diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 68548434a9..05faad5b26 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -19,13 +19,21 @@ # # import json -from typing import TYPE_CHECKING, cast +from collections.abc import Set +from typing import TYPE_CHECKING, Collection, cast +import attr from canonicaljson import encode_canonical_json -from synapse.api.constants import ProfileFields +from synapse.api.constants import ( + EventTypes, + Membership, + ProfileFields, + ProfileUpdateAction, +) from synapse.api.errors import Codes, StoreError -from synapse.storage._base import SQLBaseStore +from synapse.replication.tcp.streams._base import ProfileUpdatesStream +from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, @@ -33,7 +41,9 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.roommember import ProfileInfo from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, JsonValue, UserID +from synapse.util.json import json_encoder if TYPE_CHECKING: from synapse.server import HomeServer @@ -43,6 +53,16 @@ if TYPE_CHECKING: MAX_PROFILE_SIZE = 65536 +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ProfileUpdate: + """An update to a user's profile.""" + + stream_id: int + user_id: str + action: str + affected_fields: frozenset[str] | None + + class ProfileWorkerStore(SQLBaseStore): def __init__( self, @@ -52,6 +72,7 @@ class ProfileWorkerStore(SQLBaseStore): ): super().__init__(database, db_conn, hs) self.server_name: str = hs.hostname + self._instance_name: str = hs.get_instance_name() self.database_engine = database.engine self.db_pool.updates.register_background_index_update( "profiles_full_user_id_key_idx", @@ -65,6 +86,22 @@ class ProfileWorkerStore(SQLBaseStore): "populate_full_user_id_profiles", self.populate_full_user_id_profiles ) + self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._is_events_writer = self._instance_name in hs.config.worker.writers.events + self._profile_updates_id_gen: MultiWriterIdGenerator = MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name="profile_updates", + server_name=self.server_name, + instance_name=self._instance_name, + tables=[ + ("profile_updates", "instance_name", "stream_id"), + ], + sequence_name="profile_updates_sequence", + writers=hs.config.worker.writers.events, + ) + async def populate_full_user_id_profiles( self, progress: JsonDict, batch_size: int ) -> int: @@ -76,13 +113,15 @@ class ProfileWorkerStore(SQLBaseStore): lower_bound_id = progress.get("lower_bound_id", "") def _get_last_id(txn: LoggingTransaction) -> str | None: - sql = """ - SELECT user_id FROM profiles - WHERE user_id > ? - ORDER BY user_id - LIMIT 1 OFFSET 1000 - """ - txn.execute(sql, (lower_bound_id,)) + txn.execute( + """ + SELECT user_id FROM profiles + WHERE user_id > ? + ORDER BY user_id + LIMIT 1 OFFSET 1000 + """, + (lower_bound_id,), + ) res = txn.fetchone() if res: upper_bound_id = res[0] @@ -93,21 +132,22 @@ class ProfileWorkerStore(SQLBaseStore): def _process_batch( txn: LoggingTransaction, lower_bound_id: str, upper_bound_id: str ) -> None: - sql = """ - UPDATE profiles - SET full_user_id = '@' || user_id || ? - WHERE ? < user_id AND user_id <= ? AND full_user_id IS NULL - """ - txn.execute(sql, (f":{self.server_name}", lower_bound_id, upper_bound_id)) + txn.execute( + """ + UPDATE profiles + SET full_user_id = '@' || user_id || ? + WHERE ? < user_id AND user_id <= ? AND full_user_id IS NULL + """, + (f":{self.server_name}", lower_bound_id, upper_bound_id), + ) def _final_batch(txn: LoggingTransaction, lower_bound_id: str) -> None: - sql = """ - UPDATE profiles - SET full_user_id = '@' || user_id || ? - WHERE ? < user_id AND full_user_id IS NULL - """ txn.execute( - sql, + """ + UPDATE profiles + SET full_user_id = '@' || user_id || ? + WHERE ? < user_id AND full_user_id IS NULL + """, ( f":{self.server_name}", lower_bound_id, @@ -115,10 +155,11 @@ class ProfileWorkerStore(SQLBaseStore): ) if isinstance(self.database_engine, PostgresEngine): - sql = """ - ALTER TABLE profiles VALIDATE CONSTRAINT full_user_id_not_null - """ - txn.execute(sql) + txn.execute( + """ + ALTER TABLE profiles VALIDATE CONSTRAINT full_user_id_not_null + """, + ) upper_bound_id = await self.db_pool.runInteraction( "populate_full_user_id_profiles", _get_last_id @@ -152,6 +193,13 @@ class ProfileWorkerStore(SQLBaseStore): return 50 + def process_replication_position( + self, stream_name: str, instance_name: str, token: int + ) -> None: + if stream_name == ProfileUpdatesStream.NAME: + self._profile_updates_id_gen.advance(instance_name, token) + super().process_replication_position(stream_name, instance_name, token) + async def get_profileinfo(self, user_id: UserID) -> ProfileInfo: """ Fetch the display name and avatar URL of a user. @@ -210,7 +258,9 @@ class ProfileWorkerStore(SQLBaseStore): desc="get_profile_avatar_url", ) - async def get_profile_field(self, user_id: UserID, field_name: str) -> JsonValue: + async def get_profile_field( + self, user_id: UserID, field_name: str + ) -> JsonValue | dict[str, JsonValue]: """ Get a custom profile field for a user. @@ -222,42 +272,46 @@ class ProfileWorkerStore(SQLBaseStore): The string value if the field exists, otherwise raises 404. """ - def get_profile_field(txn: LoggingTransaction) -> JsonValue: + def get_profile_field( + txn: LoggingTransaction, + ) -> JsonValue | dict[str, JsonValue]: # This will error if field_name has double quotes in it, but that's not # possible due to the grammar. field_path = f'$."{field_name}"' if isinstance(self.database_engine, PostgresEngine): - sql = """ - SELECT JSONB_PATH_EXISTS(fields, ?), JSONB_EXTRACT_PATH(fields, ?) - FROM profiles - WHERE user_id = ? - """ txn.execute( - sql, + """ + SELECT JSONB_PATH_EXISTS(fields, ?), JSONB_EXTRACT_PATH(fields, ?) + FROM profiles + WHERE user_id = ? + """, (field_path, field_name, user_id.localpart), ) # Test exists first since value being None is used for both # missing and a null JSON value. - exists, value = cast(tuple[bool, JsonValue], txn.fetchone()) + exists, value = cast( + tuple[bool, JsonValue | dict[str, JsonValue]], txn.fetchone() + ) if not exists: raise StoreError(404, "No row found") return value else: - sql = """ - SELECT JSON_TYPE(fields, ?), JSON_EXTRACT(fields, ?) - FROM profiles - WHERE user_id = ? - """ txn.execute( - sql, + """ + SELECT JSON_TYPE(fields, ?), JSON_EXTRACT(fields, ?) + FROM profiles + WHERE user_id = ? + """, (field_path, field_path, user_id.localpart), ) # If value_type is None, then the value did not exist. - value_type, value = cast(tuple[str | None, JsonValue], txn.fetchone()) + value_type, value = cast( + tuple[str | None, JsonValue | dict[str, JsonValue]], txn.fetchone() + ) if not value_type: raise StoreError(404, "No row found") # If value_type is object or array, then need to deserialize the JSON. @@ -285,12 +339,321 @@ class ProfileWorkerStore(SQLBaseStore): retcol="fields", desc="get_profile_fields", ) - # The SQLite driver doesn't automatically convert JSON to - # Python objects + # The SQLite driver doesn't have a JSON datatype. if isinstance(self.database_engine, Sqlite3Engine) and result: result = json.loads(result) return result or {} + def get_max_profile_updates_stream_id(self) -> int: + """Get the current maximum stream_id for profile updates.""" + return self._profile_updates_id_gen.get_current_token() + + def get_profile_updates_stream_id_generator(self) -> MultiWriterIdGenerator: + return self._profile_updates_id_gen + + async def get_updated_profile_updates( + self, *, from_id: int, to_id: int, limit: int + ) -> list[tuple[int, str, str, frozenset[str] | None]]: + """Get updates to profile updates between two stream IDs. + + Bounds: from_id < ... <= to_id + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + list of tuples representing stream_id, user_id, action and field_name + """ + if from_id >= to_id: + return [] + + def _get_updated_profile_updates_txn( + txn: LoggingTransaction, + ) -> list[tuple[int, str, str, frozenset[str] | None]]: + txn.execute( + """ + SELECT + stream_id, user_id, action, affected_fields + FROM profile_updates + WHERE + ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC LIMIT ? + """, + (from_id, to_id, limit), + ) + + return [ + ( + stream_id, + user_id, + action, + ( + # affected_fields is a JSON array, turn it to a frozenset[str] + frozenset(db_to_json(affected_fields)) + if affected_fields is not None + else None + ), + ) + for stream_id, user_id, action, affected_fields in txn + ] + + return await self.db_pool.runInteraction( + "get_updated_profile_updates", _get_updated_profile_updates_txn + ) + + async def get_profile_updates_for_fields( + self, + *, + from_id: int, + to_id: int, + field_names: Set[str], + ) -> list[ProfileUpdate]: + """Get profile update markers for the given fields in a stream range. + + Bounds: from_id < ... <= to_id + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + field_names: List of field names to filter against. + + Returns: + list of ProfileUpdates update rows + The `affected_fields` entry in the ProfileUpdates will be filtered. + """ + if from_id >= to_id: + return [] + + if not field_names: + return [] + + def _get_profile_updates_for_fields_txn( + txn: LoggingTransaction, + ) -> list[ProfileUpdate]: + wanted_field_in_elems_clause, wanted_field_in_elems_args = ( + make_in_list_sql_clause( + txn.database_engine, "field_names.value", field_names + ) + ) + + if isinstance(txn.database_engine, PostgresEngine): + # Note that if we had a GIN index on `affected_fields`, this would defeat it. + # If we decide we want one, we should consider using the `?|` operator or its + # clearer-named `jsonb_exists_any` equivalent. + all_field_names_table_expression = ( + "jsonb_array_elements_text(affected_fields) AS field_names(value)" + ) + else: + # json_each is a table-valued function that gives `value` as one of its column names + all_field_names_table_expression = ( + "json_each(affected_fields) AS field_names" + ) + + txn.execute( + f""" + SELECT stream_id, user_id, action, affected_fields + FROM profile_updates + WHERE ? < stream_id AND stream_id <= ? + AND ( + (EXISTS (SELECT 1 FROM {all_field_names_table_expression} WHERE {wanted_field_in_elems_clause})) + OR action != ? + ) + ORDER BY stream_id ASC + """, + ( + from_id, + to_id, + *wanted_field_in_elems_args, + ProfileUpdateAction.UPDATE.value, + ), + ) + rows = cast(list[tuple[int, str, str, str | None]], txn.fetchall()) + + updates: list[ProfileUpdate] = [] + for stream_id, user_id, action, affected_fields_dbjson in rows: + updates.append( + ProfileUpdate( + stream_id=stream_id, + user_id=user_id, + action=action, + affected_fields=( + # Get the field names that were affected by this update + # and intersect with the field names we care about + frozenset(db_to_json(affected_fields_dbjson)) & field_names + ) + if affected_fields_dbjson is not None + else None, + ) + ) + + return updates + + return await self.db_pool.runInteraction( + "get_profile_updates_for_fields", _get_profile_updates_for_fields_txn + ) + + async def get_profile_updates_for_user_and_fields( + self, + *, + from_id: int, + to_id: int, + user_id: str, + field_names: Set[str], + include_users: set[str] | None = None, + ) -> list[ProfileUpdate]: + """Get profile update markers for a user in a stream range. + + The returned profile update rows are restricted to those with a + corresponding `profile_updates_per_user` row for the syncing user. + + Bounds: from_id < ... <= to_id + + Args: + from_id: The starting stream ID (exclusive). + to_id: The ending stream ID (inclusive). + user_id: The full user ID to filter on. + field_names: Set of field names to filter update actions against. + include_users: If given, only include updates for these user IDs. + + Returns: + A list of ProfileUpdates update rows. + """ + if from_id >= to_id: + return [] + + if len(field_names) == 0: + return [] + + if include_users is not None and len(include_users) == 0: + # All updates have been filtered out by lazy-loading. + return [] + + def _get_profile_updates_for_user_and_fields_txn( + txn: LoggingTransaction, + ) -> list[ProfileUpdate]: + wanted_field_in_elems_clause, wanted_field_in_elems_args = ( + make_in_list_sql_clause( + txn.database_engine, "field_names.value", field_names + ) + ) + + if isinstance(txn.database_engine, PostgresEngine): + # Note that if we had a GIN index on `affected_fields`, this would defeat it. + # If we decide we want one, we should consider using the `?|` operator or its + # clearer-named `jsonb_exists_any` equivalent. + all_field_names_table_expression = "jsonb_array_elements_text(pu.affected_fields) AS field_names(value)" + else: + # json_each is a table-valued function that gives `value` as one of its column names + all_field_names_table_expression = ( + "json_each(pu.affected_fields) AS field_names" + ) + + user_clause = "" + user_args: list[str] = [] + if include_users is not None: + # Filter out rows that aren't in `include_users`, if defined. + # This is only relevant when lazy-loading. + user_clause, user_args = make_in_list_sql_clause( + txn.database_engine, "pu.user_id", include_users + ) + user_clause = f"AND {user_clause}" + + # Retrieve profile updates where there's a corresponding row in + # `profile_updates_per_user` within the given `stream_id` bounds + # and the `user_id` and `field_names` match. + txn.execute( + f""" + SELECT pu.stream_id, pu.user_id, pu.action, pu.affected_fields + FROM profile_updates AS pu + INNER JOIN profile_updates_per_user AS puf + ON pu.stream_id = puf.stream_id + WHERE ? < pu.stream_id AND pu.stream_id <= ? + AND puf.user_id = ? + {user_clause} + AND ( + (EXISTS (SELECT 1 FROM {all_field_names_table_expression} WHERE {wanted_field_in_elems_clause})) + OR pu.action != ? + ) + ORDER BY pu.stream_id ASC + """, + ( + from_id, + to_id, + user_id, + *user_args, + *wanted_field_in_elems_args, + ProfileUpdateAction.UPDATE.value, + ), + ) + rows = cast(list[tuple[int, str, str, str | None]], txn.fetchall()) + + updates: list[ProfileUpdate] = [] + for stream_id, updated_user_id, action, affected_fields_dbjson in rows: + updates.append( + ProfileUpdate( + stream_id=stream_id, + user_id=updated_user_id, + action=action, + affected_fields=( + # Get the field names that were affected by this update + # and intersect with the field names we care about + frozenset(db_to_json(affected_fields_dbjson)) & field_names + ) + if affected_fields_dbjson is not None + else None, + ) + ) + + return updates + + return await self.db_pool.runInteraction( + "get_profile_updates_for_user_and_fields", + _get_profile_updates_for_user_and_fields_txn, + ) + + async def get_profile_data_for_users( + self, user_ids: Collection[str] + ) -> dict[str, dict[str, JsonValue | dict[str, JsonValue]]]: + """Fetch displayname/avatar_url/custom fields for a list of users. + + Currently, this returns only local users as the `profiles` table only + tracks local users. + + Args: + user_ids: List of user IDs to filter against. + + Returns: + Dictionary of displayname/avatar_url/custom fields for a list of users. + """ + if not user_ids: + return {} + + rows = await self.db_pool.simple_select_many_batch( + table="profiles", + column="full_user_id", + iterable=user_ids, + retcols=("full_user_id", "displayname", "avatar_url", "fields"), + desc="get_profile_data_for_users", + ) + + results: dict[str, dict[str, JsonValue | dict[str, JsonValue]]] = {} + for full_user_id, displayname, avatar_url, fields in rows: + user_fields = fields or {} + # The SQLite driver doesn't have a JSON datatype. + if isinstance(self.database_engine, Sqlite3Engine) and fields: + user_fields = json.loads(fields) + base_fields = { + ProfileFields.DISPLAYNAME: displayname, + ProfileFields.AVATAR_URL: avatar_url, + } + user_fields.update(base_fields) + + results[full_user_id] = user_fields + + return results + async def create_profile(self, user_id: UserID) -> None: """ Create a blank profile for a user, if one does not already exist. @@ -312,7 +675,7 @@ class ProfileWorkerStore(SQLBaseStore): txn: LoggingTransaction, user_id: UserID, new_field_name: str, - new_value: JsonValue, + new_value: JsonValue | dict[str, JsonValue], ) -> None: # For each entry there are 4 quotes (2 each for key and value), 1 colon, # and 1 comma. @@ -372,103 +735,57 @@ class ProfileWorkerStore(SQLBaseStore): if total_bytes > MAX_PROFILE_SIZE: raise StoreError(400, "Profile too large", Codes.PROFILE_TOO_LARGE) - async def set_profile_displayname( - self, user_id: UserID, new_displayname: str | None - ) -> None: + def _set_profile_field_txn( + self, + txn: LoggingTransaction, + user_id: UserID, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + ) -> int | None: """ - Set the display name of a user. + Wrapper function to set a profile field value and write to the profile + update stream tables in one transaction. Args: - user_id: The user's ID. - new_displayname: The new display name. If this is None, the user's display - name is removed. + txn: The transaction to use + user_id: The user to set the profile field for + field_name: The field to set the value for + new_value: New value for the profile field + + Returns: + The profile updates stream ID that was created in this transaction """ - user_localpart = user_id.localpart + if self._msc4429_enabled: + assert self._is_events_writer - def set_profile_displayname(txn: LoggingTransaction) -> None: - if new_displayname is not None: - self._check_profile_size( - txn, user_id, ProfileFields.DISPLAYNAME, new_displayname - ) + self._check_profile_size(txn, user_id, field_name, new_value) + if field_name in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL): self.db_pool.simple_upsert_txn( txn, table="profiles", - keyvalues={"user_id": user_localpart}, + keyvalues={"user_id": user_id.localpart}, values={ - "displayname": new_displayname, + field_name: new_value, "full_user_id": user_id.to_string(), }, ) - - await self.db_pool.runInteraction( - "set_profile_displayname", set_profile_displayname - ) - - async def set_profile_avatar_url( - self, user_id: UserID, new_avatar_url: str | None - ) -> None: - """ - Set the avatar of a user. - - Args: - user_id: The user's ID. - new_avatar_url: The new avatar URL. If this is None, the user's avatar is - removed. - """ - user_localpart = user_id.localpart - - def set_profile_avatar_url(txn: LoggingTransaction) -> None: - if new_avatar_url is not None: - self._check_profile_size( - txn, user_id, ProfileFields.AVATAR_URL, new_avatar_url - ) - - self.db_pool.simple_upsert_txn( - txn, - table="profiles", - keyvalues={"user_id": user_localpart}, - values={ - "avatar_url": new_avatar_url, - "full_user_id": user_id.to_string(), - }, - ) - - await self.db_pool.runInteraction( - "set_profile_avatar_url", set_profile_avatar_url - ) - - async def set_profile_field( - self, user_id: UserID, field_name: str, new_value: JsonValue - ) -> None: - """ - Set a custom profile field for a user. - - Args: - user_id: The user's ID. - field_name: The name of the custom profile field. - new_value: The value of the custom profile field. - """ - - # Encode to canonical JSON. - canonical_value = encode_canonical_json(new_value) - - def set_profile_field(txn: LoggingTransaction) -> None: - self._check_profile_size(txn, user_id, field_name, new_value) + else: + # Encode to canonical JSON. + canonical_value = encode_canonical_json(new_value) if isinstance(self.database_engine, PostgresEngine): from psycopg2.extras import Json # Note that the || jsonb operator is not recursive, any duplicate # keys will be taken from the second value. - sql = """ - INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_BUILD_OBJECT(?, ?::jsonb)) - ON CONFLICT (user_id) - DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = COALESCE(profiles.fields, '{}'::jsonb) || EXCLUDED.fields - """ - txn.execute( - sql, + """ + INSERT INTO profiles + (user_id, full_user_id, fields) VALUES (?, ?, JSON_BUILD_OBJECT(?, ?::jsonb)) + ON CONFLICT (user_id) + DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = COALESCE(profiles.fields, '{}'::jsonb) || EXCLUDED.fields + """, ( user_id.localpart, user_id.to_string(), @@ -479,19 +796,18 @@ class ProfileWorkerStore(SQLBaseStore): ), ) else: - # You may be tempted to use json_patch instead of providing the parameters - # twice, but that recursively merges objects instead of replacing. - sql = """ - INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_OBJECT(?, JSON(?))) - ON CONFLICT (user_id) - DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = JSON_SET(COALESCE(profiles.fields, '{}'), ?, JSON(?)) - """ # This will error if field_name has double quotes in it, but that's not # possible due to the grammar. json_field_name = f'$."{field_name}"' txn.execute( - sql, + # You may be tempted to use json_patch instead of providing the parameters + # twice, but that recursively merges objects instead of replacing. + """ + INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_OBJECT(?, JSON(?))) + ON CONFLICT (user_id) + DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = JSON_SET(COALESCE(profiles.fields, '{}'), ?, JSON(?)) + """, ( user_id.localpart, user_id.to_string(), @@ -502,9 +818,190 @@ class ProfileWorkerStore(SQLBaseStore): ), ) - await self.db_pool.runInteraction("set_profile_field", set_profile_field) + if not self._msc4429_enabled: + return None - async def delete_profile_field(self, user_id: UserID, field_name: str) -> None: + # Record updates in the profile updates stream + stream_id = self.record_profile_updates_txn( + txn=txn, + user_id=user_id, + action=ProfileUpdateAction.UPDATE, + field_names=[field_name], + ) + + return stream_id + + def record_profile_updates_for_user_joined_room_txn( + self, *, txn: LoggingTransaction, room_id: str, joined_users: set[str] + ) -> None: + """ + Record profile updates for membership additions to a room. + + Currently, updates are only recorded for local users. + + Args: + txn: The transaction to use. + room_id: The room ID concerned. + joined_users: A list of users who have "joined" the room, which here also + means "invited" or "knocked", as in either case we consider that the + users profile should be pushed to the client, should they need it + already even if the user hasn't actually joined the room. + """ + if not self._msc4429_enabled: + return + + assert self._is_events_writer + + # Ensure we're working with local users only + users = {user_id for user_id in joined_users if self.hs.is_mine_id(user_id)} + + # Record the profile updates for each user + for user_id in users: + self.record_profile_updates_txn( + txn=txn, + user_id=UserID.from_string(user_id), + action=ProfileUpdateAction.JOINED_ROOM, + field_names=None, + user_rooms={room_id}, + ) + + def record_profile_updates_txn( + self, + *, + txn: LoggingTransaction, + user_id: UserID, + action: ProfileUpdateAction, + field_names: Collection[str] | None, + user_rooms: set[str] | None = None, + target_users: set[str] | None = None, + ) -> int | None: + """ + Record updates into the profile updates stream tables. + + Currently, updates are only recorded for local users. + + Args: + txn: Transaction to use + user_id: User ID that made the profile update + action: The profile update action, either `update`, `left_room` or + `joined_room` + field_names: A list of fields that were set, if ProfileUpdateAction.UPDATE + user_rooms: Optionally, a set of rooms that the update concerns. If not + given, a database lookup will be done to fetch all the users rooms. + target_users: Optionally, set of users to create profile update stream rows + for. If not given, a database lookup will be done based on `user_rooms`, + or if that is not set, the result of the rooms lookup. + + Returns: + The latest stream ID created in this transaction + """ + if not self._msc4429_enabled: + return None + + if action == ProfileUpdateAction.UPDATE: + assert field_names + else: + assert not field_names + + if not target_users: + if not user_rooms: + rows = self.db_pool.simple_select_onecol_txn( + txn=txn, + table="current_state_events", + keyvalues={ + "type": EventTypes.Member, + "membership": Membership.JOIN, + "state_key": user_id.to_string(), + }, + retcol="room_id", + ) + user_rooms = set(rows) + + rows = self.db_pool.simple_select_many_txn( + txn=txn, + table="local_current_membership", + column="room_id", + iterable=user_rooms, + retcols=("user_id",), + keyvalues={ + "membership": Membership.JOIN, + }, + ) + target_users = {row[0] for row in rows} + + # Ensure we only write updates for local users + users = {user for user in target_users if self.hs.is_mine_id(user)} + + if action in (ProfileUpdateAction.JOINED_ROOM, ProfileUpdateAction.LEFT_ROOM): + users.discard(user_id.to_string()) + if not users: + # No point writing an update for ourselves, if a membership change and no + # other users interested + return None + elif action == ProfileUpdateAction.UPDATE: + # Always include ourselves when updating field values + users.add(user_id.to_string()) + + # Record the profile update + inserted_ts = self.clock.time_msec() + stream_id = self._profile_updates_id_gen.get_next_txn(txn) + + self.db_pool.simple_insert_txn( + txn, + table="profile_updates", + values={ + "stream_id": stream_id, + "instance_name": self._instance_name, + "user_id": user_id.to_string(), + "action": action.value, + "affected_fields": json_encoder.encode(sorted(field_names)) + if field_names + else None, + "inserted_ts": inserted_ts, + }, + ) + + # Add per user tracking rows for each generated stream ID + per_user_values = [(stream_id, user_id, inserted_ts) for user_id in users] + self.db_pool.simple_insert_many_txn( + txn, + table="profile_updates_per_user", + keys=[ + "stream_id", + "user_id", + "inserted_ts", + ], + values=per_user_values, + ) + return stream_id + + async def set_profile_field( + self, + user_id: UserID, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + ) -> int | None: + """ + Set a custom profile field for a user. + + Args: + user_id: The user's ID. + field_name: The name of the custom profile field. + new_value: The value of the custom profile field. + """ + return await self.db_pool.runInteraction( + "set_profile_field", + self._set_profile_field_txn, + user_id, + field_name, + new_value, + ) + + async def delete_profile_field( + self, + user_id: UserID, + field_name: str, + ) -> int | None: """ Remove a custom profile field for a user. @@ -513,39 +1010,68 @@ class ProfileWorkerStore(SQLBaseStore): field_name: The name of the custom profile field. """ - def delete_profile_field(txn: LoggingTransaction) -> None: + if self._msc4429_enabled: + assert self._is_events_writer + + def delete_profile_field(txn: LoggingTransaction) -> int | None: if isinstance(self.database_engine, PostgresEngine): - sql = """ - UPDATE profiles SET fields = fields - ? - WHERE user_id = ? - """ txn.execute( - sql, + """ + UPDATE profiles SET fields = fields - ? + WHERE user_id = ? + """, (field_name, user_id.localpart), ) else: - sql = """ - UPDATE profiles SET fields = json_remove(fields, ?) - WHERE user_id = ? - """ txn.execute( - sql, + """ + UPDATE profiles SET fields = json_remove(fields, ?) + WHERE user_id = ? + """, # This will error if field_name has double quotes in it. (f'$."{field_name}"', user_id.localpart), ) - await self.db_pool.runInteraction("delete_profile_field", delete_profile_field) + if not self._msc4429_enabled: + return None - async def delete_profile(self, user_id: UserID) -> None: + stream_id = self.record_profile_updates_txn( + txn=txn, + user_id=user_id, + action=ProfileUpdateAction.UPDATE, + field_names=[field_name], + ) + return stream_id + + return await self.db_pool.runInteraction( + "delete_profile_field", delete_profile_field + ) + + async def delete_profile( + self, + user_id: UserID, + ) -> None: """ - Deletes an entire user profile, including displayname, avatar_url and all custom fields. - Used at user deactivation when erasure is requested. + Deletes an entire user profile, including displayname, avatar_url and all + custom fields. Used at user deactivation when erasure is requested. + + Args: + user_id: User ID whose profile is going to be deleted. """ - await self.db_pool.simple_delete( - desc="delete_profile", - table="profiles", - keyvalues={"full_user_id": user_id.to_string()}, + def _delete_profile_txn(txn: LoggingTransaction) -> None: + # Delete the profile + txn.execute( + """ + DELETE FROM profiles + WHERE full_user_id = ? + """, + (user_id.to_string(),), + ) + + await self.db_pool.runInteraction( + "delete_profile", + _delete_profile_txn, ) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 667ad1ace8..499b114cb6 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -25,6 +25,7 @@ from typing import ( AbstractSet, Collection, Iterable, + Literal, Mapping, Sequence, cast, @@ -844,63 +845,111 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): @cached(max_entries=10000) async def does_pair_of_users_share_a_room( - self, user_id: str, other_user_id: str + self, + user_id: str, + other_user_id: str, + exclude_room_ids: list[str] | None = None, ) -> bool: raise NotImplementedError() - @cachedList( - cached_method_name="does_pair_of_users_share_a_room", list_name="other_user_ids" - ) - async def _do_users_share_a_room( - self, user_id: str, other_user_ids: Collection[str] - ) -> Mapping[str, bool | None]: + def do_users_share_a_room_txn( + self, + txn: LoggingTransaction, + user_id: str, + other_user_ids: Collection[str], + exclude_room_id: str | None = None, + ) -> dict[str, Literal[True]]: """Return mapping from user ID to whether they share a room with the given user. - Note: `None` and `False` are equivalent and mean they don't share a - room. + Optionally, exclude a room when querying the database. + + Users sharing rooms get `True` returned, users who don't are omitted from the return. + (This is for friendliness with `cachedList` on `_do_users_share_a_room`) """ + state_key_clause, state_key_args = make_in_list_sql_clause( + self.database_engine, "state_key", other_user_ids + ) + # Build SQL args based on whether we are excluding a room ID or not + exclude_room_id_clause = "" + exclude_room_id_args: tuple[str, ...] = () - def do_users_share_a_room_txn( - txn: LoggingTransaction, user_ids: Collection[str] - ) -> dict[str, bool]: - clause, args = make_in_list_sql_clause( - self.database_engine, "state_key", user_ids - ) + if exclude_room_id: + exclude_room_id_clause = "AND room_id != ?" + exclude_room_id_args = (exclude_room_id,) - # This query works by fetching both the list of rooms for the target - # user and the set of other users, and then checking if there is any - # overlap. - sql = f""" + # This query works by fetching both the list of rooms for the target + # user and the set of other users, and then checking if there is any + # overlap. + txn.execute( + f""" SELECT DISTINCT b.state_key FROM ( SELECT room_id FROM current_state_events - WHERE type = 'm.room.member' AND membership = 'join' AND state_key = ? + WHERE type = 'm.room.member' + AND membership = 'join' + AND state_key = ? + {exclude_room_id_clause} ) AS a INNER JOIN ( SELECT room_id, state_key FROM current_state_events - WHERE type = 'm.room.member' AND membership = 'join' AND {clause} - ) AS b using (room_id) - """ + WHERE type = 'm.room.member' + AND membership = 'join' + AND {state_key_clause} + {exclude_room_id_clause} + ) AS b USING (room_id) + """, + ( + user_id, + *exclude_room_id_args, + *state_key_args, + *exclude_room_id_args, + ), + ) + return {u: True for (u,) in txn} - txn.execute(sql, (user_id, *args)) - return {u: True for (u,) in txn} + @cachedList( + cached_method_name="does_pair_of_users_share_a_room", + list_name="other_user_ids", + ) + async def _do_users_share_a_room( + self, + user_id: str, + other_user_ids: Collection[str], + exclude_room_id: str | None = None, + ) -> Mapping[str, Literal[True] | None]: + """Return mapping from user ID to whether they share a room with the + given user. + + Optionally, exclude a room when querying the database. + + This returns `True` for users that share a room and `None` for users that don't. + (This is because of the `cachedList` annotation.) + """ to_return = {} for batch_user_ids in batch_iter(other_user_ids, 1000): res = await self.db_pool.runInteraction( - "do_users_share_a_room", do_users_share_a_room_txn, batch_user_ids + "do_users_share_a_room", + self.do_users_share_a_room_txn, + user_id, + batch_user_ids, + exclude_room_id, ) to_return.update(res) return to_return async def do_users_share_a_room( - self, user_id: str, other_user_ids: Collection[str] + self, + user_id: str, + other_user_ids: Collection[str], + exclude_room_id: str | None = None, ) -> set[str]: """Return the set of users who share a room with the first users""" - - user_dict = await self._do_users_share_a_room(user_id, other_user_ids) + user_dict = await self._do_users_share_a_room( + user_id, other_user_ids, exclude_room_id + ) return {u for u, share_room in user_dict.items() if share_room} @@ -992,6 +1041,20 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): return user_who_share_room + async def get_local_users_who_share_room_with_user(self, user_id: str) -> set[str]: + """Returns the set of local users who share a room with `user_id`. + + This also includes the `user_id` themselves. + """ + room_ids = await self.get_rooms_for_user(user_id) + + user_who_share_room: set[str] = set() + for room_id in room_ids: + user_ids = await self.get_local_users_in_room(room_id) + user_who_share_room.update(user_ids) + + return user_who_share_room + @cached(cache_context=True, iterable=True) async def get_mutual_rooms_between_users( self, user_ids: frozenset[str], cache_context: _CacheContext diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 1afc6d0b2a..3495dce866 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -175,6 +175,7 @@ Changes in SCHEMA_VERSION = 93 Changes in SCHEMA_VERSION = 94 - Add `recheck` column (boolean, default true) to the `redactions` table. - MSC4242: Add state DAG tables. + - MSC4429: Track updates to user profile fields via a new stream. """ diff --git a/synapse/storage/schema/main/delta/94/07_profile_updates.sql b/synapse/storage/schema/main/delta/94/07_profile_updates.sql new file mode 100644 index 0000000000..720f958e09 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/07_profile_updates.sql @@ -0,0 +1,60 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd. +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Track updates to profile fields. +-- For MSC4429 legacy /sync and others. +-- See https://github.com/element-hq/synapse/issues/19981 for potential future directions of this table. +CREATE TABLE IF NOT EXISTS profile_updates ( + stream_id BIGINT NOT NULL PRIMARY KEY, + instance_name TEXT NOT NULL, + + -- The full user ID + user_id TEXT NOT NULL, + + -- Profile action that has happened, see ProfileUpdateAction enum. + action TEXT NOT NULL, + + -- JSON array of the profile field names that have been + -- added, updated or removed in this update. + -- See https://spec.matrix.org/unstable/client-server-api/#profiles + -- This is only present if `action` is `update`. + -- + -- We support multiple field updates at once because it is easy to foresee features + -- involving multiple fields (where getting the illusion of a torn write might be harmful), + -- as well as synchronisation over federation being likely to lead to multiple field changes + -- at once. + affected_fields JSONB NULL, + + -- Unix timestamp (milliseconds) for debugging purposes + inserted_ts BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS profile_updates_by_user ON profile_updates (user_id, stream_id); + +-- We aren't creating a GIN index on `affected_fields` at this time because we don't expect +-- field names to be very selective and therefore an index might not be that useful. + +-- Track which local users should receive each profile update. +CREATE TABLE IF NOT EXISTS profile_updates_per_user ( + -- Stream ID reference to `profile_updates` + stream_id BIGINT NOT NULL REFERENCES profile_updates (stream_id), + + -- The full user ID of the local user that should receive the profile update. + user_id TEXT NOT NULL, + + -- Unix timestamp (milliseconds). Used to determine when to prune rows (to prevent the table + -- from growing indefinitely). + inserted_ts BIGINT NOT NULL, + + PRIMARY KEY (user_id, stream_id) +); diff --git a/synapse/storage/schema/main/delta/94/07_profile_updates_seq.sql.postgres b/synapse/storage/schema/main/delta/94/07_profile_updates_seq.sql.postgres new file mode 100644 index 0000000000..9abf79b68d --- /dev/null +++ b/synapse/storage/schema/main/delta/94/07_profile_updates_seq.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd. +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE SEQUENCE profile_updates_sequence; +-- Synapse streams start at 2, because the default position is 1 +-- so any item inserted at position 1 is ignored. +-- We have to use nextval not START WITH 2, see https://github.com/element-hq/synapse/issues/18712 +SELECT nextval('profile_updates_sequence'); diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 36490fcb35..24120eb736 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -86,6 +86,7 @@ class EventSources: thread_subscriptions_key = self.store.get_max_thread_subscriptions_stream_id() sticky_events_key = self.store.get_max_sticky_events_stream_id() quarantined_media_key = self.store.get_quarantined_media_stream_token() + profile_updates_key = self.store.get_max_profile_updates_stream_id() token = StreamToken( room_key=self.sources.room.get_current_key(), @@ -102,6 +103,7 @@ class EventSources: thread_subscriptions_key=thread_subscriptions_key, sticky_events_key=sticky_events_key, quarantined_media_key=quarantined_media_key, + profile_updates_key=profile_updates_key, ) return token @@ -131,6 +133,7 @@ class EventSources: StreamKeyType.THREAD_SUBSCRIPTIONS: self.store.get_thread_subscriptions_stream_id_generator(), StreamKeyType.STICKY_EVENTS: self.store.get_sticky_events_stream_id_generator(), StreamKeyType.QUARANTINED_MEDIA: self.store.get_quarantined_media_stream_id_generator(), + StreamKeyType.PROFILE_UPDATES: self.store.get_profile_updates_stream_id_generator(), } for _, key in StreamKeyType.__members__.items(): diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index b42893e39d..7516847303 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -1137,6 +1137,7 @@ class StreamKeyType(Enum): THREAD_SUBSCRIPTIONS = "thread_subscriptions_key" STICKY_EVENTS = "sticky_events_key" QUARANTINED_MEDIA = "quarantined_media_key" + PROFILE_UPDATES = "profile_updates_key" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -1144,7 +1145,7 @@ class StreamToken: """A collection of keys joined together by underscores in the following order and which represent the position in their respective streams. - ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242_4141_4343` + ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242_4141_4343_4444` 1. `room_key`: `s2633508` which is a `RoomStreamToken` - `RoomStreamToken`'s can also look like `t426-2633508` or `m56~2.58~3.59` - See the docstring for `RoomStreamToken` for more details. @@ -1160,6 +1161,7 @@ class StreamToken: 11. `thread_subscriptions_key`: 4242 12. `sticky_events_key`: 4141 13. `quarantined_media_key`: 4343 + 14. `profile_updates_key`: 4444 You can see how many of these keys correspond to the various fields in a "/sync" response: @@ -1223,6 +1225,7 @@ class StreamToken: quarantined_media_key: MultiWriterStreamToken = attr.ib( validator=attr.validators.instance_of(MultiWriterStreamToken) ) + profile_updates_key: int _SEPARATOR = "_" START: ClassVar["StreamToken"] @@ -1253,6 +1256,7 @@ class StreamToken: thread_subscriptions_key, sticky_events_key, quarantined_media_key, + profile_updates_key, ) = keys return cls( @@ -1273,6 +1277,7 @@ class StreamToken: quarantined_media_key=await MultiWriterStreamToken.parse( store, quarantined_media_key ), + profile_updates_key=int(profile_updates_key), ) except CancelledError: raise @@ -1298,6 +1303,7 @@ class StreamToken: str(self.thread_subscriptions_key), str(self.sticky_events_key), await self.quarantined_media_key.to_string(store), + str(self.profile_updates_key), ] ) @@ -1371,6 +1377,7 @@ class StreamToken: StreamKeyType.UN_PARTIAL_STATED_ROOMS, StreamKeyType.THREAD_SUBSCRIPTIONS, StreamKeyType.STICKY_EVENTS, + StreamKeyType.PROFILE_UPDATES, ], ) -> int: ... @@ -1426,9 +1433,10 @@ class StreamToken: f"typing: {self.typing_key}, receipt: {self.receipt_key}, " f"account_data: {self.account_data_key}, push_rules: {self.push_rules_key}, " f"to_device: {self.to_device_key}, device_list: {self.device_list_key}, " - f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}," - f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key}" - f"quarantined_media: {self.quarantined_media_key})" + f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}, " + f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key}, " + f"quarantined_media: {self.quarantined_media_key}, " + f"profile_updates: {self.profile_updates_key})" ) @@ -1446,6 +1454,7 @@ StreamToken.START = StreamToken( thread_subscriptions_key=0, sticky_events_key=0, quarantined_media_key=MultiWriterStreamToken(stream=0), + profile_updates_key=0, ) diff --git a/tests/handlers/test_deactivate_account.py b/tests/handlers/test_deactivate_account.py index f8b4098c71..fe5c3ff67f 100644 --- a/tests/handlers/test_deactivate_account.py +++ b/tests/handlers/test_deactivate_account.py @@ -21,7 +21,13 @@ from twisted.internet.testing import MemoryReactor -from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules, Membership +from synapse.api.constants import ( + AccountDataTypes, + EventTypes, + JoinRules, + Membership, + ProfileFields, +) from synapse.push.rulekinds import PRIORITY_CLASS_MAP from synapse.rest import admin from synapse.rest.client import account, login, room @@ -515,8 +521,12 @@ class DeactivateAccountTestCase(HomeserverTestCase): # Setting a display name now works again. user = UserID.from_string(self.user) self.get_success( - self.hs.get_profile_handler().set_displayname( - user, create_requester(user), "Reactivated", by_admin=True + self.hs.get_profile_handler().set_field( + target_user=user, + requester=create_requester(user), + field_name=ProfileFields.DISPLAYNAME, + new_value="Reactivated", + by_admin=True, ) ) self.assertEqual( @@ -530,8 +540,12 @@ class DeactivateAccountTestCase(HomeserverTestCase): """ user = UserID.from_string(self.user) self.get_success( - self.hs.get_profile_handler().set_displayname( - user, create_requester(user), "Original", by_admin=True + self.hs.get_profile_handler().set_field( + target_user=user, + requester=create_requester(user), + field_name=ProfileFields.DISPLAYNAME, + new_value="Original", + by_admin=True, ) ) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 561b45827f..c53c04fbc8 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -26,12 +26,17 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor import synapse.types -from synapse.api.constants import EventTypes +from synapse.api.constants import ( + EventTypes, + ProfileFields, + ProfileUpdateAction, +) from synapse.api.errors import AuthError, SynapseError from synapse.rest import admin -from synapse.rest.client import login, room +from synapse.rest.client import knock, login, room from synapse.server import HomeServer -from synapse.types import JsonDict, UserID +from synapse.storage.databases.main.profile import ProfileUpdate +from synapse.types import JsonDict, StreamKeyType, UserID from synapse.types.state import StateFilter from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -48,6 +53,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): admin.register_servlets, login.register_servlets, room.register_servlets, + knock.register_servlets, ] def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: @@ -62,8 +68,10 @@ class ProfileTestCase(unittest.HomeserverTestCase): self.query_handlers[query_type] = handler self.mock_registry.register_query_handler = register_query_handler + self.mock_hs_notifier = Mock() hs = self.setup_test_homeserver( + notifier=self.mock_hs_notifier, federation_client=self.mock_federation, federation_server=Mock(), federation_registry=self.mock_registry, @@ -83,9 +91,17 @@ class ProfileTestCase(unittest.HomeserverTestCase): self.frank_token = self.login(self.frank.localpart, "frankpassword") self.handler = hs.get_profile_handler() + self.on_new_event = self.mock_hs_notifier.on_new_event def test_get_my_name(self) -> None: - self.get_success(self.store.set_profile_displayname(self.frank, "Frank")) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", + ) + ) displayname = self.get_success(self.handler.get_displayname(self.frank)) @@ -93,8 +109,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): def test_set_my_name(self) -> None: self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ) ) @@ -105,8 +124,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Set displayname again self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", ) ) @@ -117,8 +139,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Set displayname to an empty string self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="", ) ) @@ -130,8 +155,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): """Test that `set_displayname` updates membership events in rooms.""" self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", ) ) @@ -149,8 +177,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): self.assertEqual(membership[state_tuple].content["displayname"], "Frank") self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ) ) @@ -161,12 +192,511 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) self.assertEqual(membership[state_tuple].content["displayname"], "Frank Jr.") + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + def test_update_profile_does_not_update_stream_on_set_field_if_msc4429_not_enabled( + self, + field_name: str, + new_value: str, + ) -> None: + """Test that profile updates don't get recorded in the profile updates stream + if MSC4429 is not enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + updates = self.get_success( + self.store.get_updated_profile_updates( + from_id=1, + to_id=2, + limit=1, + ) + ) + self.assertEqual(len(updates), 0) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + def test_update_profile_does_not_notify_notifier_on_set_field_if_msc4429_not_enabled( + self, + field_name: str, + new_value: str, + ) -> None: + """Test that profile updates do not cause the profile updates stream notifier + to wake up if MSC4429 is not enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 0) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_rooms( + self, field_name: str, new_value: str + ) -> None: + """Test that profile updates do not cause the profile updates stream notifier + to wake up if the user is not in any rooms, if MSC4429 is enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 0) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_updates_stream_on_set_field( + self, field_name: str, new_value: str + ) -> None: + """Test that profile updates get recorded in the profile updates stream if + MSC4429 is enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + updates = self.get_success( + self.store.get_updated_profile_updates( + from_id=1, + to_id=2, + limit=1, + ) + ) + self.assertEqual( + updates[0], + ( + 2, + "@1234abcd:test", + ProfileUpdateAction.UPDATE.value, + {field_name}, + ), + ) + + fields_updates = self.get_success( + self.store.get_profile_updates_for_fields( + from_id=1, + to_id=2, + field_names={field_name}, + ) + ) + self.assertEqual( + fields_updates[0], + ProfileUpdate( + stream_id=2, + user_id="@1234abcd:test", + action=ProfileUpdateAction.UPDATE.value, + affected_fields=frozenset({field_name}), + ), + ) + + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value="", + ) + ) + delete_updates = self.get_success( + self.store.get_updated_profile_updates( + from_id=2, + to_id=3, + limit=1, + ) + ) + self.assertEqual( + delete_updates[0], + (3, "@1234abcd:test", ProfileUpdateAction.UPDATE.value, {field_name}), + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( + self, + ) -> None: + """Test that profiles updates get recorded in the 'per user' profile updates + stream tracking table, if MSC4429 is enabled.""" + self.register_user("roger", "password") + roger_token = self.login("roger", "password") + self.register_user("millie", "password") + millie_token = self.login("millie", "password") + room_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.helper.join(room_id, "@roger:test", tok=roger_token) + self.helper.join(room_id, "@millie:test", tok=millie_token) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="m.status", + new_value='{"text": "Holiday"}', + ) + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=3, + user_id="@millie:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@millie:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id=self.frank.to_string(), + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=2, + user_id="@roger:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=3, + user_id="@millie:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_membership_addition_to_room_adds_the_right_join_action_to_profile_streams( + self, + ) -> None: + """Test that a membership event, which adds a user as joined to a room, + adds the relevant joined action to the profile update stream tables. + + Here we consider join, knock and invite to all be additions to the room + list of members for answering the question "which profiles should we send + information about to clients based on memberships appearing". + """ + self.register_user("roger", "password") + roger_token = self.login("roger", "password") + room_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + + self.helper.join(room_id, "@roger:test", tok=roger_token) + + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id=self.frank.to_string(), + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=2, + user_id="@roger:test", + action="joined_room", + affected_fields=None, + ), + ], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_previous_profile_updates_stream_rows_cleared_if_no_longer_sharing_a_room( + self, + ) -> None: + """Test that previous profile update stream rows are removed for a user if + the user no longer shares rooms with another user, if MSC4429 is enabled. + + This test ensures that when a user leaves a room, we clear all old profile + update rows of users who the user no longer shares rooms with, to avoid + leaking any further profile field updates from those users. + """ + self.register_user("roger", "password") + roger_token = self.login("roger", "password") + self.register_user("millie", "password") + millie_token = self.login("millie", "password") + self.register_user("gracie", "password") + gracie_token = self.login("gracie", "password") + room_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + room_with_millie_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.helper.join(room_id, "@roger:test", tok=roger_token) + self.helper.join(room_with_millie_id, "@millie:test", tok=millie_token) + self.helper.join(room_id, "@gracie:test", tok=gracie_token) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="m.status", + new_value='{"text": "Holiday"}', + ) + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id="@gracie:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=5, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@millie:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=5, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + + # Make frank leave room and verify only the "left room" + gracies join exists + # for roger + self.helper.leave(room_id, self.frank.to_string(), tok=self.frank_token) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id="@gracie:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=6, + user_id=self.frank.to_string(), + action="left_room", + affected_fields=None, + ), + ], + ) + # Make gracie leave room and verify only the "left room"'s + self.helper.leave(room_id, "@gracie:test", tok=gracie_token) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=6, + user_id=self.frank.to_string(), + action="left_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=7, + user_id="@gracie:test", + action="left_room", + affected_fields=None, + ), + ], + ) + + # Sanity check we didn't clear any rows for millie + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@millie:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=5, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_notifies_notifier_on_set_field( + self, + field_name: str, + new_value: str, + ) -> None: + """Test that profile updates wake up the profile updates stream on profile + field updates, if MSC4429 is enabled.""" + self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 1) + def test_background_update_room_membership_on_set_displayname(self) -> None: """Test that `set_displayname` returns immediately and that room membership updates are still done in background.""" self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", ) ) @@ -187,8 +717,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): ): state_tuple = (EventTypes.Member, self.frank.to_string()) self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ) ) @@ -215,8 +748,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): """Test that room membership updates triggered by changing the avatar or the display name are resumed after a restart.""" self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", ) ) @@ -253,8 +789,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): ): state_tuple = (EventTypes.Member, self.frank.to_string()) self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ) ) @@ -320,7 +859,13 @@ class ProfileTestCase(unittest.HomeserverTestCase): @override_config({"enable_set_displayname": False}) def test_set_my_name_if_disabled(self) -> None: # Setting displayname for the first time is allowed - self.get_success(self.store.set_profile_displayname(self.frank, "Frank")) + self.get_success( + self.store.set_profile_field( + user_id=self.frank, + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", + ) + ) self.assertEqual( (self.get_success(self.store.get_profile_displayname(self.frank))), @@ -329,16 +874,22 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Setting displayname a second time is forbidden self.get_failure( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ), SynapseError, ) def test_set_my_name_noauth(self) -> None: self.get_failure( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.bob), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.bob), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ), AuthError, ) @@ -361,8 +912,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): self.store.create_profile(UserID.from_string("@caroline:test")) ) self.get_success( - self.store.set_profile_displayname( - UserID.from_string("@caroline:test"), "Caroline" + self.handler.set_field( + target_user=UserID.from_string("@caroline:test"), + requester=synapse.types.create_requester("@caroline:test"), + field_name=ProfileFields.DISPLAYNAME, + new_value="Caroline", ) ) @@ -380,16 +934,31 @@ class ProfileTestCase(unittest.HomeserverTestCase): def test_get_my_avatar(self) -> None: self.get_success( - self.store.set_profile_avatar_url(self.frank, "http://my.server/me.png") + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/me.png", + ) ) avatar_url = self.get_success(self.handler.get_avatar_url(self.frank)) self.assertEqual("http://my.server/me.png", avatar_url) def test_get_profile_empty_displayname(self) -> None: - self.get_success(self.store.set_profile_displayname(self.frank, None)) self.get_success( - self.store.set_profile_avatar_url(self.frank, "http://my.server/me.png") + self.store.set_profile_field( + user_id=self.frank, + field_name=ProfileFields.DISPLAYNAME, + new_value=None, + ) + ) + self.get_success( + self.store.set_profile_field( + user_id=self.frank, + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/me.png", + ) ) profile = self.get_success(self.handler.get_profile(self.frank.to_string())) @@ -398,10 +967,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): def test_set_my_avatar(self) -> None: self.get_success( - self.handler.set_avatar_url( - self.frank, - synapse.types.create_requester(self.frank), - "http://my.server/pic.gif", + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/pic.gif", ) ) @@ -412,10 +982,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Set avatar again self.get_success( - self.handler.set_avatar_url( - self.frank, - synapse.types.create_requester(self.frank), - "http://my.server/me.png", + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/me.png", ) ) @@ -426,10 +997,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Set avatar to an empty string self.get_success( - self.handler.set_avatar_url( - self.frank, - synapse.types.create_requester(self.frank), - "", + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="", ) ) @@ -441,7 +1013,12 @@ class ProfileTestCase(unittest.HomeserverTestCase): def test_set_my_avatar_if_disabled(self) -> None: # Setting displayname for the first time is allowed self.get_success( - self.store.set_profile_avatar_url(self.frank, "http://my.server/me.png") + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/me.png", + ) ) self.assertEqual( @@ -451,10 +1028,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Set avatar a second time is forbidden self.get_failure( - self.handler.set_avatar_url( - self.frank, - synapse.types.create_requester(self.frank), - "http://my.server/pic.gif", + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/pic.gif", ), SynapseError, ) diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 0db7f30b1f..182ff7a8fc 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -25,7 +25,7 @@ from unittest.mock import AsyncMock, Mock from twisted.internet.testing import MemoryReactor from synapse.api.auth.internal import InternalAuth -from synapse.api.constants import UserTypes +from synapse.api.constants import ProfileFields, UserTypes from synapse.api.errors import ( CodeMessageException, Codes, @@ -824,8 +824,12 @@ class RegistrationTestCase(unittest.HomeserverTestCase): if displayname is not None: # logger.info("setting user display name: %s -> %s", user_id, displayname) - await self.hs.get_profile_handler().set_displayname( - user, requester, displayname, by_admin=True + await self.hs.get_profile_handler().set_field( + target_user=user, + requester=requester, + field_name=ProfileFields.DISPLAYNAME, + new_value=displayname, + by_admin=True, ) return user_id, token diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index d2b2523321..bfb687a6c7 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -18,7 +18,7 @@ # # from http import HTTPStatus -from typing import Collection, ContextManager +from typing import Collection, ContextManager, cast from unittest.mock import AsyncMock, Mock, patch from parameterized import parameterized, parameterized_class @@ -33,6 +33,7 @@ from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.handlers.sync import ( + LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE, SyncConfig, SyncRequestKey, SyncResult, @@ -43,6 +44,7 @@ from synapse.rest.client import knock, login, room from synapse.server import HomeServer from synapse.types import ( JsonDict, + JsonValue, MultiWriterStreamToken, RoomStreamToken, StreamKeyType, @@ -55,6 +57,8 @@ from synapse.util.duration import Duration import tests.unittest import tests.utils from tests.test_utils.event_builders import make_test_pdu_event +from tests.test_utils.event_injection import inject_member_event +from tests.unittest import override_config _request_key = 0 @@ -1152,6 +1156,1543 @@ def generate_sync_config( ) +class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): + """Tests Sync Handler for profile updates.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self.sync_handler = self.hs.get_sync_handler() + self.profile_handler = self.hs.get_profile_handler() + self.store = self.hs.get_datastores().main + self.user = self.register_user("user", "password") + self.tok = self.login("user", "password") + self.other_user = self.register_user("other_user", "password") + self.other_tok = self.login("other_user", "password") + self.joined_room = self.helper.create_room_as(self.user, tok=self.tok) + self.get_success( + self.store.set_profile_field( + user_id=UserID.from_string(self.user), + field_name="m.status", + new_value={"text": "Swimming in the Great Lakes!", "emoji": "🏊"}, + ) + ) + self.helper.join( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + + def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: + """Test that without MSC4429 enabled the initial sync response does not + contain any profile updates.""" + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + self.user, + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual(initial_result.profile_updates, {}) + + @override_config({"include_profile_updates_in_sync": True}) + def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: + """Test that with MSC4429 enabled the initial sync response does not + contain any profile updates, if fields are not filtered for.""" + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual( + initial_result.profile_updates, + {}, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: + """Test that with MSC4429 enabled the initial sync response does + contain profile updates for users who share rooms, for the fields the + client requests. This response should include our syncing user.""" + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + # Also set a field the client doesn't want + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="displayname", + new_value="New displayname", + ) + ) + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": {"ids": ["m.status"]} + }, + ), + ), + request_key=generate_request_key(), + ) + ) + assert initial_result.profile_updates[self.user] is not None + assert initial_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + initial_result.profile_updates["@other_user:test"]["m.status"], + {"text": "On holiday", "emoji": "🏖"}, + ) + self.assertFalse( + "displayname" in initial_result.profile_updates["@other_user:test"].keys(), + ) + self.assertCountEqual( + initial_result.profile_updates.keys(), + [ + self.user, + "@other_user:test", + ], + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_initial_sync_does_not_include_untracked_users_profile_updates( + self, is_lazy: bool + ) -> None: + """Test that with MSC4429 enabled the initial sync response does not + contain profile updates for users who do not share rooms.""" + third_user = self.register_user("third_user", "password") + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertIsNone(initial_result.profile_updates.get(third_user)) + + @override_config({"include_profile_updates_in_sync": True}) + def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( + self, + ) -> None: + """Test that with MSC4429 enabled the initial sync lazy loading response does + contain profile updates for events in the timeline. + + This test ensures lazy loading sync only returns profiles that we also have + events for in the sync response. The second room in this test has the most + recent events from "third_user" and thus we don't get the profile of + "other_user" down the line, who is in the the same rooms as the syncer, + but not in the second room. + """ + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + + requester = create_requester(self.user) + + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + # Check that lazy-loading filters out profile updates as well on initial sync. + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value={"text": "On fire", "emoji": "🔥"}, + ) + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # Only third_user is returned, as lazy loading filters out the events from + # the other users + self.assertCountEqual( + initial_result.profile_updates.keys(), + [ + "@third_user:test", + ], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_profile_update_diffs( + self, + ) -> None: + """Test that with MSC4429 enabled the incremental sync response does + contain profile update diffs.""" + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + # Set a field the client didn't ask for + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="uninterestingfield", + new_value="Content", + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["m.status"], + {"text": "On holiday", "emoji": "🏖"}, + ) + # We only send diffs in incremental sync for profile field updates + self.assertFalse( + "displayname" + in incremental_result.profile_updates["@other_user:test"].keys(), + ) + # The client didn't ask for this field + self.assertFalse( + "uninterestingfield" + in incremental_result.profile_updates["@other_user:test"].keys(), + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( + self, + ) -> None: + """Test that with MSC4429 enabled the incremental sync lazy loading response + does contain profile updates even if the user would be filtered out by lazy + loading. + """ + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value={"text": "On fire", "emoji": "🔥"}, + ) + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="uninterestingfield", + new_value="Content", + ) + ) + # Join a federated user to the room + self.get_success( + inject_member_event( + self.hs, + self.joined_room, + "@federateduser:federatedhs", + "join", + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + + # Ensure our federated user is filtered out, even though they have an + # event in the joined room timeline + self.assertFalse( + "@federateduser:federatedhs" in incremental_result.profile_updates.keys() + ) + + # Lazy loading only filters initial sync profile updates. Incremental syncs + # should include all tracked profile updates for the syncing user. + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [ + "@other_user:test", + "@third_user:test", + ], + ) + assert incremental_result.profile_updates["@other_user:test"] is not None + + # This is a field update, so should be here + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["m.status"], + {"text": "On holiday", "emoji": "🏖"}, + ) + + # We don't have events for this user in this response, so their full profile + # is not included + self.assertFalse( + "displayname" + in incremental_result.profile_updates["@other_user:test"].keys(), + ) + assert incremental_result.profile_updates["@third_user:test"] is not None + + # This user has events in the timeline, thus the fields the client asked for + # are included + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["m.status"], + {"text": "On fire", "emoji": "🔥"}, + ) + self.assertFalse( + "uninterestingfield" + in incremental_result.profile_updates["@third_user:test"].keys(), + ) + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["displayname"], + "third_user", + ) + + @parameterized.expand( + [ + [True, True], + [False, False], + [True, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_sync_filters_out_profile_updates_from_federated_users( + self, + is_initial: bool, + is_lazy: bool, + ) -> None: + """Test that with MSC4429 enabled any sync response + doesn't contain federated users even if there are timeline events from them. + """ + # Join a federated user to the room, causing a membership event into + # the joined rooms sync response + self.get_success( + inject_member_event( + self.hs, + self.joined_room, + "@federateduser1:federatedhs", + "join", + ) + ) + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # Ensure our federated user is filtered out, even though they have an + # event in the joined room timeline + self.assertFalse( + "@federateduser1:federatedhs" in initial_result.profile_updates.keys() + ) + if not is_initial: + # Join another federated user to the room, causing a membership event into + # the joined rooms sync response + self.get_success( + inject_member_event( + self.hs, + self.joined_room, + "@federateduser2:federatedhs", + "join", + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + + # Ensure our federated user is filtered out, even though they have an + # event in the joined room timeline + self.assertFalse( + "@federateduser2:federatedhs" + in incremental_result.profile_updates.keys() + ) + + @parameterized.expand( + [ + [True, True], + [False, False], + [True, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_sync_response_always_includes_the_user_themselves( + self, + is_initial: bool, + is_lazy: bool, + ) -> None: + """Test that with MSC4429 enabled any sync response always contains the users + own updates. + + This test is made with a user that is not in any rooms, to prove our code + to collect interested users from the profile updates always collect the user. + """ + third_user = self.register_user("third_user", "password") + requester = create_requester(third_user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="field", + new_value="Content", + ) + ) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=third_user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + if is_initial: + assert initial_result.profile_updates["@third_user:test"] is not None + self.assertEqual( + initial_result.profile_updates["@third_user:test"]["field"], + "Content", + ) + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="field", + new_value="Content", + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=third_user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@third_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["field"], + "Content", + ) + + @parameterized.expand( + [ + [True, True], + [False, False], + [True, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_sync_profile_updates_works_correctly_with_falsey_values( + self, + is_initial: bool, + is_lazy: bool, + ) -> None: + """Test that with MSC4429 enabled a sync response correctly includes falsey + profile field values. + """ + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": {"ids": ["falseyvaluefield"]}, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + for value in [False, 0, "", [], {}, None]: + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="falseyvaluefield", + new_value=cast(JsonValue | dict[str, JsonValue], value), + ) + ) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + if is_initial: + assert initial_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + initial_result.profile_updates["@other_user:test"][ + "falseyvaluefield" + ], + value, + ) + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="falseyvaluefield", + new_value=cast(JsonValue | dict[str, JsonValue], value), + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + assert ( + incremental_result.profile_updates["@other_user:test"] is not None + ) + self.assertEqual( + incremental_result.profile_updates["@other_user:test"][ + "falseyvaluefield" + ], + value, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_fields( + self, + ) -> None: + """Test that with MSC4429 enabled the incremental sync lazy loading response + filters out unchanged profiles or fields we have recently sent to the client. + """ + requester = create_requester(self.user) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="sooninterestingfield", + new_value="Content", + ) + ) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.helper.send_messages( + room_id=self.joined_room, + num_events=1, + tok=self.other_tok, + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # Lazy loading incremental sync should include profiles from events + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [ + "@other_user:test", + ], + ) + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + set(incremental_result.profile_updates["@other_user:test"].keys()), + {"avatar_url", "displayname"}, + ) + + # If we have more events from the other_user, and do another lazy sync, + # we don't expect the full profile to be sent again due to our cache. + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [], + ) + # However, if we again add an event, we do expect any fields the client didn't + # previously ask for to be there. + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": [ + "m.status", + "displayname", + "avatar_url", + "sooninterestingfield", + ] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [ + "@other_user:test", + ], + ) + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + set(incremental_result.profile_updates["@other_user:test"].keys()), + {"sooninterestingfield"}, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( + self, + ) -> None: + """Test that with MSC4429 enabled the incremental sync response + includes a 'null' for users who are no longer sharing rooms. + """ + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.helper.leave( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertIsNone( + incremental_result.profile_updates["@other_user:test"], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joined( + self, + ) -> None: + """Test that with MSC4429 enabled the incremental sync response + includes all the requested fields of a user who has joined a room with the + syncing user. + """ + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + # Set a status field we don't except to see in sync + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value={"text": "On fire", "emoji": "🔥"}, + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@third_user:test"] is not None + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [third_user], + ) + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["displayname"], + "third_user", + ) + self.assertIsNone( + incremental_result.profile_updates["@third_user:test"]["avatar_url"], + ) + self.assertFalse( + "m.status" in incremental_result.profile_updates["@third_user:test"].keys(), + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> None: + """Test that with MSC4429 enabled the incremental sync response includes + ones own profile updates.""" + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": {"ids": ["m.status", "avatar_url"]} + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.user), + requester=requester, + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@user:test"]["m.status"], + {"text": "On holiday", "emoji": "🏖"}, + ) + # We didn't ask for displayname + self.assertFalse( + "displayname" in incremental_result.profile_updates["@user:test"].keys(), + ) + + @parameterized.expand([[True, False], [True, True], [False, False], [False, True]]) + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_join_leave_join_leave_includes_user_joining_and_leaving( + self, + eager_sync: bool, + is_lazy: bool, + ) -> None: + """Test that with MSC4429 enabled the incremental sync response + correctly handles multiple join / leave / join / leave in a row. + + In the first variant we sync and check after each iteration of join/leave. + In the second variant we only sync at the end of all the join/leaves. + We do both of these as lazy and not-lazy variants. + + This test checks that for a syncing user that is joining and leaving, a member + of the room gets the right profile information down the line. + """ + # Use third_user for this test as other_user is already joined + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": {"ids": ["displayname", "avatar_url"]} + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # Join the room + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + next_token = initial_result.next_batch + if eager_sync: + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=next_token, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # We expect there to be the users profile + self.assertIsNotNone( + incremental_result.profile_updates["@third_user:test"], + ) + next_token = incremental_result.next_batch + # Leave the room + self.helper.leave( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + if eager_sync: + # Ensure we don't get caught by the cache + self.reactor.advance((LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE / 1000) + 1) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=next_token, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # We expect there to be a null profile + self.assertIsNone( + incremental_result.profile_updates["@third_user:test"], + ) + next_token = incremental_result.next_batch + # Join the room + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + if eager_sync: + # Ensure we don't get caught by the cache + self.reactor.advance((LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE / 1000) + 1) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=next_token, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # We expect there to be the users profile + self.assertIsNotNone( + incremental_result.profile_updates["@third_user:test"], + ) + next_token = incremental_result.next_batch + # Leave the room + self.helper.leave( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + # Ensure we don't get caught by the cache + self.reactor.advance((LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE / 1000) + 1) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=next_token, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # The end result should be null profile + self.assertIsNone( + incremental_result.profile_updates["@third_user:test"], + ) + + @parameterized.expand( + [ + ["string value", "new string value"], + [True, False], + [None, "not None"], + [[], ["with item"]], + [{}, {"key": "value"}], + [{"foo": "bar"}, {"bar": "foo"}], + [42, 42.24], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_profile_updates_dont_get_silenced_by_cache( + self, + value: str | bool | list | dict | int | float | None, + new_value: str | bool | list | dict | int | float | None, + ) -> None: + """Test that with MSC4429 enabled the incremental lazy sync response + includes all the profile update changes for the user, even if the profile + field has been recently sent and is in our lazy loading cache. + + Parameterize across different types of potential value types that profile + field updates could have to ensure robustness. + """ + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertFalse( + "@other_user:test" in initial_result.profile_updates, + ) + + # Update the field + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value=cast(JsonValue | dict[str, JsonValue], value), + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # We should have the field change in our sync response. + # It will also be added to the lazy loading cache, so the same field value + # isn't sent again immediately. + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + value, + ) + + # Update the field again, busting our cache + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value=cast(JsonValue | dict[str, JsonValue], new_value), + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # Even though the field was added to the lazy loading members cache, + # it should come through as an update, as the field value changed. + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + new_value, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_cache_and_multiple_updates_to_the_same_field( + self, + ) -> None: + """Test that with MSC4429 enabled the incremental lazy sync response + includes an update to a field, even when the value changes back to a + value set and cached previously. + """ + requester = create_requester(self.user) + filter_json = { + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertFalse( + "@other_user:test" in initial_result.profile_updates, + ) + + # Update the field + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # We should have the field change in our sync response. + # It will also be added to the lazy loading cache, so the same field value + # isn't sent again immediately. + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + "value", + ) + + # Update the field again, busting our cache + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="new value", + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # Even though the field was added to the lazy loading members cache, + # it should come through as an update, as the field value changed. + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + "new value", + ) + + # Update the field again, but to the previous value + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # Even though we've quite recently sent down this value, we should still + # see it again as it is a change + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + "value", + ) + + class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler state behavior when using `use_state_after.""" diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index c4e4170c6f..4deb3c29f4 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2549,7 +2549,7 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase): def test_topo_token_is_accepted(self) -> None: """Test Topo Token is accepted.""" - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), @@ -2563,7 +2563,7 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase): def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: """Test that stream token is accepted for forward pagination.""" - token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index bce199c564..e2e133a85e 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -39,6 +39,7 @@ from synapse.api.constants import ( EventContentFields, EventTypes, LoginType, + ProfileFields, UserTypes, ) from synapse.api.errors import Codes, HttpResponseException, ResourceLimitError @@ -943,18 +944,24 @@ class UsersListTestCase(unittest.HomeserverTestCase): # Set avatar URL to all users, that no user has a NULL value to avoid # different sort order between SQlite and PostreSQL self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@user1:test"), "mxc://url3" + self.store.set_profile_field( + user_id=UserID.from_string("@user1:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://url3", ) ) self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@user2:test"), "mxc://url2" + self.store.set_profile_field( + user_id=UserID.from_string("@user2:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://url2", ) ) self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@admin:test"), "mxc://url1" + self.store.set_profile_field( + user_id=UserID.from_string("@admin:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://url1", ) ) @@ -1546,8 +1553,10 @@ class DeactivateAccountTestCase(unittest.HomeserverTestCase): # set attributes for user self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@user:test"), "mxc://servername/mediaid" + self.store.set_profile_field( + user_id=UserID.from_string("@user:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://servername/mediaid", ) ) self.get_success( @@ -1679,7 +1688,11 @@ class DeactivateAccountTestCase(unittest.HomeserverTestCase): """ # Patch `self.other_user` to have an empty string as their avatar. self.get_success( - self.store.set_profile_avatar_url(UserID.from_string("@user:test"), "") + self.store.set_profile_field( + user_id=UserID.from_string("@user:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="", + ) ) # Check we can still erase them. @@ -2758,8 +2771,10 @@ class UserRestTestCase(unittest.HomeserverTestCase): # set attributes for user self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@user:test"), "mxc://servername/mediaid" + self.store.set_profile_field( + user_id=UserID.from_string("@user:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://servername/mediaid", ) ) self.get_success( diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index c0494c606f..8ffac644b6 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -30,7 +30,7 @@ from twisted.internet.interfaces import IReactorTCP from twisted.internet.testing import MemoryReactor import synapse.rest.admin -from synapse.api.constants import LoginType, Membership +from synapse.api.constants import LoginType, Membership, ProfileFields from synapse.api.errors import Codes, HttpResponseException, SynapseError from synapse.appservice import ApplicationService from synapse.rest import admin @@ -520,13 +520,19 @@ class DeactivateTestCase(unittest.HomeserverTestCase): # Set some profile data that can be checked for after the user is erased self.get_success( - profile_handler.set_displayname( - user_id, create_requester(user_id), "Kermit the Frog" + profile_handler.set_field( + target_user=user_id, + requester=create_requester(user_id), + field_name=ProfileFields.DISPLAYNAME, + new_value="Kermit the Frog", ) ) self.get_success( - profile_handler.set_avatar_url( - user_id, create_requester(user_id), "http://test/Kermit.jpg" + profile_handler.set_field( + target_user=user_id, + requester=create_requester(user_id), + field_name=ProfileFields.AVATAR_URL, + new_value="http://test/Kermit.jpg", ) ) # Verify it is set @@ -578,9 +584,19 @@ class DeactivateTestCase(unittest.HomeserverTestCase): # Can not use the profile handler to set a display name when it is disabled. Use # the database directly store = self.hs.get_datastores().main - self.get_success(store.set_profile_displayname(user_id, "Kermit the Frog")) self.get_success( - store.set_profile_avatar_url(user_id, "http://test/Kermit.jpg") + store.set_profile_field( + user_id=user_id, + field_name=ProfileFields.DISPLAYNAME, + new_value="Kermit the Frog", + ) + ) + self.get_success( + store.set_profile_field( + user_id=user_id, + field_name=ProfileFields.AVATAR_URL, + new_value="http://test/Kermit.jpg", + ) ) # Verify it is set diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 05d42a3d44..a6f5043c92 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2252,7 +2252,7 @@ class RoomMessageListTestCase(RoomBase): self.room_id = self.helper.create_room_as(self.user_id) def test_topo_token_is_accepted(self) -> None: - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2263,7 +2263,7 @@ class RoomMessageListTestCase(RoomBase): self.assertTrue("end" in channel.json_body) def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: - token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) diff --git a/tests/rest/synapse/mas/test_users.py b/tests/rest/synapse/mas/test_users.py index 6f44761bb8..9804f0195d 100644 --- a/tests/rest/synapse/mas/test_users.py +++ b/tests/rest/synapse/mas/test_users.py @@ -17,6 +17,7 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor +from synapse.api.constants import ProfileFields from synapse.api.errors import StoreError from synapse.appservice import ApplicationService from synapse.server import HomeServer @@ -54,9 +55,10 @@ class MasQueryUserResource(BaseTestCase): ) ) self.get_success( - store.set_profile_avatar_url( + store.set_profile_field( user_id=alice, - new_avatar_url="mxc://example.com/avatar", + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://example.com/avatar", ) ) @@ -729,7 +731,13 @@ class MasDeleteUserResource(BaseTestCase): store = self.hs.get_datastores().main # Add custom profile field - self.get_success(store.set_profile_field(alice, "io.element.example", "hello")) + self.get_success( + store.set_profile_field( + user_id=alice, + field_name="io.element.example", + new_value="hello", + ) + ) # Ensure we're testing what we think we are: # check the user has profile data at the start of the test diff --git a/tests/storage/test_main.py b/tests/storage/test_main.py index 7b5774b8c1..dc98876930 100644 --- a/tests/storage/test_main.py +++ b/tests/storage/test_main.py @@ -18,8 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # - - +from synapse.api.constants import ProfileFields from synapse.types import UserID from tests import unittest @@ -38,7 +37,11 @@ class DataStoreTestCase(unittest.HomeserverTestCase): self.get_success(self.store.register_user(self.user.to_string(), "pass")) self.get_success(self.store.create_profile(self.user)) self.get_success( - self.store.set_profile_displayname(self.user, self.displayname) + self.store.set_profile_field( + user_id=self.user, + field_name=ProfileFields.DISPLAYNAME, + new_value=self.displayname, + ) ) users, total = self.get_success( diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index dbaf298697..b65af28962 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -21,6 +21,7 @@ from twisted.internet.testing import MemoryReactor +from synapse.api.constants import ProfileFields from synapse.server import HomeServer from synapse.storage.database import LoggingTransaction from synapse.storage.engines import PostgresEngine @@ -39,7 +40,13 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): def test_displayname(self) -> None: self.get_success(self.store.create_profile(self.u_frank)) - self.get_success(self.store.set_profile_displayname(self.u_frank, "Frank")) + self.get_success( + self.store.set_profile_field( + user_id=self.u_frank, + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", + ) + ) self.assertEqual( "Frank", @@ -47,7 +54,13 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): ) # test set to None - self.get_success(self.store.set_profile_displayname(self.u_frank, None)) + self.get_success( + self.store.set_profile_field( + user_id=self.u_frank, + field_name=ProfileFields.DISPLAYNAME, + new_value=None, + ) + ) self.assertIsNone( self.get_success(self.store.get_profile_displayname(self.u_frank)) @@ -57,7 +70,11 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): self.get_success(self.store.create_profile(self.u_frank)) self.get_success( - self.store.set_profile_avatar_url(self.u_frank, "http://my.site/here") + self.store.set_profile_field( + user_id=self.u_frank, + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.site/here", + ) ) self.assertEqual( @@ -66,7 +83,13 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): ) # test set to None - self.get_success(self.store.set_profile_avatar_url(self.u_frank, None)) + self.get_success( + self.store.set_profile_field( + user_id=self.u_frank, + field_name=ProfileFields.AVATAR_URL, + new_value=None, + ) + ) self.assertIsNone( self.get_success(self.store.get_profile_avatar_url(self.u_frank)) From e3d9b15cc687d620ba4f3201680c6fa83c21359d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Aug 2026 18:11:32 +0100 Subject: [PATCH 075/158] Cleanup cancelled tasks as well (#20068) Rather than just completed and failed tasks. --- changelog.d/20068.misc | 1 + synapse/util/task_scheduler.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20068.misc diff --git a/changelog.d/20068.misc b/changelog.d/20068.misc new file mode 100644 index 0000000000..cb2766e80a --- /dev/null +++ b/changelog.d/20068.misc @@ -0,0 +1 @@ +Routinely purge old cancelled tasks from the database. diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index c1790fd3ae..c01ddf7339 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -400,10 +400,10 @@ class TaskScheduler: """Clean old complete or failed jobs to avoid clutter the DB.""" now = self._clock.time_msec() for task in await self._store.get_scheduled_tasks( - statuses=[TaskStatus.FAILED, TaskStatus.COMPLETE], + statuses=[TaskStatus.FAILED, TaskStatus.CANCELLED, TaskStatus.COMPLETE], max_timestamp=now - TaskScheduler.KEEP_TASKS_FOR_MS, ): - # FAILED and COMPLETE tasks should never be running + # FAILED, CANCELLED and COMPLETE tasks should never be running assert task.id not in self._running_tasks await self._store.delete_scheduled_task(task.id) From f4cf7ab29e3f3c2488425459294a87d1da636d69 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 7 Aug 2026 17:44:06 +0100 Subject: [PATCH 076/158] Clarify media upload limit callback triggers (#20018) --- changelog.d/20018.doc | 1 + docs/modules/media_repository_callbacks.md | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) create mode 100644 changelog.d/20018.doc diff --git a/changelog.d/20018.doc b/changelog.d/20018.doc new file mode 100644 index 0000000000..d644ac6f5e --- /dev/null +++ b/changelog.d/20018.doc @@ -0,0 +1 @@ +Correct the documentation for the `on_media_upload_limit_exceeded` module callback with regards to where it is called from. diff --git a/docs/modules/media_repository_callbacks.md b/docs/modules/media_repository_callbacks.md index d7c9074bde..c55d962512 100644 --- a/docs/modules/media_repository_callbacks.md +++ b/docs/modules/media_repository_callbacks.md @@ -116,10 +116,6 @@ may change without notice. Called when a user attempts to upload media that would exceed a [configured media upload limit](../usage/configuration/config_documentation.html#media_upload_limits). -This callback will only be called on workers which handle -[POST /_matrix/media/v3/upload](https://spec.matrix.org/v1.15/client-server-api/#post_matrixmediav3upload) -requests. - This could be used to inform the user that they have reached a media upload limit through some external method. From 0654c312db8bd1a5c4618f0bb0686e592ae8071f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 7 Aug 2026 15:30:49 -0500 Subject: [PATCH 077/158] Add upgrade notes to point out updated Debian package signing key (2026) (#20066) Part of https://github.com/element-hq/synapse/issues/20038 Language based on the previous time we did this: https://github.com/element-hq/synapse/blob/3cb6c3484daa59c56bbc3d8df295e011c9f0d2cf/docs/upgrade.md?plain=1#L415-L431 --- changelog.d/20066.doc | 1 + docs/upgrade.md | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 changelog.d/20066.doc diff --git a/changelog.d/20066.doc b/changelog.d/20066.doc new file mode 100644 index 0000000000..0ac03b9fa9 --- /dev/null +++ b/changelog.d/20066.doc @@ -0,0 +1 @@ +Add upgrade notes to point out updated Debian package signing key. diff --git a/docs/upgrade.md b/docs/upgrade.md index 2f65d86cbe..a3c529f1f1 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,27 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). + +# Upgrading to v1.159.0 + +## Change of signing key expiry date for the Debian/Ubuntu package repository (2026) + +Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, +please be aware that we have recently updated the expiry date on the repository's GPG signing key, +but this change must be imported into your keyring. + +If you have the `matrix-org-archive-keyring` package installed and it updates before the current key expires, this should +happen automatically. + +Otherwise, if you see an error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you +will need to get a fresh copy of the keys. You can do so with: + +```sh +sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg +``` + +The old version of the key will expire on `2027-03-15`. + # Upgrading to v1.158.0 ## Drop support for Ubuntu 25.10 'Questing Quokka', add support for Ubuntu 26.04 'Resolute Raccoon' From 8aef536bbebc0cd9093d50216e9526604b398071 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 10 Aug 2026 14:34:42 +0100 Subject: [PATCH 078/158] Introduce an `RdataSafeValue` type and correct some minor type annotation mistakes. (#20071) Introduce a `SafeRdataValue` type that represents what is safe to push over the wire in RDATA commands. Spawns from a mistake when working on https://github.com/element-hq/synapse/pull/19556 Would have caught the issue that https://github.com/element-hq/synapse/pull/19556/commits/9bcd110080849575ce11d9b50806423867b5c1b5 fixes. At the same time, correct some mistaken type annotations that flared up after this correction. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/20071.misc | 1 + synapse/replication/tcp/resource.py | 4 +- synapse/replication/tcp/streams/_base.py | 40 +++++++++++++++---- synapse/replication/tcp/streams/federation.py | 7 ++-- 4 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 changelog.d/20071.misc diff --git a/changelog.d/20071.misc b/changelog.d/20071.misc new file mode 100644 index 0000000000..77a8018eb0 --- /dev/null +++ b/changelog.d/20071.misc @@ -0,0 +1 @@ +Introduce an `RdataSafeValue` type and correct some minor type annotation mistakes. \ No newline at end of file diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 95364778cc..7d6886e0b8 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -22,7 +22,7 @@ import logging import random -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Sequence from prometheus_client import Counter @@ -317,7 +317,7 @@ class ReplicationStreamer: def _batch_updates( - updates: list[tuple[Token, StreamRow]], + updates: Sequence[tuple[Token, StreamRow]], ) -> list[tuple[Token | None, StreamRow]]: """Takes a list of updates of form [(token, row)] and sets the token to None for all rows where the next row has the same token. This is used to diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 3d7b0ec3d1..f9c7821a39 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -26,7 +26,8 @@ from typing import ( Any, Awaitable, Callable, - TypeVar, + Sequence, + Union, ) import attr @@ -50,11 +51,36 @@ _STREAM_UPDATE_TARGET_ROW_COUNT = 100 # A stream position token Token = int +RdataSafeValue = Union[ + str, + int, + bool, + float, + None, + # We could probably expand to support immutable types of these, but + # when they come off the wire they will deserialise to the mutable + # types again. + # Since nobody is using it right now, stick to `list` and `dict` + list["RdataSafeValue"], + dict[str, "RdataSafeValue"], +] +""" +Safe types that can be used in RDATA commands and thus used as +the wire format of stream rows. + +Prevents you from thinking you can push e.g. a `frozenset` over +the wire and get it back on the other end. + +At the moment, to be safe, a type has to roundtrip correctly with our JSON codec. +Consult the RdataCommand `from_line` and `to_line` for information. +""" + # The type of a stream update row, after JSON deserialisation, but before # parsing with Stream.parse_row (which turns it into a `ROW_TYPE`). Normally it's # just a row from a database query, though this is dependent on the stream in question. # -StreamRow = TypeVar("StreamRow", bound=tuple) +# NOTE: Prefer to use tuples, but since we have some streams still using list, support those for now. +StreamRow = Union[tuple[RdataSafeValue, ...], list[RdataSafeValue]] # The type returned by the update_function of a stream, as well as get_updates(), # get_updates_since, etc. @@ -64,7 +90,7 @@ StreamRow = TypeVar("StreamRow", bound=tuple) # * `new_last_token` is the new position in stream. # * `limited` is whether there are more updates to fetch. # -StreamUpdateResult = tuple[list[tuple[Token, StreamRow]], Token, bool] +StreamUpdateResult = tuple[Sequence[tuple[Token, StreamRow]], Token, bool] # The type of an update_function for a stream # @@ -407,9 +433,9 @@ class TypingStream(Stream): if hs.get_instance_name() in hs.config.worker.writers.typing: # On the writer, query the typing handler typing_writer_handler = hs.get_typing_writer_handler() - update_function: Callable[ - [str, int, int, int], Awaitable[tuple[list[tuple[int, Any]], int, bool]] - ] = typing_writer_handler.get_all_typing_updates + update_function: UpdateFunction = ( + typing_writer_handler.get_all_typing_updates + ) self.current_token_function = typing_writer_handler.get_current_token else: # Query the typing writer process @@ -819,7 +845,7 @@ class ProfileUpdatesStream(_StreamFromIdGen): updates = await self.store.get_updated_profile_updates( from_id=from_token, to_id=to_token, limit=limit ) - rows = [ + rows: list[tuple[int, tuple[RdataSafeValue, ...]]] = [ ( stream_id, # These are the args to `ProfileUpdatesStreamRow` diff --git a/synapse/replication/tcp/streams/federation.py b/synapse/replication/tcp/streams/federation.py index c99e720381..ac600441dd 100644 --- a/synapse/replication/tcp/streams/federation.py +++ b/synapse/replication/tcp/streams/federation.py @@ -18,13 +18,14 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import TYPE_CHECKING, Any, Awaitable, Callable +from typing import TYPE_CHECKING import attr from synapse.replication.tcp.streams._base import ( Stream, Token, + UpdateFunction, current_token_without_instance, make_http_update_function, ) @@ -57,9 +58,7 @@ class FederationStream(Stream): self.current_token_func = current_token_without_instance( federation_sender.get_current_token ) - update_function: Callable[ - [str, int, int, int], Awaitable[tuple[list[tuple[int, Any]], int, bool]] - ] = federation_sender.get_replication_rows + update_function: UpdateFunction = federation_sender.get_replication_rows elif hs.should_send_federation(): # federation sender: Query master process From 81a42c42ad1fb31c82eb2ea19730fc40ca907f7b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Aug 2026 15:46:39 +0100 Subject: [PATCH 079/158] Abort transactions that sits idle for too long. (#20077) Set `idle_in_transaction_session_timeout` on new postgres connections Idle transactions can block maintenance tasks server-side like vacuums, which can lead to bloat and performance issues. We should never hit this timeout in normal operation, as Synapse should always be actively using the connection when in a transaction and so it should only ever be briefly idle. If we do hit this timeout, it's likely that no progress is being made and so aborting the session is safe. In certain cases we have seen connections leak, particularly when using a connection pooler like pgcat, and this timeout will help with that. --------- Co-authored-by: Eric Eastwood --- changelog.d/20077.misc | 1 + synapse/storage/engines/postgres.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 changelog.d/20077.misc diff --git a/changelog.d/20077.misc b/changelog.d/20077.misc new file mode 100644 index 0000000000..bddbc19288 --- /dev/null +++ b/changelog.d/20077.misc @@ -0,0 +1 @@ +Set `idle_in_transaction_session_timeout` (default 30 minutes) on new PostgreSQL connections, so that wedged connections don't hold locks or block vacuum indefinitely. diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 7cd50fb8f1..225cd45e21 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -65,6 +65,24 @@ class PostgresEngine( self.statement_timeout: int | None = database_config.get( "statement_timeout", Duration(minutes=10).as_millis() ) + + # Abort transactions that sit idle for too long. + # + # Idle transactions can block maintenance tasks server-side like + # vacuums, which can lead to bloat and performance issues. + # + # We should never hit this timeout in normal operation, as Synapse + # should always be actively using the connection when in a transaction + # and so it should only ever be briefly idle. If we do hit this timeout, + # it's likely that no progress is being made and so aborting the session + # is safe. + # + # In certain cases we have seen connections leak, particularly when + # using a connection pooler like pgcat, and this timeout will help with + # that. + self.idle_in_transaction_session_timeout: int | None = database_config.get( + "idle_in_transaction_session_timeout", Duration(minutes=30).as_millis() + ) self._version: int | None = None # unknown as yet self.isolation_level_map: Mapping[int, int] = { @@ -187,6 +205,14 @@ class PostgresEngine( if self.statement_timeout is not None: cursor.execute("SET statement_timeout TO ?", (self.statement_timeout,)) + # Abort transactions that sit idle for too long, as they hold locks + # and block vacuum. + if self.idle_in_transaction_session_timeout is not None: + cursor.execute( + "SET idle_in_transaction_session_timeout TO ?", + (self.idle_in_transaction_session_timeout,), + ) + cursor.close() db_conn.commit() From 62a4bc46203880dd5034483b0e84156d03a3a8c6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Aug 2026 15:47:01 +0100 Subject: [PATCH 080/158] Fix `quarantined_media` stream not being replicated (#20085) The registration of `QuarantinedMediaStream` in `ReplicationCommandHandler._streams_to_replicate` was missed when the stream was added, so an instance configured as the quarantined_media_changes stream writer never sent RDATA/POSITION for it unless it was the main process. Also add the stream to the `instance_map` config validation. Stream was introduced in https://github.com/element-hq/synapse/pull/19558 Fixes https://github.com/element-hq/synapse/issues/20080 --------- Co-authored-by: Claude Fable 5 --- changelog.d/20085.bugfix | 1 + docker/configure_workers_and_start.py | 1 + .../synapse_architecture/streams.md | 1 + docs/workers.md | 9 +- synapse/config/workers.py | 6 ++ synapse/replication/tcp/handler.py | 10 +++ .../tcp/streams/test_quarantined_media.py | 86 +++++++++++++++++++ 7 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20085.bugfix create mode 100644 tests/replication/tcp/streams/test_quarantined_media.py diff --git a/changelog.d/20085.bugfix b/changelog.d/20085.bugfix new file mode 100644 index 0000000000..8253b20ddb --- /dev/null +++ b/changelog.d/20085.bugfix @@ -0,0 +1 @@ +Fix the `quarantined_media` replication stream never being sent when the configured `quarantined_media_changes` stream writer is a worker. Introduced in v1.152.0. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 38f8649b44..c5d785fc38 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -517,6 +517,7 @@ def add_worker_roles_to_shared_config( "typing", "push_rules", "thread_subscriptions", + "quarantined_media_changes", } # Worker-type specific sharding config. Now a single worker can fulfill multiple diff --git a/docs/development/synapse_architecture/streams.md b/docs/development/synapse_architecture/streams.md index e7ab79091e..0a2cbb98e3 100644 --- a/docs/development/synapse_architecture/streams.md +++ b/docs/development/synapse_architecture/streams.md @@ -164,6 +164,7 @@ necessary registration and event handling. - will need an [ID generator](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/storage/databases/main/thread_subscriptions.py#L75) - may need [writer configuration](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/config/workers.py#L177), if there isn't already an obvious source of configuration for which workers should be designated as writers to your new stream. - if adding new writer configuration, add Docker-worker configuration, which lets us configure the writer worker in Complement tests: [[1]](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/docker/configure_workers_and_start.py#L331), [[2]](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/docker/configure_workers_and_start.py#L440) + - Ensure that it's been correctly added to `synapse/replication/tcp/handler.py` and it's `streams_to_replicate` attribute to ensure that changes are actually replicated. - most of the time, you will likely introduce a new datastore class for the concept represented by the new stream, unless there is already an obvious datastore that covers it. - consider whether it may make sense to introduce a handler diff --git a/docs/workers.md b/docs/workers.md index 92b606607c..8959becea6 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -572,10 +572,15 @@ configured as stream writer for the `device_lists` stream: ##### The `quarantined_media_changes` stream The `quarantined_media_changes` stream supports multiple writers. The following endpoints -can be handled by any worker, but should be routed directly to one of the workers -configured as stream writer for the `quarantined_media_changes` stream: +must be routed directly to one of the workers configured as stream writer for the +`quarantined_media_changes` stream (which must also be able to run the media +repository, as these endpoints are only registered on media-capable workers): ^/_synapse/admin/v1/quarantine_media/.*$ + ^/_synapse/admin/v1/room/.*/media/quarantine$ + ^/_synapse/admin/v1/user/.*/media/quarantine$ + ^/_synapse/admin/v1/media/quarantine/.*$ + ^/_synapse/admin/v1/media/unquarantine/.*$ #### Restrict outbound federation traffic to a specific set of workers diff --git a/synapse/config/workers.py b/synapse/config/workers.py index c92534b799..5fe1d1362e 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -375,6 +375,7 @@ class WorkerConfig(Config): "push_rules", "device_lists", "thread_subscriptions", + "quarantined_media_changes", ): instances = _instance_to_list_converter(getattr(self.writers, stream)) for instance in instances: @@ -430,6 +431,11 @@ class WorkerConfig(Config): "Must specify at least one instance to handle `thread_subscriptions` messages." ) + if len(self.writers.quarantined_media_changes) == 0: + raise ConfigError( + "Must specify at least one instance to handle `quarantined_media_changes` messages." + ) + self.events_shard_config = RoutableShardedWorkerHandlingConfig( self.writers.events ) diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 1d0586abe0..b63902b3ec 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -68,6 +68,7 @@ from synapse.replication.tcp.streams import ( from synapse.replication.tcp.streams._base import ( DeviceListsStream, ProfileUpdatesStream, + QuarantinedMediaStream, StickyEventsStream, ThreadSubscriptionsStream, ) @@ -237,6 +238,15 @@ class ReplicationCommandHandler: continue + if isinstance(stream, QuarantinedMediaStream): + if ( + hs.get_instance_name() + in hs.config.worker.writers.quarantined_media_changes + ): + self._streams_to_replicate.append(stream) + + continue + # Only add any other streams if we're on master. if hs.config.worker.worker_app is not None: continue diff --git a/tests/replication/tcp/streams/test_quarantined_media.py b/tests/replication/tcp/streams/test_quarantined_media.py new file mode 100644 index 0000000000..6d8533c7a8 --- /dev/null +++ b/tests/replication/tcp/streams/test_quarantined_media.py @@ -0,0 +1,86 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from synapse.replication.tcp.streams import QuarantinedMediaStream +from synapse.types import UserID + +from tests.replication._base import BaseMultiWorkerStreamTestCase + + +class QuarantinedMediaWorkerWriterTestCase(BaseMultiWorkerStreamTestCase): + """Checks that the quarantined_media stream is replicated when the + configured stream writer is a worker rather than the main process. + """ + + def default_config(self) -> dict: + conf = super().default_config() + conf["stream_writers"] = {"quarantined_media_changes": ["worker1"]} + conf["instance_map"] = { + "main": {"host": "testserv", "port": 8765}, + "worker1": {"host": "testserv", "port": 1001}, + } + return conf + + def test_quarantine_on_worker_writer_replicates_to_main(self) -> None: + main_store = self.hs.get_datastores().main + + worker_hs = self.make_worker_hs( + "synapse.app.generic_worker", {"worker_name": "worker1"} + ) + worker_store = worker_hs.get_datastores().main + + # The worker must consider itself a source of the stream... + self.assertIn( + QuarantinedMediaStream.NAME, + { + stream.NAME + for stream in worker_hs.get_replication_command_handler().get_streams_to_replicate() + }, + ) + # ... and the main process must not, as it isn't a writer. + self.assertNotIn( + QuarantinedMediaStream.NAME, + { + stream.NAME + for stream in self.hs.get_replication_command_handler().get_streams_to_replicate() + }, + ) + + # Quarantining only records a change for media that exists. + self.get_success( + main_store.store_local_media( + media_id="media_id1", + media_type="text/plain", + time_now_ms=self.clock.time_msec(), + upload_name=None, + media_length=100, + user_id=UserID.from_string("@user:test"), + ) + ) + + initial_token = main_store.get_current_quarantined_media_stream_id() + + # Quarantine the media on the worker, i.e. the configured writer. + self.get_success( + worker_store.quarantine_media_by_id("test", "media_id1", "@admin:test") + ) + + self.replicate() + + # The main process only learns of the new stream ID over replication, + # even though the two instances share a database. + self.assertEqual( + main_store.get_current_quarantined_media_stream_id(), + initial_token + 1, + ) From cf8ebebd03175190d0379081b2b086cadab5525e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 10 Aug 2026 10:22:04 -0500 Subject: [PATCH 081/158] Run with `PYTHONUNBUFFERED=1` so we can always see log output (#20075) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We already have [`flush_buffers()` in `start.py`](https://github.com/element-hq/synapse/blob/f4cf7ab29e3f3c2488425459294a87d1da636d69/docker/start.py#L24-L31) for the happy explicit path. Follow-up to https://github.com/element-hq/synapse/pull/18420 This is the same thing we run with in the Synapse Pro for small hosts project, see https://github.com/element-hq/synapse-small-hosts/pull/42 The reason I'm making this change now is that I'm trying to make our Synapse FOSS build similar to our 'Synapse Variants' build we use for all of the Synapse Pro builds. The 'Synapse Variants' builds includes a build for the `multi_synapse` shard and we have [`PYTHONUNBUFFERED=1` in that `Dockerfile`](https://github.com/element-hq/synapse-small-hosts/blob/b64bd32c3feabdf06664a36f36ae0dced541fc66/docker/Dockerfile#L192-L193) which seems like a good measure to also have here. ### Dev notes > #### programming languages where the default “print” statement buffers > > Also, here are a few programming language where the default print statement will buffer output when writing to a pipe, and some ways to disable buffering if you want: > > - Python (disable with `python -u`, or `PYTHONUNBUFFERED=1`, or `sys.stdout.reconfigure(line_buffering=False)`, or `print(x, flush=True)`) > > _-- https://jvns.ca/blog/2024/11/29/why-pipes-get-stuck-buffering/#programming-languages-where-the-default-print-statement-buffers_ --- changelog.d/20075.docker | 1 + docker/Dockerfile | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/20075.docker diff --git a/changelog.d/20075.docker b/changelog.d/20075.docker new file mode 100644 index 0000000000..26c96d47eb --- /dev/null +++ b/changelog.d/20075.docker @@ -0,0 +1 @@ +Run with `PYTHONUNBUFFERED=1` so to ensure can always see log output when things go wrong. diff --git a/docker/Dockerfile b/docker/Dockerfile index f395127da0..30c6b670b6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -203,6 +203,9 @@ EXPOSE 8008/tcp 8448/tcp # SYNAPSE_ENABLE_METRICS=1). EXPOSE 19090/tcp +# Python's `print()` buffers output by default, this tells Python to disable buffering. +ENV PYTHONUNBUFFERED=1 + ENTRYPOINT ["/start.py"] HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ From 93f4ea43c7d7056355b9cd960f86892150956271 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 10 Aug 2026 16:34:56 -0500 Subject: [PATCH 082/158] Fix tests that use `homeserver_to_use=GenericWorkerServer` not being able to be run standalone (#20017) *Spawning from https://github.com/element-hq/synapse/pull/19057#discussion_r2427537811,* Fix tests that use `homeserver_to_use=GenericWorkerServer` not being able to be run standalone Fix https://github.com/element-hq/synapse/issues/15671 (previously https://github.com/matrix-org/synapse/issues/15671) Before this change: ```shell $ poetry run trial tests.storage.test_rollback_worker.WorkerSchemaTests.test_rolling_back tests.storage.test_rollback_worker WorkerSchemaTests test_rolling_back ... [ERROR] =============================================================================== [ERROR] Traceback (most recent call last): File "synapse/tests/unittest.py", line 129, in new return code(orig, *args, **kwargs) File "synapse/tests/unittest.py", line 223, in setUp return orig() File "synapse/tests/unittest.py", line 398, in setUp self.hs = self.make_homeserver(self.reactor, self.clock) File "synapse/tests/storage/test_rollback_worker.py", line 54, in make_homeserver hs = self.setup_test_homeserver(homeserver_to_use=GenericWorkerServer) File "synapse/tests/unittest.py", line 669, in setup_test_homeserver hs = setup_test_homeserver( File "synapse/tests/server.py", line 1260, in setup_test_homeserver prepare_database( File "synapse/synapse/storage/prepare_database.py", line 167, in prepare_database raise UpgradeDatabaseException(EMPTY_DATABASE_ON_WORKER_ERROR) synapse.storage.prepare_database.UpgradeDatabaseException: Uninitialised database: run the main synapse process to prepare the database schema before starting worker processes. tests.storage.test_rollback_worker.WorkerSchemaTests.test_rolling_back ------------------------------------------------------------------------------- Ran 1 tests in 0.034s FAILED (errors=1) ``` ### What was the problem before? [`PREPPED_SQLITE_DB_CONN`](https://github.com/element-hq/synapse/blob/1a1af7b622f219ba0f2501709298caa230ad3912/tests/server.py#L1247-L1262) is a process global and shared between all tests. Whichever test first calls `setup_test_homeserver(...)` builds the template database for the whole trial run. `prepare_database(...)` has a built-in check to refuse upgrading the database ["to avoid multiple workers doing it at once."](https://github.com/element-hq/synapse/blob/1a1af7b622f219ba0f2501709298caa230ad3912/synapse/storage/prepare_database.py#L164-L167) and throw `UpgradeDatabaseException`. This means that if we happen to first run a test that uses a worker (`homeserver_to_use=GenericWorkerServer`), `prepare_database(...)` will just throw its `UpgradeDatabaseException`. And since `PREPPED_SQLITE_DB_CONN` is assigned before `prepare_database(...)`, it will never try to prepare again and the rest of the tests will fail. --- changelog.d/20017.misc | 1 + synapse/storage/prepare_database.py | 14 ++++++++------ tests/server.py | 24 ++++++++++++++++++++---- 3 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 changelog.d/20017.misc diff --git a/changelog.d/20017.misc b/changelog.d/20017.misc new file mode 100644 index 0000000000..f65ae91f00 --- /dev/null +++ b/changelog.d/20017.misc @@ -0,0 +1 @@ +Fix tests that use `homeserver_to_use=GenericWorkerServer` not being able to be run standalone. diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 2def1e130c..979d57ca06 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -100,17 +100,19 @@ def prepare_database( """Prepares a physical database for usage. Will either create all necessary tables or upgrade from an older schema version. - If `config` is None then prepare_database will assert that no upgrade is - necessary, *or* will create a fresh database if the database is empty. - Args: db_conn: database_engine: - config : - application config, or None if we are connecting to an existing - database which we expect to be configured already + config: + application config, or `None` if we are initialising a new empty/blank + database. Note that anything which requires the config, like module schemas, + is skipped when `None` is passed. databases: The name of the databases that will be used with this physical database. Defaults to all databases. + Raises: + ValueError: Passing `config=None` when a database already has a schema raises + `ValueError`, as we can't upgrade an existing database without the config. + UpgradeDatabaseException: If you try to initialize a new database from a worker. """ try: diff --git a/tests/server.py b/tests/server.py index 052f76d755..5297e0b1ac 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1248,19 +1248,35 @@ def setup_test_homeserver( global PREPPED_SQLITE_DB_CONN if PREPPED_SQLITE_DB_CONN is None: temp_engine = create_engine(database_config) - PREPPED_SQLITE_DB_CONN = LoggingDatabaseConnection( + prepped_conn = LoggingDatabaseConnection( conn=sqlite3.connect(":memory:"), engine=temp_engine, default_txn_name="PREPPED_CONN", server_name=server_name, ) - database = DatabaseConnectionConfig("master", database_config) - config.database.databases = [database] prepare_database( - PREPPED_SQLITE_DB_CONN, create_engine(database_config), config + prepped_conn, + create_engine(database_config), + # We pass `config=None` here so that the template database is prepared the + # same way regardless of which test happens to be the first one to run. + # + # Notably, `prepare_database` refuses to initialise an empty database + # when given a worker config, which would otherwise make any test using + # `homeserver_to_use=GenericWorkerServer` fail when run on its own. + # + # Each test still runs `prepare_database` with its own config against its own + # copy of this template (via `hs.setup()`), so anything config specific (like + # module schemas) is still applied per-test. + config=None, ) + # Only publish the template once it's fully prepared. Previously, this was + # assigned before `prepare_database(...)` ran which meant that if + # `prepare_database(...)` failed, we ended up with an unitialized/partial + # database state and never tried to re-create it for subsequent tests. + PREPPED_SQLITE_DB_CONN = prepped_conn + database_config["_TEST_PREPPED_CONN"] = PREPPED_SQLITE_DB_CONN if db_txn_limit is not None: From 68bb5e220a0d97ba86532df9a904ae289aa43850 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:42:32 +0000 Subject: [PATCH 083/158] Bump gitpython from 3.1.57 to 3.1.58 (#20079) Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 329e21b740..ccf78bc962 100644 --- a/poetry.lock +++ b/poetry.lock @@ -583,14 +583,14 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.57" +version = "3.1.58" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.57-py3-none-any.whl", hash = "sha256:4ccf7d73c10f5c9e76043fbb2675ac5a1b3ff5b41e648f56bcbed5f63792ecaf"}, - {file = "gitpython-3.1.57.tar.gz", hash = "sha256:c493ec57c0ef6b19743798b6a5af859c71814b524e7e6f97baa2f8e658961488"}, + {file = "gitpython-3.1.58-py3-none-any.whl", hash = "sha256:d331e722577f0fd7fc1f857419b3ecc07af66282b933d2a4d95f84a042fdd50f"}, + {file = "gitpython-3.1.58.tar.gz", hash = "sha256:621416df10ef3fd0e19fabf9172ddeed0fa704d353d04f194eec56a625a95b22"}, ] [package.dependencies] From 235a9e1aa55fa8ca19d84ca1cf57f3a4a1e29531 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:49:01 +0000 Subject: [PATCH 084/158] Bump docker/login-action from 4.4.0 to 4.5.2 in the minor-and-patches group (#20088) Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 12 ++++++------ .github/workflows/push_complement_image.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5f7ed77d15..47ea46cd9f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,13 +41,13 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -79,7 +79,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -136,14 +136,14 @@ jobs: merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io @@ -176,7 +176,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 257b1a668d..cc20aad2aa 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -52,7 +52,7 @@ jobs: with: poetry-version: "2.4.1" - name: Login to registry - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 with: registry: ghcr.io username: ${{ github.actor }} From 940baa8c6bb77c4e6acb95e9debbbad99483cbe9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:55:11 +0000 Subject: [PATCH 085/158] Bump actions/setup-python from 6.3.0 to 7.0.0 (#20044) Signed-off-by: dependabot[bot] --- .github/workflows/docs-pr.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/latest_deps.yml | 2 +- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/release-artifacts.yml | 8 ++++---- .github/workflows/schema.yaml | 4 ++-- .github/workflows/tests.yml | 12 ++++++------ 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 51d0f6e11b..6e5428dee0 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -24,7 +24,7 @@ jobs: mdbook-version: '0.5.2' - name: Setup python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 3908003504..0d1138f988 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -64,7 +64,7 @@ jobs: run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - name: Setup python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 61f10d5335..c751968df4 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -93,7 +93,7 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: pip install .[all,test] diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 51bea8f763..6d87b87acf 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.x' - run: pip install tomli diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 87812566ea..291c0c75bd 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - id: set-distros @@ -78,7 +78,7 @@ jobs: ${{ runner.os }}-buildx- - name: Set up python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" @@ -131,7 +131,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: # setup-python@v4 doesn't impose a default python version. Need to use 3.x # here, because `python` on osx points to Python 2.7. @@ -168,7 +168,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 3aad64a601..d9105be981 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - name: Install check-jsonschema @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - name: Install PyYAML diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d2fa1ca1c4..c66f3a7de6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -127,7 +127,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20' 'sqlglot>=28.0.0'" @@ -137,7 +137,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: .ci/scripts/check_lockfile.py @@ -220,7 +220,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: "pip install 'towncrier>=18.6.0rc1'" @@ -345,7 +345,7 @@ jobs: if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: "pip install rstcheck" @@ -394,7 +394,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - id: get-matrix @@ -485,7 +485,7 @@ jobs: sudo apt-get -qq install build-essential libffi-dev python3-dev \ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.10" From 3caf9171655b4665207e47db0cbb3399fbcd4590 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:55:35 +0000 Subject: [PATCH 086/158] Bump actions/setup-go from 6.5.0 to 7.0.0 (#20045) Signed-off-by: dependabot[bot] --- .github/workflows/complement_tests.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index cb0c5ba1ea..a2510a247c 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -65,7 +65,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c66f3a7de6..fe33ead2f1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -328,7 +328,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod From d80a4e69dab75bcbf712bba7ec8f892870126486 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 11 Aug 2026 12:55:59 +0100 Subject: [PATCH 087/158] Dust off `make_full_schema` and add CI using it to show schema diffs. (#20027) It seems a lot of time in our trial tests goes towards setting up the database. (The same is probably true of Complement too) We haven't done a full schema for about 20 schema versions, so no surprise! As a result, I want to produce a full schema soon. In this PR I dust off `make_full_schema.sh` (which seems to have broken after some SQLite changes) and add a CI workflow that runs it (producing a diff) when someone changes the schema. The CI workflow also adds a sticky comment showing the diff on the schema, so you can better appreciate the final effect of a change. --- **Dead changes:** I wanted to make it possible to generate a versioned full schema without the manual work, but you can't run the background updates without essentially starting up a homeserver, at which point it might fail because you haven't run all the deltas yet. There's no actual good way to do this, short of deleting the latest deltas (+ tweaking code to not crash without them) or rolling back in the git history. Backed out those changes, but they're preserved on the PR if interesting. --------- Signed-off-by: Olivier 'reivilibre --- .ci/scripts/schema_diff.py | 227 ++++++++++++++++++++++++++++ .github/workflows/schema_diff.yml | 101 +++++++++++++ changelog.d/20027.misc | 1 + docs/development/database_schema.md | 14 +- scripts-dev/make_full_schema.sh | 48 ++++-- 5 files changed, 367 insertions(+), 24 deletions(-) create mode 100755 .ci/scripts/schema_diff.py create mode 100644 .github/workflows/schema_diff.yml create mode 100644 changelog.d/20027.misc diff --git a/.ci/scripts/schema_diff.py b/.ci/scripts/schema_diff.py new file mode 100755 index 0000000000..9b7af72434 --- /dev/null +++ b/.ci/scripts/schema_diff.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +# +# Get a diff showing the change in the database schema. +# +# Usage: +# # Compare against develop (default): +# PGUSER=postgres PGPASSWORD=postgres scripts-dev/schema_diff.py +# +# # Compare against a specific branch/commit: +# PGUSER=postgres PGPASSWORD=postgres scripts-dev/schema_diff.py --base origin/release-v1.100 + +import argparse +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +SCHEMA_DIR = "synapse/storage/schema" +MAKE_FULL_SCHEMA_SCRIPT = REPO_ROOT / "scripts-dev" / "make_full_schema.sh" + + +def run_make_full_schema(output_dir: Path) -> None: + """Run make_full_schema.sh, piping the password via stdin.""" + pg_user = os.environ.get("PGUSER", "") + pg_password = os.environ.get("PGPASSWORD", "") + if not pg_user: + print("ERROR: PGUSER environment variable not set.", file=sys.stderr) + sys.exit(1) + if not pg_password: + print("ERROR: PGPASSWORD environment variable not set.", file=sys.stderr) + sys.exit(1) + + cmd: list[str] = [ + "poetry", + "run", + str(MAKE_FULL_SCHEMA_SCRIPT), + "-p", + pg_user, + "-o", + str(output_dir), + "-c", + "-n", + "9999", + ] + + print(f"Running: {' '.join(cmd)}", file=sys.stderr) + + proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=REPO_ROOT, + text=True, + ) + stdout, _ = proc.communicate(input=pg_password + "\n") + # Forward script output to stderr so stdout stays clean for markdown + if stdout: + print(stdout, file=sys.stderr, end="") + if proc.returncode != 0: + print( + f"ERROR: make_full_schema.sh failed with exit code {proc.returncode}", + file=sys.stderr, + ) + sys.exit(proc.returncode) + + +def diff_schemas( + before_dir: Path, after_dir: Path, before_ref: str, after_ref: str +) -> str: + """Diff SQLite and Postgres full schemas, return a Markdown report.""" + parts: list[str] = [ + "## Schema Diff", + "", + "Please check that this looks as expected!", + "", + ] + + for db in ["common", "main", "state"]: + for engine in ["sqlite", "postgres"]: + filename = f"full.sql.{engine}" + + before_file = before_dir / db / "full_schemas" / "9999" / filename + + after_file = after_dir / db / "full_schemas" / "9999" / filename + + if not before_file.exists(): + raise RuntimeError(f"No before file found for {db = }, {engine = }") + if not after_file.exists(): + raise RuntimeError(f"No after file found for {db = }, {engine = }") + + result = subprocess.run( + ["diff", "-U", "10", str(before_file), str(after_file)], + capture_output=True, + text=True, + ) + + if result.returncode == 0: + parts.append(f"### {db} ({engine})\n\nUnchanged\n") + else: + parts.append(f"### {db} ({engine})\n\n```diff\n{result.stdout}\n```\n") + + return "\n".join(parts) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Show database schema changes") + parser.add_argument( + "--base", + default="develop", + help="Base commit/branch to compare against (default: develop)", + ) + args = parser.parse_args() + + # Create temp output directory with before/after subdirectories + with tempfile.TemporaryDirectory(prefix="schema_diff_") as tmpdir: + after_dir = Path(tmpdir) / "after" + before_dir = Path(tmpdir) / "before" + after_dir.mkdir() + before_dir.mkdir() + + print("\n--- Running make_full_schema.sh (after) ---", file=sys.stderr) + run_make_full_schema(after_dir) + + # Checkout base and run make_full_schema.sh + print( + f"\n--- Checking out {args.base} and running make_full_schema.sh (before) ---", + file=sys.stderr, + ) + + # Save current ref so we can return to it without detaching. + # (Not useful in CI, but is useful for local development.) + # If we are on a named branch, use the branch name; otherwise use the SHA. + head_ref = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=True, + ).stdout.strip() + + before_sha = subprocess.run( + ["git", "rev-parse", args.base], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=True, + ).stdout.strip() + after_sha = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=True, + ).stdout.strip() + + # Check if working tree is dirty before potentially stashing + status = subprocess.run( + [ + "git", + "status", + # Machine-readable output for easy parsing + "--porcelain", + ], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=True, + ).stdout.strip() + + did_stash = False + if status: + print("Stashing local changes before checkout...", file=sys.stderr) + subprocess.run( + [ + "git", + "stash", + "push", + "--include-untracked", + "-m", + "schema_diff temporary stash", + ], + cwd=REPO_ROOT, + check=True, + ) + did_stash = True + + try: + subprocess.run(["git", "checkout", args.base], cwd=REPO_ROOT, check=True) + + # Refresh dependencies + print("Installing dependencies for base commit...", file=sys.stderr) + subprocess.run( + ["poetry", "install", "--no-root", "--extras", "postgres"], + cwd=REPO_ROOT, + check=True, + # Poetry install is noisy, so pipe its stdout to stderr + stdout=sys.stderr, + ) + + run_make_full_schema(before_dir) + finally: + print("Returning to HEAD...", file=sys.stderr) + subprocess.run( + [ + "git", + "checkout", + head_ref, + ], + cwd=REPO_ROOT, + check=True, + ) + if did_stash: + subprocess.run(["git", "stash", "pop"], cwd=REPO_ROOT, check=True) + print("✓ Restored stashed changes.", file=sys.stderr) + + # Diff + print("\n--- Diffing schemas ---", file=sys.stderr) + markdown = diff_schemas(before_dir, after_dir, before_sha, after_sha) + + print(markdown) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/schema_diff.yml b/.github/workflows/schema_diff.yml new file mode 100644 index 0000000000..0c32ef33f4 --- /dev/null +++ b/.github/workflows/schema_diff.yml @@ -0,0 +1,101 @@ +name: Schema Diff + +on: + pull_request: + paths: + - synapse/storage/schema/*/delta/** + - synapse/storage/schema/*/full_schemas/** + - .github/workflows/schema_diff.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # Posts a GitHub PR comment that shows what the effective change to the schema is. + # Provides an excuse to run the `make_full_schema.sh` script in CI (so we keep it working) + # and can act as a review aid for schema changes, letting you easily see the diff of the + # end result, even when background updates or complex schema deltas are present. + show-schema-diff: + name: Show schema diff + runs-on: ubuntu-latest + + permissions: + pull-requests: write + + steps: + - name: Start postgres with a faked clock + background: true + id: postgres + run: | + # Build a docker image with faketime + mkdir /tmp/postgres-faketime + cat > /tmp/postgres-faketime/Dockerfile <<'EOF' + FROM postgres:14-alpine + RUN apk add --no-cache libfaketime + + # It seems like it could be harmful to fake the monotonic timer + # as it might prevent deadlock detection, etc. + # But not sure, just doing out of precaution. + ENV FAKETIME_DONT_FAKE_MONOTONIC=1 + ENTRYPOINT ["faketime", "-f", "2001-05-25 12:42:42", "docker-entrypoint.sh"] + CMD ["postgres"] + EOF + docker build -t localhost/postgres-faketime /tmp/postgres-faketime + + # Run it in the background + docker run -d --name postgres -p 5432:5432 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ + --health-cmd pg_isready --health-interval 10s \ + --health-timeout 5s --health-retries 5 \ + localhost/postgres-faketime + + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Install PostgreSQL client + run: sudo apt-get -qq install postgresql-client + + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + poetry-version: "2.4.1" + extras: "postgres" + python-version: "3.x" + + - name: Wait for Postgres to be up + run: | + until [ "$(docker inspect -f '{{.State.Health.Status}}' postgres)" = healthy ]; do sleep 2; done + + - name: Generate schema diff + id: schema_diff + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + run: | + poetry run python .ci/scripts/schema_diff.py \ + --base origin/develop \ + > "${{ runner.temp }}/schema_diff.md" + + - name: Stop postgres + cancel: postgres + + # If the generation step failed, write an error message so the sticky + # comment step still has a file to read. + - name: Ensure output file exists on failure + if: always() && steps.schema_diff.outcome == 'failure' + run: | + echo "⚠️ Schema diff generation failed. See job logs for details." \ + > "${{ runner.temp }}/schema_diff.md" + + - name: Post sticky PR comment + uses: marocchino/sticky-pull-request-comment@3d7b8546315c63df45a03981d50a43ec19237f80 # v3 + if: always() + with: + header: schema-diff + path: ${{ runner.temp }}/schema_diff.md diff --git a/changelog.d/20027.misc b/changelog.d/20027.misc new file mode 100644 index 0000000000..bdac0dc058 --- /dev/null +++ b/changelog.d/20027.misc @@ -0,0 +1 @@ +Dust off `make_full_schema` and add CI using it to show schema diffs. \ No newline at end of file diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md index 620d1c16b0..25153a06ad 100644 --- a/docs/development/database_schema.md +++ b/docs/development/database_schema.md @@ -84,17 +84,11 @@ reference only. If you want to recreate these schemas, they need to be made from a database that has had all background updates run. -To do so, use `scripts-dev/make_full_schema.sh`. This will produce new -`full.sql.postgres` and `full.sql.sqlite` files. +To do so, use the `make_full_schema.sh` script. This will produce new +`full.sql.postgres` and `full.sql.sqlite` files, after having applied ALL +schema deltas and background updates. -Ensure postgres is installed, then run: - -```sh -./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/ -``` - -NB at the time of writing, this script predates the split into separate `state`/`main` -databases so will require updates to handle that correctly. +Run `poetry run scripts-dev/make_full_schema.sh -h` for usage instructions. ## Delta files diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index 196869ce8f..3a40b02ff8 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -7,11 +7,14 @@ export PGHOST="localhost" POSTGRES_MAIN_DB_NAME="synapse_full_schema_main.$$" POSTGRES_COMMON_DB_NAME="synapse_full_schema_common.$$" POSTGRES_STATE_DB_NAME="synapse_full_schema_state.$$" -REQUIRED_DEPS=("matrix-synapse" "psycopg2") + +# Python package names that must be importable +REQUIRED_DEPS=("synapse" "sqlite3" "psycopg2") usage() { echo echo "Usage: $0 -p -o [-c] [-n ] [-h]" + echo "It is the caller's responsibility to be in the correct Python environment (e.g. using \`poetry run scripts-dev/make_full_schema.sh\`)." echo echo "-p " echo " Username to connect to local postgres instance. The password will be requested" @@ -24,14 +27,19 @@ usage() { echo "-n " echo " Schema number for the new snapshot. Used to set the location of files within " echo " the output directory, mimicking that of synapse/storage/schemas." + echo " NOTE: This does not influence which schema deltas are applied to build the full schema." + echo " Schema deltas past this version will still be applied; if you want to build a" + echo " full schema at a particular version, later deltas first need to be" + echo " temporarily deleted (as well as homeserver code temporarily tweaked" + echo " not to rely on any of those changes when applying background updates)" echo " Defaults to 9999." echo "-h" echo " Display this help text." echo "" echo "" echo "You probably want to invoke this with something like" - echo " docker run --rm -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=synapse -p 5432:5432 postgres:11-alpine" - echo " echo postgres | scripts-dev/make_full_schema.sh -p postgres -n MY_SCHEMA_NUMBER -o synapse/storage/schema" + echo " docker run --rm -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=synapse -p 5432:5432 postgres:14-alpine" + echo " echo postgres | poetry run scripts-dev/make_full_schema.sh -p postgres -n MY_SCHEMA_NUMBER -o synapse/storage/schema" echo "" echo " NB: make sure to run this against the *oldest* supported version of postgres," echo " or else pg_dump might output non-backwards-compatible syntax." @@ -69,10 +77,10 @@ done # Check that required dependencies are installed unsatisfied_requirements=() for dep in "${REQUIRED_DEPS[@]}"; do - pip show "$dep" --quiet || unsatisfied_requirements+=("$dep") + python -c "import $dep" &> /dev/null || unsatisfied_requirements+=("$dep") done if [ ${#unsatisfied_requirements} -ne 0 ]; then - echo "Please install the following python packages: ${unsatisfied_requirements[*]}" + echo 'Please `poetry install --extras postgres` first as the following Python modules are not importable: '"${unsatisfied_requirements[*]}" exit 1 fi @@ -240,16 +248,25 @@ psql "$POSTGRES_STATE_DB_NAME" -w <<< "$DROP_COMMON_TABLES" echo "Dumping SQLite3 schema..." mkdir -p "$OUTPUT_DIR/"{common,main,state}"/full_schemas/$SCHEMA_NUMBER" -sqlite3 "$SQLITE_COMMON_DB" ".schema" > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" +# `--nosys` prevents emitting (some) SQLite internal tables like `sqlite_sequence` that +# aren't managed by the Synapse application and don't need to be part of our full schema. +# +# (SQLite creates `sqlite_sequence` automatically when the database contains at least one +# table with an AUTOINCREMENT column.) +sqlite3 "$SQLITE_COMMON_DB" ".schema --nosys" > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" sqlite3 "$SQLITE_COMMON_DB" ".dump --data-only --nosys" >> "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" -sqlite3 "$SQLITE_MAIN_DB" ".schema" > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" +sqlite3 "$SQLITE_MAIN_DB" ".schema --nosys" > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" sqlite3 "$SQLITE_MAIN_DB" ".dump --data-only --nosys" >> "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" -sqlite3 "$SQLITE_STATE_DB" ".schema" > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" +sqlite3 "$SQLITE_STATE_DB" ".schema --nosys" > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" sqlite3 "$SQLITE_STATE_DB" ".dump --data-only --nosys" >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" cleanup_pg_schema() { # Cleanup as follows: # - Remove empty lines. pg_dump likes to output a lot of these. + # - Remove the `\restrict` and `\unrestrict` psql meta-commands. + # We don't run the pg_dump output through psql so no meta-commands are + # supported and so the security feature (which doesn't apply anyway + # as we trust the source database) is not relevant. # - Remove comment-only lines. pg_dump also likes to output a lot of these to visually # separate tables etc. # - Remove "public." prefix --- the schema name. @@ -274,6 +291,8 @@ cleanup_pg_schema() { # is `true` or omitted, this marks the given integer as having been consumed and # will NOT appear as the nextval. sed -e '/^$/d' \ + -e '/^\\restrict REMOVEME$/d' \ + -e '/^\\unrestrict REMOVEME$/d' \ -e '/^--/d' \ -e 's/public\.//g' \ -e '/^SET /d' \ @@ -282,12 +301,13 @@ cleanup_pg_schema() { echo "Dumping Postgres schema..." -pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +# --restrict-key: set the \restrict key to a static value (normally a random string) for easy find/replacement in the code above. +pg_dump --restrict-key=REMOVEME --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" if [[ "$OUTPUT_DIR" == *synapse/storage/schema ]]; then echo "Updating contrib/datagrip symlinks..." From 0b086d153162e7bbf741fdffb5d70a44fc035e31 Mon Sep 17 00:00:00 2001 From: catfromplan9 <104175360+catfromplan9@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:52:49 +0000 Subject: [PATCH 088/158] Fix thumbnailing MPO images (#20025) Signed-off-by: cat --- changelog.d/20025.bugfix | 1 + synapse/media/thumbnailer.py | 72 ++++++++++++++----- tests/media/test_media_storage.py | 114 ++++++++++++++++++++++++++++++ tests/rest/client/test_media.py | 48 +++++++++++++ 4 files changed, 217 insertions(+), 18 deletions(-) create mode 100644 changelog.d/20025.bugfix diff --git a/changelog.d/20025.bugfix b/changelog.d/20025.bugfix new file mode 100644 index 0000000000..b5feabd30b --- /dev/null +++ b/changelog.d/20025.bugfix @@ -0,0 +1 @@ +Fix thumbnail generation failing for MPO images. Animations that cannot be decoded now fall back to a static thumbnail. diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index d60ca16f26..27f016931a 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -79,6 +79,10 @@ class Thumbnailer: # format in this list becomes part of our trusted computing base. PILLOW_FORMATS = ("jpeg", "png", "webp", "gif") + # Pillow reports MPO (a JPEG holding a stereo pair) as multi-frame, so + # frame count alone doesn't tell us whether something is an animation. + ANIMATED_FORMATS = frozenset({"GIF", "PNG", "WEBP"}) + @staticmethod def set_limits(max_image_pixels: int) -> None: Image.MAX_IMAGE_PIXELS = max_image_pixels @@ -86,6 +90,12 @@ class Thumbnailer: def __init__(self, input_path: str): # Have we closed the image? self._closed = False + # Whether attempting to thumbnail the image failed for some reason. The + # thumbnailing code should fallback to treating the image as static in + # this case. + # + # Cached so a broken animation isn't re-decoded for every thumbnail size. + self._animation_broken = False try: self.image = Image.open(input_path, formats=self.PILLOW_FORMATS) @@ -166,30 +176,52 @@ class Thumbnailer: @property def is_animated(self) -> bool: + if self._animation_broken: + return False + if self.image.format not in self.ANIMATED_FORMATS: + return False return getattr(self.image, "is_animated", False) def _encode_animated_from( self, transform: Callable[[Image.Image], Image.Image] - ) -> BytesIO: + ) -> BytesIO | None: """Apply `transform` to every frame of the source image and encode the - result as an animated thumbnail.""" + result as an animated thumbnail, or None if the source could not be + decoded as an animation and the caller should fall back to a static one. + """ frames = [] durations = [] loop = self.image.info.get("loop", 0) - for frame in ImageSequence.Iterator(self.image): - # Copy the frame to avoid referencing the original image memory. - f = frame.copy() - if f.mode != "RGBA": - f = f.convert("RGBA") - frames.append(transform(f)) - # A duration of 0 is valid (interpretation is implementation-defined, - # see RFC 9649 section 2.7.1.1), so only fall back when unset. The - # 100ms default matches libwebp's animation tools. - duration = frame.info.get("duration") - if duration is None: - duration = self.image.info.get("duration", 100) - durations.append(duration) - return self._encode_animated(frames, durations, loop) + try: + for frame in ImageSequence.Iterator(self.image): + # Copy the frame to avoid referencing the original image memory. + f = frame.copy() + if f.mode != "RGBA": + f = f.convert("RGBA") + frames.append(transform(f)) + # A duration of 0 is valid (interpretation is implementation-defined, + # see RFC 9649 section 2.7.1.1), so only fall back when unset. The + # 100ms default matches libwebp's animation tools. + duration = frame.info.get("duration") + if duration is None: + duration = self.image.info.get("duration", 100) + durations.append(duration) + return self._encode_animated(frames, durations, loop) + except Exception as e: + logger.warning( + "Failed to generate an animated thumbnail, falling back to a " + "static one: %s", + e, + ) + self._animation_broken = True + + try: + # Leave the source on its first frame for the static fallback. + self.image.seek(0) + except Exception as e: + logger.warning("Failed to rewind image to its first frame: %s", e) + + return None @trace def scale( @@ -204,9 +236,11 @@ class Thumbnailer: The bytes of the encoded image ready to be written to disk """ if animated and self.is_animated: - return self._encode_animated_from( + output = self._encode_animated_from( lambda f: self._resize_image(f, width, height) ) + if output is not None: + return output with self._resize_image(self.image, width, height) as scaled: return self._encode_image(scaled, output_type) @@ -242,9 +276,11 @@ class Thumbnailer: crop = (crop_left, 0, crop_right, height) if animated and self.is_animated: - return self._encode_animated_from( + output = self._encode_animated_from( lambda f: self._resize_image(f, scaled_width, scaled_height).crop(crop) ) + if output is not None: + return output with self._resize_image( self.image, scaled_width, scaled_height diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 54d1262054..430a1d6789 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -1428,6 +1428,22 @@ def _make_animated_gif() -> bytes: return out.getvalue() +def _make_mpo() -> bytes: + """Build a two-image MPO: a JPEG holding a stereo pair, not an animation.""" + frames = [Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255))] + out = BytesIO() + frames[0].save(out, format="MPO", save_all=True, append_images=frames[1:]) + return out.getvalue() + + +def _make_stale_mpo() -> bytes: + """Build an MPO whose trailing image is stripped but still advertised.""" + data = _make_mpo() + with Image.open(BytesIO(data)) as image: + primary_size = image.mpinfo[0xB002][0]["Size"] # type: ignore[attr-defined] + return data[:primary_size] + + class ThumbnailerAnimatedTestCase(unittest.TestCase): """Tests that the thumbnailer only animates when explicitly asked to.""" @@ -1444,6 +1460,24 @@ class ThumbnailerAnimatedTestCase(unittest.TestCase): with open(self.png_path, "wb") as f: f.write(SMALL_PNG) + self.mpo_path = os.path.join(self.tempdir, "stereo.jpg") + with open(self.mpo_path, "wb") as f: + f.write(_make_mpo()) + + self.stale_mpo_path = os.path.join(self.tempdir, "stale.jpg") + with open(self.stale_mpo_path, "wb") as f: + f.write(_make_stale_mpo()) + + def assert_is_first_frame(self, output: BytesIO) -> None: + """Raises an `AssertionError` unless the given image is red.""" + pixel = Image.open(output).convert("RGB").getpixel((16, 16)) + assert isinstance(pixel, tuple) + red, green, blue = pixel + # The first frame of every source here is red. + # WebP is lossy, so allow *some* green/blue to be present. + self.assertGreater(red, 200) + self.assertLess(max(green, blue), 50) + def test_scale_static_by_default(self) -> None: """An animated source produces a static thumbnail unless animated=True.""" with Thumbnailer(self.gif_path) as thumbnailer: @@ -1475,3 +1509,83 @@ class ThumbnailerAnimatedTestCase(unittest.TestCase): out = thumbnailer.scale(1, 1, ANIMATED_THUMBNAIL_TYPE, animated=True) result = Image.open(out) self.assertFalse(getattr(result, "is_animated", False)) + + @parameterized.expand([("GIF", "gif"), ("PNG", "apng"), ("WEBP", "webp")]) + def test_animated_formats(self, fmt: str, ext: str) -> None: + """Every animated format we accept produces an animated thumbnail.""" + frames = [ + Image.new("RGBA", (64, 64), color) + for color in ((255, 0, 0, 255), (0, 0, 255, 255)) + ] + out = BytesIO() + frames[0].save( + out, + format=fmt, + save_all=True, + append_images=frames[1:], + duration=100, + loop=0, + ) + path = os.path.join(self.tempdir, f"animated.{ext}") + with open(path, "wb") as f: + f.write(out.getvalue()) + + with Thumbnailer(path) as thumbnailer: + self.assertTrue(thumbnailer.is_animated) + thumbnail = thumbnailer.scale( + 32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True + ) + result = Image.open(thumbnail) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(getattr(result, "n_frames", 1), 2) + + @parameterized.expand(["scale", "crop"]) + def test_mpo_is_not_animated(self, method: str) -> None: + """An MPO packs several stills into one JPEG; not an animation.""" + with Thumbnailer(self.mpo_path) as thumbnailer: + self.assertFalse(thumbnailer.is_animated) + out = getattr(thumbnailer, method)( + 32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True + ) + self.assertFalse(getattr(Image.open(out), "is_animated", False)) + self.assert_is_first_frame(out) + + @parameterized.expand(["scale", "crop"]) + def test_stale_mpo_index_does_not_raise(self, method: str) -> None: + """An MPO advertising frames that are not in the file still thumbnails. + + Regression test for https://github.com/element-hq/synapse/issues/20024. + """ + with Thumbnailer(self.stale_mpo_path) as thumbnailer: + out = getattr(thumbnailer, method)( + 32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True + ) + self.assertEqual(Image.open(out).format, "WEBP") + self.assert_is_first_frame(out) + + def test_fallback_thumbnails_the_first_frame(self) -> None: + """Failing after the frames are read leaves the source parked on the + last one, so the fallback has to rewind.""" + with patch.object(Thumbnailer, "_encode_animated", side_effect=ValueError): + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.scale(32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True) + self.assert_is_first_frame(out) + + @parameterized.expand(["scale", "crop"]) + def test_undecodable_animation_falls_back_to_static(self, method: str) -> None: + """If the frames can't be decoded we still serve a static thumbnail of + the first frame rather than failing the request.""" + # Force the stale MPO down the animated path so decoding it fails. + with patch.object(Thumbnailer, "ANIMATED_FORMATS", frozenset({"MPO"})): + with Thumbnailer(self.stale_mpo_path) as thumbnailer: + self.assertTrue(thumbnailer.is_animated) + out = getattr(thumbnailer, method)( + 32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True + ) + # A broken source isn't retried for every other thumbnail size. + self.assertFalse(thumbnailer.is_animated) + + self.assertEqual(Image.open(out).format, "WEBP") + self.assertFalse(getattr(Image.open(out), "is_animated", False)) + self.assert_is_first_frame(out) diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 1cf401ed4f..b20417a6ed 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -3441,6 +3441,23 @@ class AnimatedThumbnailTestCase(unittest.HomeserverTestCase): ) return out.getvalue() + def _make_mpo(self, stale: bool = False) -> bytes: + """Build a two-image MPO: a JPEG holding a stereo pair, not an + animation. If `stale` is set, the trailing image is stripped but still + advertised.""" + frames = [ + Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255)) + ] + out = io.BytesIO() + frames[0].save(out, format="MPO", save_all=True, append_images=frames[1:]) + data = out.getvalue() + if not stale: + return data + + with Image.open(io.BytesIO(data)) as image: + primary_size = image.mpinfo[0xB002][0]["Size"] # type: ignore[attr-defined] + return data[:primary_size] + def _upload(self, data: bytes, content_type: str) -> MXCUri: return self.get_success( self.repo.create_or_update_content( @@ -3515,6 +3532,37 @@ class AnimatedThumbnailTestCase(unittest.HomeserverTestCase): result = Image.open(io.BytesIO(channel.result["body"])) self.assertFalse(getattr(result, "is_animated", False)) + @parameterized.expand([("intact", False), ("stale_index", True)]) + def test_mpo_is_thumbnailed_as_a_still(self, _name: str, stale: bool) -> None: + """An MPO holds several stills rather than an animation, so it never + gets an animated thumbnail. + + Regression test for https://github.com/element-hq/synapse/issues/20024. + """ + mpo_uri = self._upload(self._make_mpo(stale=stale), "image/jpeg") + + thumbnails = self.get_success( + self.store.get_local_media_thumbnails(mpo_uri.media_id) + ) + self.assertTrue(thumbnails) + self.assertNotIn("image/webp", [info.type for info in thumbnails]) + + channel = self._thumbnail(mpo_uri, animated="true") + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + @parameterized.expand([("intact", False), ("stale_index", True)]) + @override_config({"dynamic_thumbnails": True}) + def test_dynamic_thumbnails_of_mpo_are_static( + self, _name: str, stale: bool + ) -> None: + """The same holds when the thumbnail is generated on demand.""" + mpo_uri = self._upload(self._make_mpo(stale=stale), "image/jpeg") + + channel = self._thumbnail(mpo_uri, animated="true") + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + @override_config({"dynamic_thumbnails": True}) def test_dynamic_thumbnails_generates_and_caches_animated(self) -> None: """With dynamic thumbnails, animated thumbnails are generated on demand From 65853147465de8dee55787c53682196cc96559e3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 11 Aug 2026 10:27:02 -0500 Subject: [PATCH 089/158] Update stream cheatsheet docs to re-link `synapse/config/workers.py` with more references (#20086) The [old link](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/config/workers.py#L177) only references `thread_subscriptions` once but in the latest state of the code with the [new link](https://github.com/element-hq/synapse/blob/62a4bc46203880dd5034483b0e84156d03a3a8c6/synapse/config/workers.py#L184-L187) there are 5 places in the file to look at. Spawning from seeing a few more changes in https://github.com/element-hq/synapse/pull/20085 that were missed in https://github.com/element-hq/synapse/pull/19558 and wondering why I didn't notice before. In fact, some of this clean-up for `thread_subscriptions` (the reference stream from the cheatsheet) wasn't updated until recently (this week) as part of https://github.com/element-hq/synapse/pull/19556 --- changelog.d/20086.doc | 1 + docs/development/synapse_architecture/streams.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20086.doc diff --git a/changelog.d/20086.doc b/changelog.d/20086.doc new file mode 100644 index 0000000000..4c688250c9 --- /dev/null +++ b/changelog.d/20086.doc @@ -0,0 +1 @@ +Update stream cheatsheet docs to re-link `synapse/config/workers.py` which has more references. diff --git a/docs/development/synapse_architecture/streams.md b/docs/development/synapse_architecture/streams.md index 0a2cbb98e3..a647082524 100644 --- a/docs/development/synapse_architecture/streams.md +++ b/docs/development/synapse_architecture/streams.md @@ -162,7 +162,7 @@ necessary registration and event handling. - Update `synapse/_scripts/synapse_port_db.py` so it knows about your new `SEQUENCE`: [add a new `_setup_sequence(...)`](https://github.com/element-hq/synapse/blob/35b55e962aa0bed3b2da5a3c12e3783ddf7604ca/synapse/_scripts/synapse_port_db.py#L883C24-L888) - [create a stream class and stream row class](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/replication/tcp/streams/_base.py#L728) - will need an [ID generator](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/storage/databases/main/thread_subscriptions.py#L75) - - may need [writer configuration](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/config/workers.py#L177), if there isn't already an obvious source of configuration for which workers should be designated as writers to your new stream. + - may need [writer configuration](https://github.com/element-hq/synapse/blob/62a4bc46203880dd5034483b0e84156d03a3a8c6/synapse/config/workers.py#L184-L187), if there isn't already an obvious source of configuration for which workers should be designated as writers to your new stream. - if adding new writer configuration, add Docker-worker configuration, which lets us configure the writer worker in Complement tests: [[1]](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/docker/configure_workers_and_start.py#L331), [[2]](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/docker/configure_workers_and_start.py#L440) - Ensure that it's been correctly added to `synapse/replication/tcp/handler.py` and it's `streams_to_replicate` attribute to ensure that changes are actually replicated. - most of the time, you will likely introduce a new datastore class for the concept represented by the new stream, unless there is already an obvious datastore that covers it. From c3b044b45a81739a5a16f9757360ffbd89fbc84d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 11 Aug 2026 11:25:53 -0500 Subject: [PATCH 090/158] 1.159.0rc1 --- CHANGES.md | 42 +++++++++++++++++++++++++++++++ changelog.d/19556.feature | 2 -- changelog.d/20017.misc | 1 - changelog.d/20018.doc | 1 - changelog.d/20021.misc | 1 - changelog.d/20023.misc | 1 - changelog.d/20025.bugfix | 1 - changelog.d/20027.misc | 1 - changelog.d/20028.misc | 1 - changelog.d/20039.misc | 1 - changelog.d/20039.removal | 1 - changelog.d/20048.misc | 1 - changelog.d/20066.doc | 1 - changelog.d/20068.misc | 1 - changelog.d/20071.misc | 1 - changelog.d/20075.docker | 1 - changelog.d/20077.misc | 1 - changelog.d/20085.bugfix | 1 - changelog.d/20086.doc | 1 - debian/changelog | 6 +++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 22 files changed, 50 insertions(+), 21 deletions(-) delete mode 100644 changelog.d/19556.feature delete mode 100644 changelog.d/20017.misc delete mode 100644 changelog.d/20018.doc delete mode 100644 changelog.d/20021.misc delete mode 100644 changelog.d/20023.misc delete mode 100644 changelog.d/20025.bugfix delete mode 100644 changelog.d/20027.misc delete mode 100644 changelog.d/20028.misc delete mode 100644 changelog.d/20039.misc delete mode 100644 changelog.d/20039.removal delete mode 100644 changelog.d/20048.misc delete mode 100644 changelog.d/20066.doc delete mode 100644 changelog.d/20068.misc delete mode 100644 changelog.d/20071.misc delete mode 100644 changelog.d/20075.docker delete mode 100644 changelog.d/20077.misc delete mode 100644 changelog.d/20085.bugfix delete mode 100644 changelog.d/20086.doc diff --git a/CHANGES.md b/CHANGES.md index bd8b40bab3..7e4a600066 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,45 @@ +# Synapse 1.159.0rc1 (2026-08-11) + +## Features + +- Add optional support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). + Currently defaults to not enabled, and is limited to local users only for the sync results. ([\#19556](https://github.com/element-hq/synapse/issues/19556)) + +## Bugfixes + +- Fix thumbnail generation failing for MPO images. Animations that cannot be decoded now fall back to a static thumbnail. ([\#20025](https://github.com/element-hq/synapse/issues/20025)) +- Fix the `quarantined_media` replication stream never being sent when the configured `quarantined_media_changes` stream writer is a worker. Introduced in v1.152.0. ([\#20085](https://github.com/element-hq/synapse/issues/20085)) + +## Updates to the Docker image + +- Run with `PYTHONUNBUFFERED=1` so to ensure can always see log output when things go wrong. ([\#20075](https://github.com/element-hq/synapse/issues/20075)) + +## Improved Documentation + +- Correct the documentation for the `on_media_upload_limit_exceeded` module callback with regards to where it is called from. ([\#20018](https://github.com/element-hq/synapse/issues/20018)) +- Add upgrade notes to point out updated Debian package signing key. ([\#20066](https://github.com/element-hq/synapse/issues/20066)) +- Update stream cheatsheet docs to re-link `synapse/config/workers.py` which has more references. ([\#20086](https://github.com/element-hq/synapse/issues/20086)) + +## Deprecations and Removals + +- Remove package build targets for Ubuntu 25.10 'Questing Quokka' (end-of-life 2026-07-01). ([\#20039](https://github.com/element-hq/synapse/issues/20039)) + +## Internal Changes + +- Fix tests that use `homeserver_to_use=GenericWorkerServer` not being able to be run standalone. ([\#20017](https://github.com/element-hq/synapse/issues/20017)) +- Fix `RemoteJoinHelper` test helper to handle room version "12" rooms. Contributed by @famedly @jason-famedly. ([\#20021](https://github.com/element-hq/synapse/issues/20021)) +- Fix release script announcement to link to correct release branch of changelog. ([\#20023](https://github.com/element-hq/synapse/issues/20023)) +- Dust off `make_full_schema` and add CI using it to show schema diffs. ([\#20027](https://github.com/element-hq/synapse/issues/20027)) +- Remove broken `DROP` statements for SQLite in `make_full_schema` script. ([\#20028](https://github.com/element-hq/synapse/issues/20028)) +- Add package build targets for Ubuntu 26.04 'Resolute Raccoon'. ([\#20039](https://github.com/element-hq/synapse/issues/20039)) +- Document how to capture a JSON snapshot of a Grafana dashboard to aid in debugging. ([\#20048](https://github.com/element-hq/synapse/issues/20048)) +- Routinely purge old cancelled tasks from the database. ([\#20068](https://github.com/element-hq/synapse/issues/20068)) +- Introduce an `RdataSafeValue` type and correct some minor type annotation mistakes. ([\#20071](https://github.com/element-hq/synapse/issues/20071)) +- Set `idle_in_transaction_session_timeout` (default 30 minutes) on new PostgreSQL connections, so that wedged connections don't hold locks or block vacuum indefinitely. ([\#20077](https://github.com/element-hq/synapse/issues/20077)) + + + + # Synapse 1.158.0 (2026-08-04) ## Deprecations and Removals diff --git a/changelog.d/19556.feature b/changelog.d/19556.feature deleted file mode 100644 index bcb6c5c983..0000000000 --- a/changelog.d/19556.feature +++ /dev/null @@ -1,2 +0,0 @@ -Add optional support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). -Currently defaults to not enabled, and is limited to local users only for the sync results. \ No newline at end of file diff --git a/changelog.d/20017.misc b/changelog.d/20017.misc deleted file mode 100644 index f65ae91f00..0000000000 --- a/changelog.d/20017.misc +++ /dev/null @@ -1 +0,0 @@ -Fix tests that use `homeserver_to_use=GenericWorkerServer` not being able to be run standalone. diff --git a/changelog.d/20018.doc b/changelog.d/20018.doc deleted file mode 100644 index d644ac6f5e..0000000000 --- a/changelog.d/20018.doc +++ /dev/null @@ -1 +0,0 @@ -Correct the documentation for the `on_media_upload_limit_exceeded` module callback with regards to where it is called from. diff --git a/changelog.d/20021.misc b/changelog.d/20021.misc deleted file mode 100644 index 9291f5eea0..0000000000 --- a/changelog.d/20021.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `RemoteJoinHelper` test helper to handle room version "12" rooms. Contributed by @famedly @jason-famedly. diff --git a/changelog.d/20023.misc b/changelog.d/20023.misc deleted file mode 100644 index 17dd927536..0000000000 --- a/changelog.d/20023.misc +++ /dev/null @@ -1 +0,0 @@ -Fix release script announcement to link to correct release branch of changelog. diff --git a/changelog.d/20025.bugfix b/changelog.d/20025.bugfix deleted file mode 100644 index b5feabd30b..0000000000 --- a/changelog.d/20025.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix thumbnail generation failing for MPO images. Animations that cannot be decoded now fall back to a static thumbnail. diff --git a/changelog.d/20027.misc b/changelog.d/20027.misc deleted file mode 100644 index bdac0dc058..0000000000 --- a/changelog.d/20027.misc +++ /dev/null @@ -1 +0,0 @@ -Dust off `make_full_schema` and add CI using it to show schema diffs. \ No newline at end of file diff --git a/changelog.d/20028.misc b/changelog.d/20028.misc deleted file mode 100644 index ad3738f00e..0000000000 --- a/changelog.d/20028.misc +++ /dev/null @@ -1 +0,0 @@ -Remove broken `DROP` statements for SQLite in `make_full_schema` script. \ No newline at end of file diff --git a/changelog.d/20039.misc b/changelog.d/20039.misc deleted file mode 100644 index d8900d1223..0000000000 --- a/changelog.d/20039.misc +++ /dev/null @@ -1 +0,0 @@ -Add package build targets for Ubuntu 26.04 'Resolute Raccoon'. diff --git a/changelog.d/20039.removal b/changelog.d/20039.removal deleted file mode 100644 index 3a83c77275..0000000000 --- a/changelog.d/20039.removal +++ /dev/null @@ -1 +0,0 @@ -Remove package build targets for Ubuntu 25.10 'Questing Quokka' (end-of-life 2026-07-01). diff --git a/changelog.d/20048.misc b/changelog.d/20048.misc deleted file mode 100644 index 07ee0f045a..0000000000 --- a/changelog.d/20048.misc +++ /dev/null @@ -1 +0,0 @@ -Document how to capture a JSON snapshot of a Grafana dashboard to aid in debugging. diff --git a/changelog.d/20066.doc b/changelog.d/20066.doc deleted file mode 100644 index 0ac03b9fa9..0000000000 --- a/changelog.d/20066.doc +++ /dev/null @@ -1 +0,0 @@ -Add upgrade notes to point out updated Debian package signing key. diff --git a/changelog.d/20068.misc b/changelog.d/20068.misc deleted file mode 100644 index cb2766e80a..0000000000 --- a/changelog.d/20068.misc +++ /dev/null @@ -1 +0,0 @@ -Routinely purge old cancelled tasks from the database. diff --git a/changelog.d/20071.misc b/changelog.d/20071.misc deleted file mode 100644 index 77a8018eb0..0000000000 --- a/changelog.d/20071.misc +++ /dev/null @@ -1 +0,0 @@ -Introduce an `RdataSafeValue` type and correct some minor type annotation mistakes. \ No newline at end of file diff --git a/changelog.d/20075.docker b/changelog.d/20075.docker deleted file mode 100644 index 26c96d47eb..0000000000 --- a/changelog.d/20075.docker +++ /dev/null @@ -1 +0,0 @@ -Run with `PYTHONUNBUFFERED=1` so to ensure can always see log output when things go wrong. diff --git a/changelog.d/20077.misc b/changelog.d/20077.misc deleted file mode 100644 index bddbc19288..0000000000 --- a/changelog.d/20077.misc +++ /dev/null @@ -1 +0,0 @@ -Set `idle_in_transaction_session_timeout` (default 30 minutes) on new PostgreSQL connections, so that wedged connections don't hold locks or block vacuum indefinitely. diff --git a/changelog.d/20085.bugfix b/changelog.d/20085.bugfix deleted file mode 100644 index 8253b20ddb..0000000000 --- a/changelog.d/20085.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the `quarantined_media` replication stream never being sent when the configured `quarantined_media_changes` stream writer is a worker. Introduced in v1.152.0. diff --git a/changelog.d/20086.doc b/changelog.d/20086.doc deleted file mode 100644 index 4c688250c9..0000000000 --- a/changelog.d/20086.doc +++ /dev/null @@ -1 +0,0 @@ -Update stream cheatsheet docs to re-link `synapse/config/workers.py` which has more references. diff --git a/debian/changelog b/debian/changelog index a92e6d7ce6..53cca6e378 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.159.0~rc1) stable; urgency=medium + + * New synapse release 1.159.0rc1. + + -- Synapse Packaging team Tue, 11 Aug 2026 16:25:42 +0000 + matrix-synapse-py3 (1.158.0) stable; urgency=medium * New synapse release 1.158.0. diff --git a/pyproject.toml b/pyproject.toml index a7f8b124e5..b03bbacd16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.158.0" +version = "1.159.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 6e7880a900..2282e31b39 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.158/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.159/synapse-config.schema.json type: object properties: modules: From 2eeb4a00f72fe0017dfb15aa4fb6a8f5d53a0ebf Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 11 Aug 2026 11:28:45 -0500 Subject: [PATCH 091/158] Fix grammar --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7e4a600066..a85f6996c2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,7 +12,7 @@ ## Updates to the Docker image -- Run with `PYTHONUNBUFFERED=1` so to ensure can always see log output when things go wrong. ([\#20075](https://github.com/element-hq/synapse/issues/20075)) +- Run with `PYTHONUNBUFFERED=1` to ensure that we can always see log output when things go wrong. ([\#20075](https://github.com/element-hq/synapse/issues/20075)) ## Improved Documentation From 9f1cf3f482c7b9cbe011c163bbd4de369adebd92 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 11 Aug 2026 17:31:15 +0100 Subject: [PATCH 092/158] Fix the documentation on the `federation_domain_whitelist` config option. (#20089) [As discussed in Backend Lobby](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$jz87yx9uFcwKYhwejCLiVlfRLKseUHRnRrz2jZcGvOs?via=jki.re&via=element.io&via=matrix.org) As for justification for calling this the recommended way, - from memory this is accurate - a previous changelog implies this: https://github.com/element-hq/synapse/blob/287904c03ac8892407be960d475c4d25007e8917/docs/changelogs/CHANGES-2022.md?plain=1#L1296 - this is what we are doing internally ([Backend Lobby example](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$MTCVg5-D0k_jq9QZxVSSy1CeMGs8mbG6ld-BWOOy9JI?via=jki.re&via=element.io&via=matrix.org)) I can't find a definitive source though --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/20089.doc | 1 + docs/usage/configuration/config_documentation.md | 9 ++++----- schema/synapse-config.schema.yaml | 12 +++++++----- 3 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 changelog.d/20089.doc diff --git a/changelog.d/20089.doc b/changelog.d/20089.doc new file mode 100644 index 0000000000..e72db0dbde --- /dev/null +++ b/changelog.d/20089.doc @@ -0,0 +1 @@ +Fix the documentation on the `federation_domain_whitelist` config option. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 3fb961d6f8..7654039e5c 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1287,11 +1287,10 @@ Options related to federation. --- ### `federation_domain_whitelist` -*(array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. If not specified, the default is to whitelist everything. - -Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to establish a "private federation", where a group of servers all whitelist each other and have the same whitelist. - -Defaults to `[]`. +*(array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. +If specified as an empty list (`[]`), federation will be denied with all servers. Specifying an empty list (`[]`) here is the recommended way of disabling federation. +If not specified, the default is to allow federation with all servers. +Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to establish a "private federation", where a group of servers all whitelist each other and have the same whitelist. There is no default for this option. Example configuration: ```yaml diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 6e7880a900..7ab9191181 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -280,10 +280,10 @@ properties: description: >- Use this option to include updates of other users' profiles in sync responses, for users who share rooms. - + Requires an [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) compatible client, and is currently limited to legacy sync and local users only. - + This feature is under development and should be used with caution on busy servers or servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring profile information doesn't leak across rooms. @@ -1578,9 +1578,12 @@ properties: Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this - application-layer restriction. If not specified, the default is to - whitelist everything. + application-layer restriction. + If specified as an empty list (`[]`), federation will be denied with all servers. + Specifying an empty list (`[]`) here is the recommended way of disabling federation. + + If not specified, the default is to allow federation with all servers. Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to @@ -1588,7 +1591,6 @@ properties: each other and have the same whitelist. items: type: string - default: [] examples: - - lon.example.com - nyc.example.com From 837d502c987511ac213bd4e12948a8ca199e4f2b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 11 Aug 2026 11:32:20 -0500 Subject: [PATCH 093/158] Call out new Debian package signing key since the old one is expiring More context: https://github.com/element-hq/synapse/issues/20038 --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a85f6996c2..d8e6a17702 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Synapse 1.159.0rc1 (2026-08-11) +Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check +[the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.159/docs/upgrade.md#upgrading-to-v11590) +as we have recently updated the expiry date on the repository's GPG signing key. The old version of the key will expire on `2027-03-15`. + ## Features - Add optional support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). From e08ffd2852ab85668f320d8d978de59004296465 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 11 Aug 2026 11:34:01 -0500 Subject: [PATCH 094/158] Remove already released changes (already in Synapse 1.158.0) --- CHANGES.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d8e6a17702..c1c9c3293d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,10 +24,6 @@ as we have recently updated the expiry date on the repository's GPG signing key. - Add upgrade notes to point out updated Debian package signing key. ([\#20066](https://github.com/element-hq/synapse/issues/20066)) - Update stream cheatsheet docs to re-link `synapse/config/workers.py` which has more references. ([\#20086](https://github.com/element-hq/synapse/issues/20086)) -## Deprecations and Removals - -- Remove package build targets for Ubuntu 25.10 'Questing Quokka' (end-of-life 2026-07-01). ([\#20039](https://github.com/element-hq/synapse/issues/20039)) - ## Internal Changes - Fix tests that use `homeserver_to_use=GenericWorkerServer` not being able to be run standalone. ([\#20017](https://github.com/element-hq/synapse/issues/20017)) @@ -35,7 +31,6 @@ as we have recently updated the expiry date on the repository's GPG signing key. - Fix release script announcement to link to correct release branch of changelog. ([\#20023](https://github.com/element-hq/synapse/issues/20023)) - Dust off `make_full_schema` and add CI using it to show schema diffs. ([\#20027](https://github.com/element-hq/synapse/issues/20027)) - Remove broken `DROP` statements for SQLite in `make_full_schema` script. ([\#20028](https://github.com/element-hq/synapse/issues/20028)) -- Add package build targets for Ubuntu 26.04 'Resolute Raccoon'. ([\#20039](https://github.com/element-hq/synapse/issues/20039)) - Document how to capture a JSON snapshot of a Grafana dashboard to aid in debugging. ([\#20048](https://github.com/element-hq/synapse/issues/20048)) - Routinely purge old cancelled tasks from the database. ([\#20068](https://github.com/element-hq/synapse/issues/20068)) - Introduce an `RdataSafeValue` type and correct some minor type annotation mistakes. ([\#20071](https://github.com/element-hq/synapse/issues/20071)) From e6cc157cbdca31c9457baee1a373103a3f354379 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Aug 2026 11:54:53 -0500 Subject: [PATCH 095/158] Update release script to check more often for actions being completed (every 1m) (#20093) (`_wait_for_actions`) Spawning from seeing the release CI being complete but needing to wait up to 5 minutes longer to continue on. ### Dev notes Originally the waiting was introduced in https://github.com/matrix-org/synapse/pull/13483 GitHub rate limit: > The primary rate limit for unauthenticated requests is 60 requests per hour. > > *-- https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2026-03-10#primary-rate-limit-for-unauthenticated-users* --- changelog.d/20093.misc | 1 + scripts-dev/release.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20093.misc diff --git a/changelog.d/20093.misc b/changelog.d/20093.misc new file mode 100644 index 0000000000..60c745eb47 --- /dev/null +++ b/changelog.d/20093.misc @@ -0,0 +1 @@ +Update release script to check more often for actions being completed so you don't have to wait around as much. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index f78c2c0ab0..58d36f7dcc 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -600,9 +600,15 @@ def _wait_for_actions(gh_token: str | None) -> None: headers["authorization"] = f"token {gh_token}" req = urllib.request.Request(url, headers=headers) + # Initially, wait 10 minutes as we know the CI typically takes 15m+ anyway (no need + # to check over and over when we know it won't be finished yet) time.sleep(10 * 60) while True: - time.sleep(5 * 60) + # Then check once every minute. Short enough to not have to wait around too long + # while not spamming the GitHub API and running into the unauthenticated API + # request rate limit (60 requests per hour so 1 request/minute perfectly aligns + # to not run into any problems) + time.sleep(1 * 60) response = urllib.request.urlopen(req) resp = json.loads(response.read()) From 4bbc6ad74fedb5af556655a9e6d986a32b429b5a Mon Sep 17 00:00:00 2001 From: William L Thomson Jr Date: Thu, 13 Aug 2026 06:12:32 -0400 Subject: [PATCH 096/158] Document lighttpd configuration example from matrix.jaxlug.ngo (#19875) --- changelog.d/19875.doc | 1 + docs/reverse_proxy.md | 85 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19875.doc diff --git a/changelog.d/19875.doc b/changelog.d/19875.doc new file mode 100644 index 0000000000..6ab67a08a2 --- /dev/null +++ b/changelog.d/19875.doc @@ -0,0 +1 @@ +Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo, a contribution from the JaxLUG, the Jacksonville Linux Users Group Inc.. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 0e3303df57..df953bbccc 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -4,8 +4,10 @@ It is recommended to put a reverse proxy such as [nginx](https://nginx.org/en/docs/http/ngx_http_proxy_module.html), [Apache](https://httpd.apache.org/docs/current/mod/mod_proxy_http.html), [Caddy](https://caddyserver.com/docs/quick-starts/reverse-proxy), -[HAProxy](https://www.haproxy.org/) or -[relayd](https://man.openbsd.org/relayd.8) in front of Synapse. +[HAProxy](https://www.haproxy.org/), +[relayd](https://man.openbsd.org/relayd.8) or +[lighttpd](https://www.lighttpd.net/) +in front of Synapse. This has the advantage of being able to expose the default HTTPS port (443) to Matrix clients without requiring Synapse to bind to a privileged port (port numbers less than 1024), avoiding the need for `CAP_NET_BIND_SERVICE` or running as root. @@ -312,6 +314,85 @@ relay "matrix_federation" { } ``` +### lighttpd +```conf +server.modules = ( + "mod_rewrite", + "mod_redirect", + "mod_access", + "mod_setenv", + "mod_openssl", + "mod_proxy", + "mod_accesslog" +) + +server.username = "lighttpd" +server.groupname = "lighttpd" + +# disable for wildcard IPv6 on all, enable for specific IPv6 addresses +# see below IPv4 0.0.0.0 & IPv6 [::] +server.use-ipv6 = "disable" + +ssl.pemfile = "/etc/lighttpd/cert+privkey.pem" +ssl.ca-file = "/etc/lighttpd/fullchain.pem" + +# redirect HTTP traffic to HTTPS, same for IPv6 below +$SERVER["socket"] == "0.0.0.0:80" { + url.redirect = ( + "" => "https://${url.authority.noport}${url.path}${qsa}" + ) +} +$SERVER["socket"] == "0.0.0.0:443" { ssl.engine = "enable" } +$SERVER["socket"] == "0.0.0.0:8448" { ssl.engine = "enable" } +$SERVER["socket"] == "[::]:80" { + url.redirect = ( + "" => "https://${url.authority.noport}${url.path}${qsa}" + ) +} +$SERVER["socket"] == "[::]:443" { ssl.engine = "enable" } +$SERVER["socket"] == "[::]:8448" { ssl.engine = "enable" } + + + +# both lighttpd and synapse need permissions for socket r/w +$HTTP["url"] =~ "(/_matrix|_synapse/admin|/_synapse/client)" { + proxy.balance = "hash" + proxy.server = ( + "" => ( + "backend-socket" => ( + "host" => "/var/lib/synapse/main_public.sock", + "port" => 0 + ) + ) + ) + proxy.forwarded = ( + "for" => 1, + "proto" => 1, + "host" => 1, + ) +} +# protect admin access IPv6 ULA only +$HTTP["remoteip"] !="fd00::/8" { + $HTTP["url"] =~ "^/_synapse/admin/" { + url.access-deny = ( "" ) + } +} +``` + +[Delegation](delegate.md) example: +```conf +url.rewrite-once = ( + "^/\.well-known/matrix/client$" => "/.well-known/matrix/client.json", + "^/\.well-known/matrix/server$" => "/.well-known/matrix/server.json" +) + +# This condition intentionally matches the post-rewrite URLs. +$HTTP["url"] =~ "^/\.well-known/matrix/(client|server)\.json$" { + mimetype.assign = ( ".json" => "application/json" ) + setenv.set-response-header = ( "Access-Control-Allow-Origin" => "*" ) +} +``` + ## Health check endpoint From b7db66c21e845f2ada12d38e5ec848c5bbfe6f07 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 13 Aug 2026 11:13:36 +0100 Subject: [PATCH 097/158] Clarify comment From https://github.com/element-hq/synapse/pull/19875#discussion_r3774386943. I was unable to do so on the PR quickly as the branch was not available for maintainers to edit. --- docs/reverse_proxy.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index df953bbccc..fc38c06335 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -329,8 +329,11 @@ server.modules = ( server.username = "lighttpd" server.groupname = "lighttpd" -# disable for wildcard IPv6 on all, enable for specific IPv6 addresses -# see below IPv4 0.0.0.0 & IPv6 [::] +# We set this to "disable" and use IPv6 `[::]` explicitly below, +# in order to listen on all incoming IPv6 addresses. +# +# If you only want to listen on specific IPv6 addresses, set this +# to "enable" and specify said addresses below. server.use-ipv6 = "disable" ssl.pemfile = "/etc/lighttpd/cert+privkey.pem" From 0c6714d0d291b1cc2d450b11964038bca424814a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 13 Aug 2026 14:02:07 +0100 Subject: [PATCH 098/158] Allow specifying multiple `action_name` and `status` params in the scheduled tasks admin API (#20067) We have an internal usage of `/scheduled_tasks` that would like to fetch multiple actions at once (janitor). We also make it so that invalid `status` values now return a 400 rather than a 500. Co-authored-by: Claude Fable 5 --- changelog.d/20067.feature | 1 + docs/admin_api/scheduled_tasks.md | 4 +- synapse/rest/admin/scheduled_tasks.py | 33 +++++++++++---- tests/rest/admin/test_scheduled_tasks.py | 53 ++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20067.feature diff --git a/changelog.d/20067.feature b/changelog.d/20067.feature new file mode 100644 index 0000000000..123d4df745 --- /dev/null +++ b/changelog.d/20067.feature @@ -0,0 +1 @@ +Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. diff --git a/docs/admin_api/scheduled_tasks.md b/docs/admin_api/scheduled_tasks.md index 949a03ee39..7d7c68d987 100644 --- a/docs/admin_api/scheduled_tasks.md +++ b/docs/admin_api/scheduled_tasks.md @@ -31,8 +31,10 @@ It returns a JSON body like the following: **Query parameters:** * `action_name`: string - Is optional. Returns only the scheduled tasks with the given action name. + May be given multiple times to return tasks matching any of the given action names. * `resource_id`: string - Is optional. Returns only the scheduled tasks with the given resource id. -* `status`: string - Is optional. Returns only the scheduled tasks matching the given status, one of +* `status`: string - Is optional. Returns only the scheduled tasks matching the given status. + May be given multiple times to return tasks matching any of the given statuses. The status must be one of - "scheduled" - Task is scheduled but not active - "active" - Task is active and probably running, and if not will be run on next scheduler loop run - "complete" - Task has completed successfully diff --git a/synapse/rest/admin/scheduled_tasks.py b/synapse/rest/admin/scheduled_tasks.py index 5b3526c7e5..08c7bec783 100644 --- a/synapse/rest/admin/scheduled_tasks.py +++ b/synapse/rest/admin/scheduled_tasks.py @@ -15,7 +15,12 @@ # from typing import TYPE_CHECKING -from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_integer, + parse_string, + parse_strings_from_args, +) from synapse.http.site import SynapseRequest from synapse.rest.admin import admin_patterns, assert_requester_is_admin from synapse.types import JsonDict, TaskStatus @@ -38,19 +43,33 @@ class ScheduledTasksRestServlet(RestServlet): async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: await assert_requester_is_admin(self._auth, request) + # twisted.web.server.Request.args is incorrectly defined as Any | None + args: dict[bytes, list[bytes]] = request.args # type: ignore + # extract query params - action_name = parse_string(request, "action_name") + actions = parse_strings_from_args(args, "action_name") resource_id = parse_string(request, "resource_id") - status = parse_string(request, "status") + status_strings = parse_strings_from_args( + args, + "status", + allowed_values=[status.value for status in TaskStatus], + ) # This parameter was historically called `job_status`, while the Admin API docs # defined it as `status`. We now support both, as `status` is generally # a nicer name. A v2 of this endpoint should keep only `status`. - if status is None: - status = parse_string(request, "job_status") + if status_strings is None: + status_strings = parse_strings_from_args( + args, + "job_status", + allowed_values=[status.value for status in TaskStatus], + ) max_timestamp = parse_integer(request, "max_timestamp") - actions = [action_name] if action_name else None - statuses = [TaskStatus(status)] if status else None + statuses = ( + [TaskStatus(status) for status in status_strings] + if status_strings + else None + ) tasks = await self._store.get_scheduled_tasks( actions=actions, diff --git a/tests/rest/admin/test_scheduled_tasks.py b/tests/rest/admin/test_scheduled_tasks.py index 4b7adb6b89..388570df0b 100644 --- a/tests/rest/admin/test_scheduled_tasks.py +++ b/tests/rest/admin/test_scheduled_tasks.py @@ -190,3 +190,56 @@ class ScheduledTasksAdminApiTestCase(unittest.HomeserverTestCase): # only the task with the matching resource id should have been returned self.assertEqual(len(found_tasks), 1) self.assertEqual(found_tasks[0]["resource_id"], "failed_task") + + def test_filtering_scheduled_tasks_multiple_values(self) -> None: + """ + Test that the `action_name` and `status` filters can be given multiple + times, returning tasks matching any of the given values. + """ + # filter via multiple statuses + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?status=active&status=failed", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + found_tasks = self.check_scheduled_tasks_response( + channel.json_body["scheduled_tasks"] + ) + + # the active and failed tasks should have been returned + self.assertEqual(len(found_tasks), 2) + self.assertEqual({task["status"] for task in found_tasks}, {"active", "failed"}) + + # filter via multiple action names + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?action_name=test_task&action_name=finished_test_task", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + found_tasks = self.check_scheduled_tasks_response( + channel.json_body["scheduled_tasks"] + ) + + # only the tasks with the given action names should have been returned + self.assertEqual(len(found_tasks), 2) + self.assertEqual( + {task["action"] for task in found_tasks}, + {"test_task", "finished_test_task"}, + ) + + def test_filtering_scheduled_tasks_invalid_status(self) -> None: + """ + Test that an invalid `status` value is rejected with a 400 error. + """ + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?status=unknown_status", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) From c78c274f172a05439bd8267eae139773abfdf499 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 13 Aug 2026 16:17:40 +0100 Subject: [PATCH 099/158] Fix the schema diff CI not using `faketime` for SQLite. (#20099) Issue spotted in: https://github.com/element-hq/synapse/pull/20098 Follows: #20027 We already use `faketime` for Postgres, but I forgot that the SQLite schema delta would have the same problem and somehow tuned it out of the preview diff on the original PR. --------- Signed-off-by: Olivier 'reivilibre --- .github/workflows/schema_diff.yml | 9 ++++++--- changelog.d/20099.misc | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20099.misc diff --git a/.github/workflows/schema_diff.yml b/.github/workflows/schema_diff.yml index 0c32ef33f4..3dfc02a469 100644 --- a/.github/workflows/schema_diff.yml +++ b/.github/workflows/schema_diff.yml @@ -30,6 +30,7 @@ jobs: - name: Start postgres with a faked clock background: true id: postgres + # Use faketime here for schema deltas that are wall-clock sensitive under Postgres run: | # Build a docker image with faketime mkdir /tmp/postgres-faketime @@ -58,8 +59,8 @@ jobs: with: fetch-depth: 0 - - name: Install PostgreSQL client - run: sudo apt-get -qq install postgresql-client + - name: Install PostgreSQL client and faketime + run: sudo apt-get -qq install postgresql-client faketime - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -77,8 +78,10 @@ jobs: PGHOST: localhost PGUSER: postgres PGPASSWORD: postgres + # Use faketime here for schema deltas that are wall-clock sensitive under SQLite run: | - poetry run python .ci/scripts/schema_diff.py \ + faketime -f "2001-05-25 12:42:42" \ + poetry run python .ci/scripts/schema_diff.py \ --base origin/develop \ > "${{ runner.temp }}/schema_diff.md" diff --git a/changelog.d/20099.misc b/changelog.d/20099.misc new file mode 100644 index 0000000000..f17b66915b --- /dev/null +++ b/changelog.d/20099.misc @@ -0,0 +1 @@ +Fix the schema diff CI not using `faketime` for SQLite. \ No newline at end of file From c0357de4eda2f32919d89b4cffa306d68562c022 Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Thu, 13 Aug 2026 17:30:51 +0200 Subject: [PATCH 100/158] fix: presence stream stalling intermittently (#20090) This is a fix for presence updates silently stalling when a `/sync` request is cancelled mid-write, causing a stream ID to be leaked into `_unfinished_ids` and permanently pinning the persisted stream position. Fixes https://github.com/element-hq/synapse/issues/19800 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20090.bugfix | 1 + synapse/storage/util/id_generators.py | 42 +++++++++++-- tests/storage/test_id_generators.py | 88 +++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 changelog.d/20090.bugfix diff --git a/changelog.d/20090.bugfix b/changelog.d/20090.bugfix new file mode 100644 index 0000000000..69b8035a44 --- /dev/null +++ b/changelog.d/20090.bugfix @@ -0,0 +1 @@ +Fix a bug where presence updates could stop being sent to clients (the presence stream position becoming stuck) if a `/sync` request was cancelled while a presence write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index c9c339b235..d294119bb2 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -906,14 +906,44 @@ class _MultiWriterCtxManager: stream_ids: list[int] = attr.Factory(list) async def __aenter__(self) -> int | list[int]: + def _load(txn: LoggingTransaction) -> list[int]: + ids = self.id_gen._load_next_mult_id_txn(txn, self.multiple_ids or 1) + # Record the allocated IDs on the context manager as a side effect + # (rather than only via the return value), so that if this coroutine + # is cancelled after the transaction has committed we still know + # which IDs to release below. + self.stream_ids = ids + return ids + # It's safe to run this in autocommit mode as fetching values from a # sequence ignores transaction semantics anyway. - self.stream_ids = await self.id_gen._db.runInteraction( - "_load_next_mult_id", - self.id_gen._load_next_mult_id_txn, - self.multiple_ids or 1, - db_autocommit=True, - ) + try: + await self.id_gen._db.runInteraction( + "_load_next_mult_id", + _load, + db_autocommit=True, + ) + except BaseException: + # We catch `BaseException` rather than `Exception`, + # because request cancellation surfaces here as exceptions that are + # not `Exception` subclasses: `asyncio.CancelledError` + # and `GeneratorExit` (raised when a paused coroutine is garbage + # collected). + # + # If we're interrupted (e.g. the enclosing request was cancelled) + # after the transaction allocated the IDs but before we returned, + # then `__aexit__` will never run, because Python only invokes it + # once `__aenter__` has returned. The allocated IDs would then be + # leaked into `_unfinished_ids` forever, permanently pinning the + # persisted stream position and, e.g., wedging presence. + # + # So mark them as finished here to unblock the position. This mirrors + # what `__aexit__` does on the failure path (marking the IDs finished + # and notifying replication, but not persisting a new position). + if self.stream_ids: + self.id_gen._mark_ids_as_finished(self.stream_ids) + self.notifier.notify_replication() + raise if self.multiple_ids is None: return self.stream_ids[0] * self.id_gen._return_factor diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py index 9a338607ee..a42247b498 100644 --- a/tests/storage/test_id_generators.py +++ b/tests/storage/test_id_generators.py @@ -19,8 +19,12 @@ # # +from unittest import mock + +from twisted.internet.defer import CancelledError, Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor +from synapse.logging.context import LoggingContext, make_deferred_yieldable from synapse.server import HomeServer from synapse.storage.database import ( DatabasePool, @@ -225,6 +229,90 @@ class MultiWriterIdGeneratorTestCase(MultiWriterIdGeneratorBase): self.assertEqual(id_gen.get_positions(), {"master": 8}) self.assertEqual(id_gen.get_current_token_for_writer("master"), 8) + def test_cancelled_enter_does_not_wedge_position(self) -> None: + """Reproduces presence getting stuck. + + If the `get_next()` async context manager is cancelled while + `__aenter__` is allocating a stream ID, the DB interaction that runs the + sequence has already added the ID to `_unfinished_ids`, but `__aexit__` + is never called (Python only invokes `__aexit__` if `__aenter__` + returned). The abandoned ID is therefore leaked into `_unfinished_ids` + forever, which permanently pins the persisted stream position: new rows + keep getting higher IDs, but `get_current_token()` can never advance past + `leaked_id - 1` until the process restarts. + + This mirrors a `/sync` request being cancelled part-way through + persisting a presence update. `/sync` became `@cancellable` in #19499, + and on a monolith the presence write in `PresenceStore.update_presence` + is awaited inside that cancellable request scope. + """ + # Prefill table with 7 rows written by 'master'; position starts at 7. + self._insert_rows("master", 7) + + id_gen = self._create_id_generator() + self.assertEqual(id_gen.get_current_token_for_writer("master"), 7) + + # We model the cancellation at the seam it actually happens in + # production: `__aenter__` awaits `runInteraction("_load_next_mult_id")`, + # whose transaction runs in a thread pool and so *always* completes - + # allocating stream ID 8 and adding it to `_unfinished_ids` - but the + # awaiting coroutine is handed a `CancelledError` because the enclosing + # `/sync` request was cancelled. We reproduce that by letting the real + # interaction run (applying its side effects) and then failing the + # awaited deferred with `CancelledError`. + cancel_enter: "Deferred[None]" = Deferred() + original_run_interaction = id_gen._db.runInteraction + + async def blocking_run_interaction(desc, func, *args, **kwargs): # type: ignore[no-untyped-def] + result = await original_run_interaction(desc, func, *args, **kwargs) + if desc == "_load_next_mult_id": + # Stream ID 8 is now allocated and recorded in `_unfinished_ids`. + # Deliver the cancellation here, exactly as a cancelled `/sync` + # would land it on this `await`. + await make_deferred_yieldable(cancel_enter) + return result + + async def presence_like_write() -> None: + # Mirrors `PresenceStore.update_presence`: allocate an ID and + # "persist" under the context manager. + with LoggingContext(name="sync", server_name=self.hs.hostname): + async with id_gen.get_next(): + pass + + with mock.patch.object( + id_gen._db, "runInteraction", new=blocking_run_interaction + ): + write = ensureDeferred(presence_like_write()) + + # The write is now blocked inside `__aenter__`, i.e. after stream ID + # 8 has been allocated and added to `_unfinished_ids`. + self.assertNoResult(write) + + # The client goes away and the `/sync` request is cancelled. + cancel_enter.errback(CancelledError()) + + # The cancellation must surface as a `CancelledError`. + self.get_failure(write, CancelledError) + + # The cancelled write never persisted a row for ID 8, so the generator + # must not let that abandoned ID wedge the position. A subsequent + # *successful* write should be able to advance the persisted token. + async def _successful_write() -> None: + async with id_gen.get_next(): + pass + + self.get_success(_successful_write()) + + # On the buggy code the token is still stuck at 7 (ID 8 is leaked in + # `_unfinished_ids`, blocking everything behind it). Once the leak is + # fixed, the token advances to 9: ID 8 was allocated (and abandoned) by + # the cancelled write, so the successful write above takes ID 9. + self.assertEqual( + id_gen.get_current_token_for_writer("master"), + 9, + "presence stream position is wedged by the cancelled allocation", + ) + def test_out_of_order_finish(self) -> None: """Test that IDs persisted out of order are correctly handled""" From 5a782430ec67e45dbb2c2c035f8db33aa4855179 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 17 Aug 2026 12:47:35 +0100 Subject: [PATCH 101/158] Fix the schema diff CI breaking when the Rust module was changed. (#20117) Follows: #20027 Noticed in https://github.com/element-hq/synapse/pull/20003 The problem was that `--no-root` prevents reinstalling the root project (Synapse). However, since we just did a `git checkout`, we need to reinstall the root project in case the Rust code changed, as the `poetry install` command is what causes the Rust to be recompiled. Doing otherwise causes the 'Rust module outdated' error when importing `synapse`. --------- Signed-off-by: Olivier 'reivilibre --- .ci/scripts/schema_diff.py | 2 +- changelog.d/20117.misc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20117.misc diff --git a/.ci/scripts/schema_diff.py b/.ci/scripts/schema_diff.py index 9b7af72434..8354b64233 100755 --- a/.ci/scripts/schema_diff.py +++ b/.ci/scripts/schema_diff.py @@ -193,7 +193,7 @@ def main() -> None: # Refresh dependencies print("Installing dependencies for base commit...", file=sys.stderr) subprocess.run( - ["poetry", "install", "--no-root", "--extras", "postgres"], + ["poetry", "install", "--extras", "postgres"], cwd=REPO_ROOT, check=True, # Poetry install is noisy, so pipe its stdout to stderr diff --git a/changelog.d/20117.misc b/changelog.d/20117.misc new file mode 100644 index 0000000000..92d421117e --- /dev/null +++ b/changelog.d/20117.misc @@ -0,0 +1 @@ +Fix the schema diff CI breaking when the Rust module was changed. \ No newline at end of file From 94a5f2afb36a4afdc36813fd1c24b9a1c4aec252 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Aug 2026 12:52:43 +0100 Subject: [PATCH 102/158] Speed up marking device list changes as converted (#20098) The conversion of `device_lists_changes_in_room` rows into `device_lists_outbound_pokes` is DB-bound on `mark_redundant_device_lists_pokes`. The `UPDATE` uses the `(room_id, stream_id)` index, so each call scans the unconverted backlog, getting slower the further behind the conversion is. Add a partial index matching the query via a background update, and skip the (safe-to-skip) `UPDATE` until the index has been built. Also add a metric reporting how far behind the conversion is, using the existing `inserted_ts` column. Fixes https://github.com/element-hq/backend-internal/issues/286 --------- Co-authored-by: Claude Fable 5 Co-authored-by: Eric Eastwood --- changelog.d/20098.misc | 1 + synapse/handlers/device.py | 54 ++++++++++++ synapse/storage/background_updates.py | 16 ++++ synapse/storage/databases/main/devices.py | 86 ++++++++++++++++++- ..._lists_changes_in_room_unconverted_idx.sql | 22 +++++ 5 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20098.misc create mode 100644 synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql diff --git a/changelog.d/20098.misc b/changelog.d/20098.misc new file mode 100644 index 0000000000..0504be872d --- /dev/null +++ b/changelog.d/20098.misc @@ -0,0 +1 @@ +Speed up the conversion of device list changes into outbound federation pokes, and add a metric for how far behind the conversion is. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 2225466648..ae61bc24bf 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -30,6 +30,8 @@ from typing import ( cast, ) +from prometheus_client import Gauge + from synapse.api import errors from synapse.api.constants import EduTypes, EventTypes, Membership from synapse.api.errors import ( @@ -41,6 +43,7 @@ from synapse.api.errors import ( SynapseError, ) from synapse.logging.opentracing import log_kv, set_tag, trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( wrap_as_background_process, ) @@ -89,6 +92,21 @@ DELETE_DEVICE_MSGS_TASK_NAME = "delete_device_messages" MAX_DEVICE_DISPLAY_NAME_LEN = 100 DELETE_STALE_DEVICES_INTERVAL = Duration(days=1) +device_list_conversion_lag_gauge = Gauge( + "synapse_device_lists_changes_conversion_lag_seconds", + "Age of the oldest device list change that has yet to be converted to outbound federation pokes", + labelnames=[SERVER_NAME_LABEL], +) + +device_list_conversion_stream_lag_gauge = Gauge( + "synapse_device_lists_changes_conversion_stream_lag", + "Number of stream IDs between the current device lists stream position and the position converted to outbound federation pokes", + labelnames=[SERVER_NAME_LABEL], +) + +# How often to update the device list conversion lag gauges. +DEVICE_LIST_CONVERSION_LAG_GAUGE_METRIC_UPDATE_INTERVAL = Duration(seconds=30) + def _check_device_name_length(name: str | None) -> None: """ @@ -960,6 +978,13 @@ class DeviceWriterHandler(DeviceHandler): self.device_list_updater.incoming_device_list_update, ) + # Report how far behind we are at converting device list changes + # into outbound pokes. + self.clock.looping_call( + self._report_device_list_conversion_lag, + DEVICE_LIST_CONVERSION_LAG_GAUGE_METRIC_UPDATE_INTERVAL, + ) + @trace @measure_func("notify_device_update") async def notify_device_update( @@ -1033,6 +1058,35 @@ class DeviceWriterHandler(DeviceHandler): self._handle_new_device_update_async() return + @wrap_as_background_process("_report_device_list_conversion_lag") + async def _report_device_list_conversion_lag(self) -> None: + """Report how far behind we are at converting rows in + `device_lists_changes_in_room` to `device_lists_outbound_pokes`. + """ + ( + oldest_ts, + last_converted_pos, + ) = await self.store.get_device_list_conversion_lag() + + if oldest_ts is None: + device_list_conversion_lag_ms = 0 + else: + device_list_conversion_lag_ms = max(0, self.clock.time_msec() - oldest_ts) + + device_list_conversion_lag_gauge.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(device_list_conversion_lag_ms / 1000.0) # convert to seconds + + # The stream ID lag is only an approximation of the conversion + # backlog: the converted position only advances when the conversion + # loop runs, and stream IDs in the gap may not have rows needing + # conversion at all. + current_pos = self.store.get_device_stream_token().stream + + device_list_conversion_stream_lag_gauge.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(max(0, current_pos - last_converted_pos)) + @wrap_as_background_process("_handle_new_device_update_async") async def _handle_new_device_update_async(self) -> None: """Called when we have a new local device list update that we need to diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 311534c5e7..8137b02036 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -262,6 +262,10 @@ class BackgroundUpdater: # enum? self._all_done = False + # A set of background updates that we have queried the database for and + # found to be completed. + self._completed_background_updates: set[str] = set() + # Whether we're currently running updates self._running = False @@ -394,9 +398,15 @@ class BackgroundUpdater: return perf def start_doing_background_updates(self) -> None: + """Start doing background updates in the background. + + This gets called both on startup and when the admin API is used to + reschedule background updates. + """ if self.enabled: # if we start a new background update, not all updates are done. self._all_done = False + self._completed_background_updates.clear() sleep = self.sleep_enabled self.hs.run_as_background_process( "background_updates", @@ -478,6 +488,9 @@ class BackgroundUpdater: if update_name == self._current_background_update: return False + if update_name in self._completed_background_updates: + return True + update_exists = await self.db_pool.simple_select_one_onecol( "background_updates", keyvalues={"update_name": update_name}, @@ -486,6 +499,9 @@ class BackgroundUpdater: allow_none=True, ) + if not update_exists: + self._completed_background_updates.add(update_name) + return not update_exists async def have_completed_background_updates( diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 0e4c8ac491..69470ae6e6 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -83,6 +83,10 @@ BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES = "remove_dup_outbound_pokes" # `device_lists_changes_in_room.inserted_ts`. BG_UPDATE_ADD_INSERTED_TS_INDEX = "device_lists_changes_in_room_inserted_ts_idx" +# Background update name for adding an index on unconverted rows in +# `device_lists_changes_in_room`. +BG_UPDATE_ADD_UNCONVERTED_IDX = "device_lists_changes_in_room_unconverted_idx" + # Prunes entries out of the `device_lists_changes_in_room` table that are more # than this old. @@ -2204,7 +2208,22 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): converted_upto_stream_id: int, ) -> None: """If we've calculated the outbound pokes for a given room/device list - update, mark any subsequent changes as already converted""" + update, mark any subsequent changes as already converted. + + This is an optimization only. Skipping it is always safe, and just + means the subsequent changes get converted individually. + """ + + # Without the index added by `BG_UPDATE_ADD_UNCONVERTED_IDX`, the + # UPDATE below scans the unconverted backlog on every call, getting + # slower the further behind we are. Skip it until the index exists. + unconverted_idx_ready = ( + await self.db_pool.updates.has_completed_background_update( + BG_UPDATE_ADD_UNCONVERTED_IDX + ) + ) + if not unconverted_idx_ready: + return sql = """ UPDATE device_lists_changes_in_room @@ -2446,17 +2465,69 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): `FALSE` have not been converted. """ + return await self.db_pool.runInteraction( + desc="get_device_change_last_converted_pos", + func=self.get_device_change_last_converted_pos_txn, + db_autocommit=True, + ) + + def get_device_change_last_converted_pos_txn( + self, txn: LoggingTransaction + ) -> tuple[int, str]: + """Get the position of the last row in `device_list_changes_in_room` that has been + converted to `device_lists_outbound_pokes`. + + Rows with a strictly greater position where `converted_to_destinations` is + `FALSE` have not been converted.""" + # There should be only one row in this table, though we want to # future-proof ourselves for when we have multiple rows (one for each # instance). So to handle that case we take the minimum of all rows. - rows = await self.db_pool.simple_select_list( + rows = self.db_pool.simple_select_list_txn( + txn, table="device_lists_changes_converted_stream_position", keyvalues={}, retcols=["stream_id", "room_id"], - desc="get_device_change_last_converted_pos", ) return cast(tuple[int, str], min(rows)) + async def get_device_list_conversion_lag(self) -> tuple[int | None, int]: + """Get how far behind we are at converting rows in + `device_lists_changes_in_room` to `device_lists_outbound_pokes`. + + Returns: + A tuple of: + 1. the timestamp (ms) at which the oldest unconverted change + was inserted. None if there is nothing to convert, or if + the oldest row predates the `inserted_ts` column. + 2. the stream ID of the last converted position. + """ + + # Rows for one device list update share a `stream_id` (and insertion + # time), so ordering by `stream_id` alone is fine. + sql = """ + SELECT inserted_ts FROM device_lists_changes_in_room + WHERE + (stream_id, room_id) > (?, ?) AND + NOT converted_to_destinations + ORDER BY stream_id ASC + LIMIT 1 + """ + + def get_device_list_conversion_lag_txn( + txn: LoggingTransaction, + ) -> tuple[int | None, int]: + stream_id, room_id = self.get_device_change_last_converted_pos_txn(txn) + + txn.execute(sql, (stream_id, room_id)) + row = txn.fetchone() + return row[0] if row else None, stream_id + + return await self.db_pool.runInteraction( + "get_device_list_conversion_lag", + get_device_list_conversion_lag_txn, + ) + async def set_device_change_last_converted_pos( self, stream_id: int, @@ -2699,6 +2770,15 @@ class DeviceBackgroundUpdateStore(SQLBaseStore): where_clause="inserted_ts IS NOT NULL", ) + # Add an index to speed up `mark_redundant_device_lists_pokes`. + self.db_pool.updates.register_background_index_update( + BG_UPDATE_ADD_UNCONVERTED_IDX, + index_name="device_lists_changes_in_room_unconverted_idx", + table="device_lists_changes_in_room", + columns=["user_id", "device_id", "room_id", "stream_id"], + where_clause="NOT converted_to_destinations", + ) + async def _drop_device_list_streams_non_unique_indexes( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql b/synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql new file mode 100644 index 0000000000..6bebcab11d --- /dev/null +++ b/synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql @@ -0,0 +1,22 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Add an index on `device_lists_changes_in_room(user_id, device_id, room_id, +-- stream_id)` for unconverted rows, so that marking redundant rows as +-- converted (in `mark_redundant_device_lists_pokes`) does not require a scan +-- of the unconverted backlog. +-- +-- This is a partial index as we only ever query for unconverted rows. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9408, 'device_lists_changes_in_room_unconverted_idx', '{}'); From ba3ab1876acd326fa5cc9b161e87b064ac54f3ed Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Aug 2026 03:27:50 -0500 Subject: [PATCH 103/158] Optimize `mark_as_sent_devices_by_remote` to do less CPU work on the database (#20120) ### Background On `matrix.org`, since 2026-08-15, we are seeing the database CPU being saturated more than usual ([grafana](https://grafana.matrix.org/d/rYdddlPWk/node-exporter?orgId=1&from=2026-08-10T23:23:14.237Z&to=2026-08-17T23:23:14.237Z&timezone=browser&var-DS_PROMETHEUS=default&var-job=machine&var-node=matrix-db-01.matrix.org:9100&var-diskdevices=%5Ba-z%5D%2B%7Cnvme%5B0-9%5D%2Bn%5B0-9%5D%2B&refresh=1m&viewPanel=panel-77)) CPU of database server @reivilibre [found](https://matrix.to/#/!yHWhpxlXVaLcsgDUKb:matrix.org/$fUMf60IbFocpEuJNbmbEzFiCyKCFnAl4I4XpB27DUQs?via=banzan.uk&via=element.io&via=matrix.org) `mark_as_sent_devices_by_remote` spiking in the `DB transactions by total txn time` graph ([grafana](https://grafana.matrix.org/d/000000012/synapse?var-bucket_size=$__auto&orgId=1&from=2026-08-10T22:26:07.336Z&to=2026-08-17T22:26:07.336Z&timezone=browser&var-datasource=default&var-instance=matrix.org&var-job=synapse_federation_sender&var-index=$__all&showCategory=Thresholds&viewPanel=panel-11)) 'mark_as_sent_devices_by_remote' spiking in the 'DB transactions by total txn time' graph And we indeed see a bunch of time being spent on `mark_as_sent_devices_by_remote` by looking at `pg_stat_statements`. The top two queries we're spending CPU on are the `SELECT` and `DELETE` statements in [`mark_as_sent_devices_by_remote`](https://github.com/element-hq/synapse/blob/94a5f2afb36a4afdc36813fd1c24b9a1c4aec252/synapse/storage/databases/main/devices.py#L922-L950). We also did some related work in this area recently with https://github.com/element-hq/synapse/pull/20098 although it was tackling a different bottle-neck. ### This PR This PRs combines the two separate `SELECT` and `DELETE` queries which touch the same data into one `DELETE ... RETURNING ...` query. this means we get to save the cost of one of those statements (less CPU on the database) and fewer statements per transaction (less round-trips) means connections turn over faster, and can move on to process the next thing. This is a micro-optimization I spotted while reading [`_mark_as_sent_devices_by_remote_txn`](https://github.com/element-hq/synapse/blob/develop/synapse/storage/databases/main/devices.py#L922-L950) rather than a structural fix. I'm sure there are even better things to do to where we could even avoid this kind of work altogether but this seemed like a quick win especially given how much this particular transaction is saturating the database. Doing things faster doesn't necessarily mean we solve the saturated/starved CPU problem but it does mean the same task costs less CPU. From the `pg_stat_statements` samples above, we can expect to be up to ~27% more efficient with database CPU on this code path (derived from the `cpus_busy` numbers above `22.932 / (22.932 + 61.956)`). We're removing the `SELECT` (`61.956` `cpus_busy`) but I still expect the `DELETE` (`22.932` `cpus_busy`) to take on a similar cost as I'm sure it's a warm cache situation between them. --- changelog.d/20120.misc | 1 + synapse/storage/databases/main/devices.py | 55 ++++++++++++++++------- 2 files changed, 40 insertions(+), 16 deletions(-) create mode 100644 changelog.d/20120.misc diff --git a/changelog.d/20120.misc b/changelog.d/20120.misc new file mode 100644 index 0000000000..d59db030c6 --- /dev/null +++ b/changelog.d/20120.misc @@ -0,0 +1 @@ +Reduce database CPU usage when marking device list changes as sent over federation. diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 69470ae6e6..6cf9270ff2 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -922,33 +922,56 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): def _mark_as_sent_devices_by_remote_txn( self, txn: LoggingTransaction, destination: str, stream_id: int ) -> None: - # We update the device_lists_outbound_last_success with the successfully - # poked users. + # Delete all sent outbound pokes, returning them so that we can update + # `device_lists_outbound_last_success` with the successfully poked users. + # + # This is a high frequency transaction (runs very often when processing a + # backlog of device list changes) and can bog down the database CPU with the + # sheer number of statements. + # + # We prefer to trade a little bit of processing time on the Python side + # (aggregating `max_stream_id_by_user_id`) as the alternative would be to have + # two separate queries; a `SELECT ... GROUP BY user_id` with the aggregation and + # then a `DELETE` which means we touch the same rows twice. We get to save the + # cost of one of those statements (less CPU on the database) and fewer + # statements per transaction (less round-trips) means connections turn over + # faster, and can move on to process the next thing. + # + # By the nature of `MAX_EDUS_PER_TRANSACTION`, we're only dealing with 100 rows + # at max which is pretty trivial for us to process on the Python side. sql = """ - SELECT user_id, coalesce(max(o.stream_id), 0) - FROM device_lists_outbound_pokes as o - WHERE destination = ? AND o.stream_id <= ? - GROUP BY user_id + DELETE FROM device_lists_outbound_pokes + WHERE destination = ? AND stream_id <= ? + RETURNING user_id, stream_id """ txn.execute(sql, (destination, stream_id)) - rows = txn.fetchall() + # Aggregate `max_stream_id_by_user_id` + max_stream_id_by_user_id: dict[str, int] = {} + for user_id, poke_stream_id in txn: + max_stream_id_by_user_id[user_id] = max( + max_stream_id_by_user_id.get(user_id, 0), poke_stream_id + ) + + # Update `device_lists_outbound_last_success` with the successfully poked + # users. + # + # We could potentially combine this in one big CTE with the query above but it + # isn't supported by SQLite (SQLite doesn't support `DELETE` in a CTE). self.db_pool.simple_upsert_many_txn( txn=txn, table="device_lists_outbound_last_success", key_names=("destination", "user_id"), - key_values=[(destination, user_id) for user_id, _ in rows], + key_values=[ + (destination, user_id) for user_id in max_stream_id_by_user_id.keys() + ], value_names=("stream_id",), - value_values=[(stream_id,) for _, stream_id in rows], + value_values=[ + (user_stream_id,) + for user_stream_id in max_stream_id_by_user_id.values() + ], ) - # Delete all sent outbound pokes - sql = """ - DELETE FROM device_lists_outbound_pokes - WHERE destination = ? AND stream_id <= ? - """ - txn.execute(sql, (destination, stream_id)) - async def add_user_signature_change_to_streams( self, from_user_id: str, user_ids: list[str] ) -> int: From ae30e4fb3caba82a7faf38ae0ec51ed8bdd4c26a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Aug 2026 11:07:08 -0500 Subject: [PATCH 104/158] 1.159.0 --- CHANGES.md | 7 ++++++- debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c1c9c3293d..9e714bbef4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,14 @@ -# Synapse 1.159.0rc1 (2026-08-11) +# Synapse 1.159.0 (2026-08-18) + +No significant changes since 1.159.0rc1. Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.159/docs/upgrade.md#upgrading-to-v11590) as we have recently updated the expiry date on the repository's GPG signing key. The old version of the key will expire on `2027-03-15`. + +# Synapse 1.159.0rc1 (2026-08-11) + ## Features - Add optional support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). diff --git a/debian/changelog b/debian/changelog index 53cca6e378..8a6da09864 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.159.0) stable; urgency=medium + + * New synapse release 1.159.0. + + -- Synapse Packaging team Tue, 18 Aug 2026 16:02:56 +0000 + matrix-synapse-py3 (1.159.0~rc1) stable; urgency=medium * New synapse release 1.159.0rc1. diff --git a/pyproject.toml b/pyproject.toml index b03bbacd16..8e84e6d37d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.159.0rc1" +version = "1.159.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 7b10e6b9bc2dacc33f0974c999f640b55ef831bc Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Aug 2026 11:08:10 -0500 Subject: [PATCH 105/158] Move package notice back --- CHANGES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9e714bbef4..51c41cddb7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,13 +2,13 @@ No significant changes since 1.159.0rc1. + +# Synapse 1.159.0rc1 (2026-08-11) + Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.159/docs/upgrade.md#upgrading-to-v11590) as we have recently updated the expiry date on the repository's GPG signing key. The old version of the key will expire on `2027-03-15`. - -# Synapse 1.159.0rc1 (2026-08-11) - ## Features - Add optional support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). From 11de9503fb053dcc2ad47b8038a4b98631842950 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 20 Aug 2026 07:49:03 -0400 Subject: [PATCH 106/158] Update Rust dependencies for security fixes (#20131) --- Cargo.lock | 29 ++++++++++++++--------------- changelog.d/20131.misc | 1 + changelog.d/20131.misc.1 | 1 + rust/Cargo.toml | 6 +++--- rust/src/deferred.rs | 2 +- rust/src/storage/db/mod.rs | 3 ++- 6 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 changelog.d/20131.misc create mode 100644 changelog.d/20131.misc.1 diff --git a/Cargo.lock b/Cargo.lock index 50daff2dd9..3cb739d495 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -843,9 +843,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "anyhow", "bytes", @@ -859,18 +859,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -889,9 +889,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -901,22 +901,21 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn 2.0.104", ] [[package]] name = "pythonize" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b79f670c9626c8b651c0581011b57b6ba6970bb69faf01a7c4c0cfc81c43f95" +checksum = "6ec376e1216e0c929a74964ce2020012a1a39f32d80e78aa688721219ea7fb89" dependencies = [ "pyo3", "serde", @@ -1193,9 +1192,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" dependencies = [ "ring", "rustls-pki-types", diff --git a/changelog.d/20131.misc b/changelog.d/20131.misc new file mode 100644 index 0000000000..a90fd7d4ad --- /dev/null +++ b/changelog.d/20131.misc @@ -0,0 +1 @@ +Update rustls-webpki to address GHSA-82j2-j2ch-gfr8. diff --git a/changelog.d/20131.misc.1 b/changelog.d/20131.misc.1 new file mode 100644 index 0000000000..c0c211ed8c --- /dev/null +++ b/changelog.d/20131.misc.1 @@ -0,0 +1 @@ +Update pyo3 to address GHSA-36hh-v3qg-5jq4 and GHSA-chgr-c6px-7xpp. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 612ab09f6d..f6b12c4c1b 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -31,7 +31,7 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.28.3", features = [ +pyo3 = { version = "0.29.0", features = [ "macros", "anyhow", "abi3", @@ -40,8 +40,8 @@ pyo3 = { version = "0.28.3", features = [ # https://docs.rs/pyo3/latest/pyo3/bytes/index.html "bytes", ] } -pyo3-log = "0.13.3" -pythonize = { version = "0.28.0", features = ["arbitrary_precision"] } +pyo3-log = "0.13.4" +pythonize = { version = "0.29.0", features = ["arbitrary_precision"] } regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive", "rc"] } diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 62a1ce6b90..3da5defb83 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -141,7 +141,7 @@ pub(crate) async fn run_python_awaitable( make_awaitable: F, ) -> PyResult> where - F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, + F: for<'py> Fn(Python<'py>) -> PyResult> + Send + Sync + 'static, { // Resolves when the awaitable completes; carries the resolved value or error. let (tx, rx) = oneshot::channel::>>(); diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index fd40d52f08..175f66e76d 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -32,7 +32,7 @@ pub mod python_db_pool; /// It may be invoked multiple times under certain failure modes (serialization /// and deadlock errors), so it is `Fn` rather than `FnOnce`. pub type ErasedInteraction = - Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send>; + Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send + Sync>; /// The type-erased *result* of an [`ErasedInteraction`] /// [`DatabasePool::run_interaction_erased`]. @@ -114,6 +114,7 @@ pub trait DatabasePoolExt: DatabasePool { R: Send + 'static, F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + Send + + Sync + 'static, { // Erase the concrete return type `R` into `Box` so we can call From d0b105757ed706c2a2cd543583561f4990b8345e Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Thu, 20 Aug 2026 16:27:11 +0200 Subject: [PATCH 107/158] Add experimental support for MSC4502: Targeted and unrestricted room member queries (#19974) This is another stopgap towards https://github.com/element-hq/voip-internal/issues/641 and adds experimental support for [MSC4502](https://github.com/matrix-org/matrix-spec-proposals/pull/4502). This adds a new endpoint `/rooms/{roomId}/is_joined` to query if a user or server is joined to a room known to the homeserver. Access to the endpoint is guarded behind a new OAuth scope that can be assigned when registering application services. Tracking issue: https://github.com/element-hq/synapse/issues/20118 --------- Signed-off-by: Johannes Marbach --- changelog.d/19974.feature | 1 + rust/src/config/mod.rs | 1 + rust/src/handlers/versions.rs | 4 + synapse/api/auth/__init__.py | 8 +- synapse/api/auth/base.py | 18 +- synapse/appservice/__init__.py | 17 ++ synapse/config/appservice.py | 9 + synapse/config/experimental.py | 3 + synapse/rest/__init__.py | 2 + synapse/rest/client/room_membership.py | 103 ++++++++++ tests/appservice/test_appservice.py | 36 +++- tests/rest/client/test_room_membership.py | 235 ++++++++++++++++++++++ tests/rest/client/test_versions.py | 11 + tests/storage/test_appservice.py | 80 +++++++- 14 files changed, 520 insertions(+), 8 deletions(-) create mode 100644 changelog.d/19974.feature create mode 100644 synapse/rest/client/room_membership.py create mode 100644 tests/rest/client/test_room_membership.py diff --git a/changelog.d/19974.feature b/changelog.d/19974.feature new file mode 100644 index 0000000000..4a9290ea76 --- /dev/null +++ b/changelog.d/19974.feature @@ -0,0 +1 @@ +Add experimental support for MSC4502: Targeted and unrestricted room member queries. diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index 1c97373c27..db00197107 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -74,4 +74,5 @@ pub struct ExperimentalConfig { pub msc4491_enabled: bool, pub msc4143_enabled: bool, pub msc4446_enabled: bool, + pub msc4502_enabled: bool, } diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 5d35b052bd..dc9e55e349 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -272,6 +272,9 @@ pub struct UnstableFeatureMap { /// MSC4446: Allow moving the fully read marker backwards. #[serde(rename = "com.beeper.msc4446")] msc4446_enabled: bool, + /// MSC4502: Targeted and unrestricted room member queries + #[serde(rename = "io.element.msc4502")] + msc4502: bool, // Whether new rooms will be set to encrypted or not (based on presets). #[serde(rename = "io.element.e2ee_forced.public")] @@ -324,6 +327,7 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4491_enabled: config.experimental.msc4491_enabled, msc4143_enabled: config.experimental.msc4143_enabled, msc4446_enabled: config.experimental.msc4446_enabled, + msc4502: config.experimental.msc4502_enabled, e2ee_forced_public: config .room .encryption_enabled_by_default_for_room_presets diff --git a/synapse/api/auth/__init__.py b/synapse/api/auth/__init__.py index 201c295f06..d8d3b31b9d 100644 --- a/synapse/api/auth/__init__.py +++ b/synapse/api/auth/__init__.py @@ -24,7 +24,7 @@ from prometheus_client import Histogram from twisted.web.server import Request -from synapse.appservice import ApplicationService +from synapse.appservice import ApplicationService, Scopes from synapse.http.site import SynapseRequest from synapse.metrics import SERVER_NAME_LABEL from synapse.types import Requester @@ -205,3 +205,9 @@ class Auth(Protocol): membership event ID of the user. If the user is not in the room and never has been, then `(Membership.JOIN, None)` is returned. """ + + def assert_requester_has_scope(self, requester: Requester, scope: Scopes) -> None: + """Asserts that the requester has the given scope, either directly + (e.g. via an OAuth token) or via the scopes registered against the + application service. + """ diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index 14e76b0cff..0b10640b50 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -19,6 +19,7 @@ # # import logging +from http import HTTPStatus from typing import TYPE_CHECKING from netaddr import IPAddress @@ -33,7 +34,7 @@ from synapse.api.errors import ( MissingClientTokenError, UnstableSpecAuthError, ) -from synapse.appservice import ApplicationService +from synapse.appservice import ApplicationService, Scopes from synapse.http import get_request_user_agent from synapse.http.site import SynapseRequest from synapse.logging.opentracing import trace @@ -362,6 +363,21 @@ class BaseAuth: effective_user_id, app_service=app_service, device_id=effective_device_id ) + def assert_requester_has_scope(self, requester: Requester, scope: Scopes) -> None: + """Asserts that the requester has the given scope, either directly + (e.g. via an OAuth token) or via the scopes registered against the + application service. + """ + if scope in requester.scope: + return + + if requester.app_service_id is not None: + app_service = self.store.get_app_service_by_id(requester.app_service_id) + if app_service is not None and app_service.has_scope(scope): + return + + raise AuthError(HTTPStatus.FORBIDDEN, f"Missing {scope} scope") + async def _record_request( self, request: SynapseRequest, requester: Requester ) -> None: diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index c55a83a879..19fff7f009 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -63,6 +63,14 @@ TransactionOneTimeKeysCount = dict[str, dict[str, dict[str, int]]] TransactionUnusedFallbackKeys = dict[str, dict[str, list[str]]] +class Scopes(str, Enum): + """ + All known scopes assignable to application services for extended privileges. + """ + + QUERY_ROOM_MEMBERSHIP = "urn:matrix:client:io.element.msc4502:rooms:is_joined" + + class ApplicationServiceState(Enum): DOWN = "down" UP = "up" @@ -104,6 +112,7 @@ class ApplicationService: supports_unstable_ephemeral: bool = False, msc3202_transaction_extensions: bool = False, msc4190_device_management: bool = False, + scopes: Iterable[str] = frozenset(), ): self.token = token self.url = ( @@ -140,6 +149,11 @@ class ApplicationService: else: self.protocols = set() + self.scopes = set(scopes) + unknown_scopes = self.scopes - frozenset(Scopes) + if unknown_scopes: + raise ValueError(f"Unknown application service scope(s): {unknown_scopes}") + self.rate_limited = rate_limited def _check_namespaces( @@ -379,6 +393,9 @@ class ApplicationService: def is_interested_in_protocol(self, protocol: str) -> bool: return protocol in self.protocols + def has_scope(self, scope: Scopes) -> bool: + return scope in self.scopes + def is_exclusive_alias(self, alias: str) -> bool: return self._is_exclusive(ApplicationService.NS_ALIASES, alias) diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 7a629d10bf..4e61ef694f 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -199,6 +199,14 @@ def _load_appservice( "The `io.element.msc4190` option should be true or false if specified." ) + # Opt-in list of scopes granted to this appservice for restricted C-S API + # functionality. + scopes = as_info.get("io.element.msc4502.scopes", []) + if not isinstance(scopes, list) or not all(isinstance(s, str) for s in scopes): + raise ValueError( + "The `io.element.msc4502.scopes` option should be a list of strings if specified." + ) + return ApplicationService( token=as_info["as_token"], url=as_info["url"], @@ -213,4 +221,5 @@ def _load_appservice( supports_ephemeral=supports_ephemeral, msc3202_transaction_extensions=msc3202_transaction_extensions, msc4190_device_management=msc4190_enabled, + scopes=scopes, ) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 6ad9f53517..97dc803ab7 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -203,6 +203,9 @@ class ExperimentalConfig(Config): # See https://github.com/element-hq/synapse/issues/19524 self.msc4370_enabled = experimental.get("msc4370_enabled", False) + # MSC4502: Targeted and unrestricted room member queries + self.msc4502_enabled: bool = experimental.get("msc4502_enabled", False) + auth_delegated = (config.get("matrix_authentication_service") or {}).get( "enabled", False ) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index a56a81a8e9..c8ede662aa 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -60,6 +60,7 @@ from synapse.rest.client import ( retention, room, room_keys, + room_membership, room_upgrade_rest_servlet, sendtodevice, sync, @@ -128,6 +129,7 @@ CLIENT_SERVLET_FUNCTIONS: tuple[RegisterServletsFunc, ...] = ( rendezvous.register_servlets, auth_metadata.register_servlets, thread_subscriptions.register_servlets, + room_membership.register_servlets, ) SERVLET_GROUPS: dict[str, Iterable[RegisterServletsFunc]] = { diff --git a/synapse/rest/client/room_membership.py b/synapse/rest/client/room_membership.py new file mode 100644 index 0000000000..026bee8214 --- /dev/null +++ b/synapse/rest/client/room_membership.py @@ -0,0 +1,103 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING + +from synapse.api.constants import EventTypes, Membership +from synapse.api.errors import Codes, SynapseError +from synapse.appservice import Scopes +from synapse.http.server import HttpServer +from synapse.http.servlet import RestServlet, parse_string +from synapse.http.site import SynapseRequest +from synapse.rest.client._base import client_patterns +from synapse.types import JsonDict, RoomID, UserID +from synapse.util.stringutils import parse_and_validate_server_name + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class AppserviceRoomMembershipRestServlet(RestServlet): + PATTERNS = client_patterns( + r"/io\.element\.msc4502/rooms/(?P[^/]*)/is_joined$", releases=() + ) + CATEGORY = "Client API requests" + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self.store = hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + self.is_mine_id = hs.is_mine_id + + async def on_GET( + self, request: SynapseRequest, room_id: str + ) -> tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request, allow_guest=False) + self.auth.assert_requester_has_scope(requester, Scopes.QUERY_ROOM_MEMBERSHIP) + + if not RoomID.is_valid(room_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid room ID", Codes.INVALID_PARAM + ) + + mxid = parse_string(request, "mxid") + server_name = parse_string(request, "server_name") + + if (mxid is None) == (server_name is None): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Exactly one of 'mxid' or 'server_name' query parameters must be given", + Codes.MISSING_PARAM, + ) + + if mxid is not None: + if not UserID.is_valid(mxid): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"Invalid MXID: {mxid}", + Codes.INVALID_PARAM, + ) + if self.is_mine_id(mxid): + joined = await self.store.check_local_user_in_room(mxid, room_id) + else: + event = await self.storage_controllers.state.get_current_state_event( + room_id, EventTypes.Member, mxid + ) + joined = ( + event is not None + and event.content.get("membership") == Membership.JOIN + ) + else: + assert server_name is not None + try: + parse_and_validate_server_name(server_name) + except ValueError: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"Invalid server name: {server_name}", + Codes.INVALID_PARAM, + ) + joined = await self.store.is_host_joined(room_id, server_name) + + return HTTPStatus.OK, {"joined": joined} + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + if hs.config.experimental.msc4502_enabled: + AppserviceRoomMembershipRestServlet(hs).register(http_server) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 620c2b907b..3f124d9a2c 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -24,7 +24,11 @@ from unittest.mock import AsyncMock, Mock from twisted.internet import defer -from synapse.appservice import ApplicationService, Namespace +from synapse.appservice import ( + ApplicationService, + Namespace, + Scopes, +) from synapse.types import UserID from tests import unittest @@ -257,3 +261,33 @@ class ApplicationServiceTestCase(unittest.TestCase): ) ) ) + + +class ApplicationServiceScopesTestCase(unittest.TestCase): + def test_has_no_scopes_by_default(self) -> None: + service = ApplicationService( + id="unique_identifier", + sender=UserID.from_string("@as:test"), + token="some_token", + ) + self.assertEqual(len(service.scopes), 0) + self.assertFalse(service.has_scope(Scopes.QUERY_ROOM_MEMBERSHIP)) + + def test_has_valid_scope_if_specified(self) -> None: + service = ApplicationService( + id="unique_identifier", + sender=UserID.from_string("@as:test"), + token="some_token", + scopes=[Scopes.QUERY_ROOM_MEMBERSHIP], + ) + self.assertEqual(len(service.scopes), 1) + self.assertTrue(service.has_scope(Scopes.QUERY_ROOM_MEMBERSHIP)) + + def test_unknown_scope_raises(self) -> None: + with self.assertRaises(ValueError): + ApplicationService( + id="unique_identifier", + sender=UserID.from_string("@as:test"), + token="some_token", + scopes=["does:not:exist"], + ) diff --git a/tests/rest/client/test_room_membership.py b/tests/rest/client/test_room_membership.py new file mode 100644 index 0000000000..265c1f0f49 --- /dev/null +++ b/tests/rest/client/test_room_membership.py @@ -0,0 +1,235 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from http import HTTPStatus +from unittest.mock import AsyncMock, patch + +from twisted.internet.testing import MemoryReactor + +from synapse.api.errors import Codes +from synapse.appservice import ApplicationService, Scopes +from synapse.rest import admin +from synapse.rest.client import login, room, room_membership +from synapse.server import HomeServer +from synapse.types import JsonDict, UserID, create_requester +from synapse.util.clock import Clock + +from tests import unittest +from tests.test_utils import event_injection +from tests.unittest import override_config + +AS_TOKEN = "i_am_an_app_service" +AS_TOKEN_NO_SCOPE = "i_am_an_app_service_without_scope" + + +class AppserviceRoomMembershipRestServletTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + login.register_servlets, + room.register_servlets, + room_membership.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = { + "msc4502_enabled": True, + # Merge in this order to allow `override_config` to override the flag + **config.get("experimental_features", {}), + } + return config + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.creator = self.register_user("owner", "pass") + self.creator_tok = self.login("owner", "pass") + self.room_id = self.helper.create_room_as(self.creator, tok=self.creator_tok) + + self.joined_user = self.register_user("joined_user", "pass") + self.joined_user_tok = self.login("joined_user", "pass") + self.helper.join(self.room_id, self.joined_user, tok=self.joined_user_tok) + + self.not_joined_user = self.register_user("not_joined_user", "pass") + self.not_joined_user_tok = self.login("not_joined_user", "pass") + + self.remote_server = "elsewhere.com" + self.remote_user = UserID.from_string(f"@joined_user:{self.remote_server}") + self.get_success( + event_injection.inject_member_event( + self.hs, self.room_id, self.remote_user.to_string(), "join" + ) + ) + self.not_joined_remote_user = UserID.from_string( + f"@not_joined_user:{self.remote_server}" + ) + + self.unknown_server = "unknown.org" + self.unknown_room_id = "!unknown:unknown.org" + + main_store = self.hs.get_datastores().main + main_store.services_cache.append( + ApplicationService( + AS_TOKEN, + id="as_with_scope", + sender=UserID.from_string("@as:test"), + scopes=[Scopes.QUERY_ROOM_MEMBERSHIP], + ) + ) + main_store.services_cache.append( + ApplicationService( + AS_TOKEN_NO_SCOPE, + id="as_without_scope", + sender=UserID.from_string("@as2:test"), + ) + ) + + def _get_joined( + self, room_id: str, params: str, access_token: str | None + ) -> tuple[int, JsonDict]: + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/io.element.msc4502/rooms/{room_id}/is_joined?{params}", + access_token=access_token, + ) + return channel.code, channel.json_body + + def test_invalid_room_id_format(self) -> None: + code, body = self._get_joined( + "not-a-room-id", f"mxid={self.joined_user}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.INVALID_PARAM) + + def test_both_mxid_and_server_name_given(self) -> None: + code, body = self._get_joined( + self.room_id, + f"mxid={self.joined_user}&server_name={self.hs.hostname}", + AS_TOKEN, + ) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.MISSING_PARAM) + + def test_neither_mxid_nor_server_name_given(self) -> None: + code, body = self._get_joined(self.room_id, "", AS_TOKEN) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.MISSING_PARAM) + + def test_invalid_mxid_format(self) -> None: + code, body = self._get_joined(self.room_id, "mxid=not-a-userid", AS_TOKEN) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.INVALID_PARAM) + + def test_invalid_server_name_format(self) -> None: + code, body = self._get_joined(self.room_id, "server_name=foo_bar", AS_TOKEN) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.INVALID_PARAM) + + def test_local_user_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_local_user_not_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.not_joined_user}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_remote_user_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.remote_user.to_string()}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_remote_user_not_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.not_joined_remote_user.to_string()}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_local_server_name_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"server_name={self.hs.hostname}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_remote_server_name_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"server_name={self.remote_server}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_remote_server_name_not_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"server_name={self.unknown_server}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_nonexistent_room_returns_false(self) -> None: + code, body = self._get_joined( + self.unknown_room_id, f"server_name={self.unknown_server}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_no_token_unauthorized(self) -> None: + code, body = self._get_joined(self.room_id, f"mxid={self.joined_user}", None) + self.assertEqual(code, HTTPStatus.UNAUTHORIZED, body) + self.assertEqual(body["errcode"], Codes.MISSING_TOKEN) + + def test_normal_user_token_forbidden(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", self.creator_tok + ) + self.assertEqual(code, HTTPStatus.FORBIDDEN, body) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + def test_same_user_token_forbidden(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", self.joined_user_tok + ) + self.assertEqual(code, HTTPStatus.FORBIDDEN, body) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + def test_user_with_oauth_scope_allowed(self) -> None: + requester = create_requester(self.creator, scope={Scopes.QUERY_ROOM_MEMBERSHIP}) + with patch.object( + self.hs.get_auth(), "get_user_by_req", AsyncMock(return_value=requester) + ): + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", "doesnt-matter" + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_appservice_without_scope_forbidden(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", AS_TOKEN_NO_SCOPE + ) + self.assertEqual(code, HTTPStatus.FORBIDDEN, body) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + @override_config({"experimental_features": {"msc4502_enabled": False}}) + def test_unreachable_when_experimental_flag_disabled(self) -> None: + code, _ = self._get_joined(self.room_id, f"mxid={self.joined_user}", AS_TOKEN) + self.assertEqual(code, HTTPStatus.NOT_FOUND) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index bbdbe38e07..1ed6bb145b 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -153,6 +153,17 @@ class VersionsTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 200, channel.result) self.assertTrue(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + def test_msc4502_false_by_default(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertFalse(channel.json_body["unstable_features"]["io.element.msc4502"]) + + @unittest.override_config({"experimental_features": {"msc4502_enabled": True}}) + def test_msc4502_true_if_enabled(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertTrue(channel.json_body["unstable_features"]["io.element.msc4502"]) + def _sanity_check_versions_response(self, versions_response: JsonDict) -> None: """ Make sure this looks like a `/_matrix/client/versions` response diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 4b9d069d6a..8430e54b92 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -21,7 +21,7 @@ import json import os import tempfile -from typing import cast +from typing import Any, cast from unittest.mock import AsyncMock, Mock import yaml @@ -29,7 +29,11 @@ import yaml from twisted.internet import defer from twisted.internet.testing import MemoryReactor -from synapse.appservice import ApplicationService, ApplicationServiceState +from synapse.appservice import ( + ApplicationService, + ApplicationServiceState, + Scopes, +) from synapse.config._base import ConfigError from synapse.events import EventBase from synapse.server import HomeServer @@ -38,7 +42,7 @@ from synapse.storage.databases.main.appservice import ( ApplicationServiceStore, ApplicationServiceTransactionStore, ) -from synapse.types import DeviceListUpdates +from synapse.types import DeviceListUpdates, JsonDict from synapse.util.clock import Clock from tests import unittest @@ -479,8 +483,8 @@ class TestTransactionStore(ApplicationServiceTransactionStore, ApplicationServic class ApplicationServiceStoreConfigTestCase(unittest.HomeserverTestCase): - def _write_config(self, suffix: str, **kwargs: str) -> str: - vals = { + def _write_config(self, suffix: str, **kwargs: Any) -> str: + vals: JsonDict = { "id": "id" + suffix, "url": "url" + suffix, "as_token": "as_token" + suffix, @@ -566,3 +570,69 @@ class ApplicationServiceStoreConfigTestCase(unittest.HomeserverTestCase): self.assertIn(f1, str(e)) self.assertIn(f2, str(e)) self.assertIn("as_token", str(e)) + + def test_invalid_scopes_raises(self) -> None: + f = self._write_config( + suffix="1", **{"io.element.msc4502.scopes": "not-a-list"} + ) + + self.hs.config.appservice.app_service_config_files = [f] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + with self.assertRaises(ValueError): + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_known_scope_works(self) -> None: + f = self._write_config( + suffix="1", + **{"io.element.msc4502.scopes": [Scopes.QUERY_ROOM_MEMBERSHIP.value]}, + ) + + self.hs.config.appservice.app_service_config_files = [f] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_unknown_scope_raises(self) -> None: + f = self._write_config( + suffix="1", **{"io.element.msc4502.scopes": ["does:not:exist"]} + ) + + self.hs.config.appservice.app_service_config_files = [f] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + with self.assertRaises(ValueError): + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) From ea4218b16b30ee107ecd7c3b7f57257221846dec Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 20 Aug 2026 16:43:24 +0100 Subject: [PATCH 108/158] Schema diff CI: Fix hanging when Rust module changes (#20129) Follows: #20117 schema_diff: Move faketime so it doesn't affect `poetry install` Caused a hang in the CI for https://github.com/element-hq/synapse/pull/20003 whilst recompiling the Rust module (it seems) --------- Signed-off-by: Olivier 'reivilibre --- .ci/scripts/schema_diff.py | 6 ++++++ .github/workflows/schema_diff.yml | 5 ++--- changelog.d/20129.misc | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20129.misc diff --git a/.ci/scripts/schema_diff.py b/.ci/scripts/schema_diff.py index 8354b64233..5dcc248f80 100755 --- a/.ci/scripts/schema_diff.py +++ b/.ci/scripts/schema_diff.py @@ -33,6 +33,12 @@ def run_make_full_schema(output_dir: Path) -> None: sys.exit(1) cmd: list[str] = [ + # Use faketime here for schema deltas that are wall-clock sensitive under SQLite + # We must only use faketime at this level because freezing the clock + # seems to cause `poetry install` to hang when recompiling our Rust module + "faketime", + "-f", + "2001-05-25 12:42:42", "poetry", "run", str(MAKE_FULL_SCHEMA_SCRIPT), diff --git a/.github/workflows/schema_diff.yml b/.github/workflows/schema_diff.yml index 3dfc02a469..58aa720b9a 100644 --- a/.github/workflows/schema_diff.yml +++ b/.github/workflows/schema_diff.yml @@ -31,6 +31,7 @@ jobs: background: true id: postgres # Use faketime here for schema deltas that are wall-clock sensitive under Postgres + # For SQLite, faketime is used when invoking `make_full_schema.sh` within the script run: | # Build a docker image with faketime mkdir /tmp/postgres-faketime @@ -78,10 +79,8 @@ jobs: PGHOST: localhost PGUSER: postgres PGPASSWORD: postgres - # Use faketime here for schema deltas that are wall-clock sensitive under SQLite run: | - faketime -f "2001-05-25 12:42:42" \ - poetry run python .ci/scripts/schema_diff.py \ + poetry run python .ci/scripts/schema_diff.py \ --base origin/develop \ > "${{ runner.temp }}/schema_diff.md" diff --git a/changelog.d/20129.misc b/changelog.d/20129.misc new file mode 100644 index 0000000000..92d421117e --- /dev/null +++ b/changelog.d/20129.misc @@ -0,0 +1 @@ +Fix the schema diff CI breaking when the Rust module was changed. \ No newline at end of file From 57cf536ee486b996309dd8b1e3dcd60664d25c37 Mon Sep 17 00:00:00 2001 From: Christian Paul Date: Fri, 21 Aug 2026 23:17:54 +0800 Subject: [PATCH 109/158] Update Synapse repo link in inconsistent stream error (#20128) --- changelog.d/20128.misc | 1 + synapse/storage/util/sequence.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20128.misc diff --git a/changelog.d/20128.misc b/changelog.d/20128.misc new file mode 100644 index 0000000000..1361eb0e5e --- /dev/null +++ b/changelog.d/20128.misc @@ -0,0 +1 @@ +Update Synapse repo link in inconsistent stream error. diff --git a/synapse/storage/util/sequence.py b/synapse/storage/util/sequence.py index 5bee3cf34f..0f3cfa7a7e 100644 --- a/synapse/storage/util/sequence.py +++ b/synapse/storage/util/sequence.py @@ -41,7 +41,7 @@ Postgres sequence '%(seq)s' is inconsistent with associated stream position of '%(stream_name)s' in the 'stream_positions' table. This is likely a programming error and should be reported at -https://github.com/matrix-org/synapse. +https://github.com/element-hq/synapse. A temporary workaround to fix this error is to shut down Synapse (including any and all workers) and run the following SQL: From 3debaff0f86c1f34d5c5970bd5a10bea8a67b266 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 21 Aug 2026 18:56:03 +0300 Subject: [PATCH 110/158] Fix `__len__` of Sliding Sync `PerConnectionState` ignoring account data (#20124) Noticed while working on https://github.com/element-hq/synapse/pull/20003, submitting separately. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Eric Eastwood --- changelog.d/20124.misc | 1 + synapse/types/handlers/sliding_sync.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20124.misc diff --git a/changelog.d/20124.misc b/changelog.d/20124.misc new file mode 100644 index 0000000000..d92a12ce08 --- /dev/null +++ b/changelog.d/20124.misc @@ -0,0 +1 @@ +Fix cache `__len__` of Sliding Sync `PerConnectionState` ignoring account data entries. \ No newline at end of file diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index dd913250ba..d8c90c0fdd 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -919,6 +919,7 @@ class PerConnectionState: receipts: The status of each room for the receipts stream. room_configs: Map from room_id to the `RoomSyncConfig` of all rooms that we have previously sent down. + account_data: The status of each room for the account_data stream. """ last_used_ts: int | None = None @@ -951,7 +952,12 @@ class PerConnectionState: ) def __len__(self) -> int: - return len(self.rooms) + len(self.receipts) + len(self.room_configs) + return ( + len(self.account_data) + + len(self.rooms) + + len(self.receipts) + + len(self.room_configs) + ) @attr.s(auto_attribs=True) From 3db77e80a5a9bea0d36830906661f568314d04d0 Mon Sep 17 00:00:00 2001 From: catfromplan9 <104175360+catfromplan9@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:02:44 +0000 Subject: [PATCH 111/158] Fix WebP thumbnails losing their transparency (#20094) Signed-off-by: cat --- changelog.d/20094.bugfix | 1 + synapse/media/media_repository.py | 20 +++-- synapse/media/thumbnailer.py | 30 ++++++- tests/media/test_media_storage.py | 140 +++++++++++++++++++++++++++++- 4 files changed, 181 insertions(+), 10 deletions(-) create mode 100644 changelog.d/20094.bugfix diff --git a/changelog.d/20094.bugfix b/changelog.d/20094.bugfix new file mode 100644 index 0000000000..ebea9aaa04 --- /dev/null +++ b/changelog.d/20094.bugfix @@ -0,0 +1 @@ +Thumbnail WebP images that use transparency as PNG rather than JPEG, to preserve transparency during thumbnailing. Contributed by @catfromplan9. diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 180adc20a0..31bb401e65 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -1387,17 +1387,23 @@ class MediaRepository: self.hs.get_reactor(), thumbnailer.transpose ) + # JPEG has no alpha channel, so it would flatten a transparent + # image onto a solid color background. + needs_alpha = await defer_to_thread( + self.hs.get_reactor(), lambda: thumbnailer.has_transparency + ) + # We deduplicate the thumbnail sizes by ignoring the cropped versions if # they have the same dimensions of a scaled one. thumbnails: dict[tuple[int, int, str], str] = {} for requirement in requirements: + t_type = requirement.media_type + if needs_alpha and t_type == "image/jpeg": + t_type = "image/png" + if requirement.method == "crop": thumbnails.setdefault( - ( - requirement.width, - requirement.height, - requirement.media_type, - ), + (requirement.width, requirement.height, t_type), requirement.method, ) elif requirement.method == "scale": @@ -1406,9 +1412,7 @@ class MediaRepository: ) t_width = min(m_width, t_width) t_height = min(m_height, t_height) - thumbnails[(t_width, t_height, requirement.media_type)] = ( - requirement.method - ) + thumbnails[(t_width, t_height, t_type)] = requirement.method # Now we generate the thumbnails for each dimension, store it # diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index 27f016931a..a75a420750 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -23,7 +23,7 @@ import logging from collections.abc import Callable from io import BytesIO from types import TracebackType -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from PIL import Image, ImageSequence @@ -174,6 +174,34 @@ class Thumbnailer: converted = image return converted.resize((width, height), Image.LANCZOS) + @property + def has_transparency(self) -> bool: + """Whether the current frame actually makes use of transparency. + + Having an alpha channel isn't enough: fully opaque RGBA is common. + + Note that this can block for a while on large images, as `getextrema()` + scans the entire alpha plane. Consider calling it via `defer_to_thread`. + """ + image = self.image + + if image.mode == "P" and "transparency" in image.info: + return True + + if "A" not in image.getbands(): + return False + + try: + with image.getchannel("A") as alpha: + # Single band, so `getextrema` returns a plain (min, max) pair. + min_alpha = cast(float, alpha.getextrema()[0]) + except Exception: + # Assume transparency, since dropping it is the destructive option. + logger.exception("Error inspecting image alpha channel") + return True + + return min_alpha < 255 + @property def is_animated(self) -> bool: if self._animation_broken: diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 430a1d6789..da5fa8a140 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -62,7 +62,7 @@ from synapse.module_api.callbacks.spamchecker_callbacks import load_legacy_spam_ from synapse.rest import admin from synapse.rest.client import login, media from synapse.server import HomeServer -from synapse.types import JsonDict, RoomAlias +from synapse.types import JsonDict, RoomAlias, UserID from synapse.util.clock import Clock from tests import unittest @@ -1444,6 +1444,15 @@ def _make_stale_mpo() -> bytes: return data[:primary_size] +def _make_webp(alpha: int) -> bytes: + """Build a small WebP whose pixels all have the given alpha.""" + out = BytesIO() + Image.new("RGBA", (64, 64), (255, 0, 0, alpha)).save( + out, format="WEBP", lossless=True + ) + return out.getvalue() + + class ThumbnailerAnimatedTestCase(unittest.TestCase): """Tests that the thumbnailer only animates when explicitly asked to.""" @@ -1589,3 +1598,132 @@ class ThumbnailerAnimatedTestCase(unittest.TestCase): self.assertEqual(Image.open(out).format, "WEBP") self.assertFalse(getattr(Image.open(out), "is_animated", False)) self.assert_is_first_frame(out) + + +class ThumbnailerTransparencyTestCase(unittest.TestCase): + """Tests the transparency detection that picks the thumbnail format.""" + + def setUp(self) -> None: + super().setUp() + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) + + def _thumbnailer(self, name: str, data: bytes) -> Thumbnailer: + path = os.path.join(self.tempdir, name) + with open(path, "wb") as f: + f.write(data) + thumbnailer = Thumbnailer(path) + self.addCleanup(thumbnailer.close) + return thumbnailer + + def test_transparent_webp(self) -> None: + thumbnailer = self._thumbnailer("transparent.webp", _make_webp(0)) + self.assertTrue(thumbnailer.has_transparency) + + def test_partially_transparent_webp(self) -> None: + thumbnailer = self._thumbnailer("partial.webp", _make_webp(128)) + self.assertTrue(thumbnailer.has_transparency) + + def test_opaque_webp(self) -> None: + thumbnailer = self._thumbnailer("opaque.webp", _make_webp(255)) + self.assertFalse(thumbnailer.has_transparency) + + def test_unused_alpha_channel(self) -> None: + """An alpha channel that is fully opaque doesn't count as transparency.""" + out = BytesIO() + Image.new("RGBA", (64, 64), (255, 0, 0, 255)).save(out, format="PNG") + thumbnailer = self._thumbnailer("opaque_rgba.png", out.getvalue()) + self.assertEqual(thumbnailer.image.mode, "RGBA") + self.assertFalse(thumbnailer.has_transparency) + + def test_palette_transparency(self) -> None: + """Palette images signal transparency through an index, not a channel.""" + out = BytesIO() + Image.new("P", (64, 64)).save(out, format="PNG", transparency=0) + thumbnailer = self._thumbnailer("palette.png", out.getvalue()) + self.assertEqual(thumbnailer.image.mode, "P") + self.assertTrue(thumbnailer.has_transparency) + + def test_cmyk_jpeg(self) -> None: + thumbnailer = self._thumbnailer("opaque.jpg", SMALL_CMYK_JPEG) + self.assertFalse(thumbnailer.has_transparency) + + def test_png_thumbnail_keeps_alpha(self) -> None: + """The PNG we switch to actually retains the transparency.""" + thumbnailer = self._thumbnailer("transparent.webp", _make_webp(0)) + result = Image.open(thumbnailer.scale(32, 32, "image/png")) + self.assertEqual(result.format, "PNG") + pixel = result.convert("RGBA").getpixel((16, 16)) + assert isinstance(pixel, tuple) + self.assertEqual(pixel[3], 0) + + +class ThumbnailFormatTestCase(unittest.HomeserverTestCase): + """Tests that transparent sources aren't flattened onto a black background.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + media.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.media_repo = hs.get_media_repository() + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + + def create_resource_dict(self) -> dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + def _upload(self, data: bytes, media_type: str) -> str: + """Upload the given media and return its media ID.""" + mxc = self.get_success( + self.media_repo.create_or_update_content( + media_type, + "test", + BytesIO(data), + len(data), + UserID.from_string(self.user), + ) + ) + return mxc.media_id + + def _thumbnail_types(self, media_id: str) -> set[str]: + thumbnails = self.get_success(self.store.get_local_media_thumbnails(media_id)) + self.assertTrue(thumbnails, "no thumbnails were generated") + return {thumbnail.type for thumbnail in thumbnails} + + def test_transparent_webp_thumbnails_as_png(self) -> None: + media_id = self._upload(_make_webp(0), "image/webp") + self.assertEqual(self._thumbnail_types(media_id), {"image/png"}) + + def test_opaque_webp_thumbnails_as_jpeg(self) -> None: + media_id = self._upload(_make_webp(255), "image/webp") + self.assertEqual(self._thumbnail_types(media_id), {"image/jpeg"}) + + def test_animated_thumbnail_is_still_webp(self) -> None: + """Transparency detection doesn't disturb the animated thumbnails.""" + media_id = self._upload(_make_animated_gif(), "image/gif") + self.assertIn(ANIMATED_THUMBNAIL_TYPE, self._thumbnail_types(media_id)) + + def test_served_thumbnail_keeps_transparency(self) -> None: + """The thumbnail a client actually receives still has its alpha channel.""" + media_id = self._upload(_make_webp(0), "image/webp") + + channel = self.make_request( + "GET", + f"/_matrix/client/v1/media/thumbnail/test/{media_id}" + "?width=32&height=32&method=scale", + shorthand=False, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.headers.getRawHeaders(b"Content-Type"), [b"image/png"]) + + thumbnail = Image.open(BytesIO(channel.result["body"])) + pixel = thumbnail.convert("RGBA").getpixel((16, 16)) + assert isinstance(pixel, tuple) + self.assertEqual(pixel[3], 0) From 4bb07152c8784a1052da87e94a5ea463e143956d Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 25 Aug 2026 17:37:03 +0300 Subject: [PATCH 112/158] Support for profile updates over Sliding Sync (MSC4262) (#20003) This PR implements support for profile updates over Sliding Sync: https://github.com/matrix-org/matrix-spec-proposals/pull/4262. This pr may be easier to review as a whole than commit by commit. This builds on the legacy sync profile updates feature https://github.com/element-hq/synapse/pull/19556, specifically the profile updates stream it added. Submitting for early review to get consensus on implementation. There are some things we would like to add still, from spec, mainly: * > Homeservers should only consider a profile field update "accepted" by a client > once the client returns with a new /sync request with the next /sync token, > NOT just after sending down the profile update. The client may never receive > response due to network conditions, or a bug in the client implementation. * > When a room enters this subset in this connection for the first time, all requested > fields from profiles of users in that room MAY be sent down. This gives the client > a base set of information for which future field updates can be applied on top of. > The homeserver MAY omit some fields and profiles if it believes that the client has > already received them, likewise repeat profiles MAY be sent down based on homeserver > implementation. * > Finally, if the list of fields expands to cover a new field ID, those fields should > be sent down for all users that are within the current room subset. Future incremental > updates will then include changes to this field. * Additionally, we would need to implement a lazy loading cache similar to the legacy sync. (not part of MSC as such) Depending on review these could either be added to this pr, or to keep this pr from not growing too much, be added in a follow-up pr, as they are more enhancement to this base sliding sync profile updates functionality than a part of the core functionality. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Olivier 'reivilibre' Co-authored-by: Olivier 'reivilibre --- changelog.d/20003.feature | 2 + .../conf/workers-shared-extra.yaml.j2 | 2 +- .../configuration/config_documentation.md | 2 +- rust/src/handlers/versions.rs | 4 + schema/synapse-config.schema.yaml | 8 +- synapse/api/constants.py | 6 +- synapse/config/server.py | 2 +- synapse/handlers/profile.py | 9 +- synapse/handlers/sliding_sync/extensions.py | 406 +++++- synapse/rest/client/sync.py | 39 +- synapse/storage/databases/main/events.py | 6 +- synapse/storage/databases/main/profile.py | 77 +- synapse/storage/databases/main/roommember.py | 18 +- synapse/storage/schema/__init__.py | 2 +- .../main/delta/94/07_profile_updates.sql | 2 +- synapse/types/handlers/sliding_sync.py | 15 + synapse/types/rest/client/__init__.py | 15 + synapse/util/async_helpers.py | 27 + tests/handlers/test_profile.py | 22 +- tests/handlers/test_sync.py | 77 +- .../sliding_sync/test_extension_profiles.py | 1231 +++++++++++++++++ 21 files changed, 1874 insertions(+), 98 deletions(-) create mode 100644 changelog.d/20003.feature create mode 100644 tests/rest/client/sliding_sync/test_extension_profiles.py diff --git a/changelog.d/20003.feature b/changelog.d/20003.feature new file mode 100644 index 0000000000..add6b613c8 --- /dev/null +++ b/changelog.d/20003.feature @@ -0,0 +1,2 @@ +Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). +Currently defaults to not enabled, and is limited to local users only for the sync results. \ No newline at end of file diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 64a36522fa..4dc4eb932b 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -15,7 +15,7 @@ enable_registration_without_verification: true bcrypt_rounds: 4 url_preview_enabled: true url_preview_ip_range_blacklist: [] -# MSC4429 Profile updates down legacy /sync +# MSC4429 and MSC4262 Profile updates down sync include_profile_updates_in_sync: true ## Registration ## diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 7654039e5c..00a31ff447 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -339,7 +339,7 @@ include_profile_data_on_invite: false ### `include_profile_updates_in_sync` *(boolean)* Use this option to include updates of other users' profiles in sync responses, for users who share rooms. -Requires an [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) compatible client, and is currently limited to legacy sync and local users only. +For legacy sync clients, requires [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) compatibility. For sliding sync clients, requires [MSC4262](https://github.com/matrix-org/matrix-spec-proposals/pull/4262) compatibility. Note, profile updates via sync are currently limited to local users only. This feature is under development and should be used with caution on busy servers or servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring profile information doesn't leak across rooms. Defaults to `false`. Example configuration: diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index dc9e55e349..ab4bd9a160 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -251,6 +251,9 @@ pub struct UnstableFeatureMap { /// MSC4169: Backwards-compatible redaction sending using `/send` #[serde(rename = "com.beeper.msc4169")] msc4169: bool, + /// MSC4262: Profile updates for simplified sliding sync. + #[serde(rename = "org.matrix.msc4262")] + msc4262: bool, /// MSC4354: Sticky events #[serde(rename = "org.matrix.msc4354")] msc4354: bool, @@ -320,6 +323,7 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4155: config.experimental.msc4155_enabled, msc4306: config.experimental.msc4306_enabled, msc4169: config.experimental.msc4169_enabled, + msc4262: config.server.include_profile_updates_in_sync, msc4354: config.experimental.msc4354_enabled, msc4380: true, msc4429: config.server.include_profile_updates_in_sync, diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 6a6210fcb2..a8d1719288 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -281,9 +281,11 @@ properties: Use this option to include updates of other users' profiles in sync responses, for users who share rooms. - Requires an [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) - compatible client, and is currently limited to legacy sync and local users only. - + For legacy sync clients, requires [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) + compatibility. For sliding sync clients, requires + [MSC4262](https://github.com/matrix-org/matrix-spec-proposals/pull/4262) compatibility. Note, profile updates + via sync are currently limited to local users only. + This feature is under development and should be used with caution on busy servers or servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring profile information doesn't leak across rooms. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 041a7f284a..8e91b2e203 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -441,6 +441,7 @@ class ProfileUpdateAction(str, enum.Enum): normally includes. This update action currently has no meaning for sync responses that are not incremental and non-lazy. """ + LEFT_ROOM = "left_room" """ This profile update row action represents a user leaving a room. @@ -450,9 +451,12 @@ class ProfileUpdateAction(str, enum.Enum): profiles, so clients can clear their cache containing the users profile data they are no longer interested in. """ + UPDATE = "update" """ - This profile update row action represents a user updating a profile field. + This profile update row action represents a user updating one or more + profile fields. + 'Updating' could mean creating, changing the value of, or deleting a field. Depending on the type of sync (initial/incremental, lazy/non-lazy), either the diff of profile field updates or all the current profile fields are included diff --git a/synapse/config/server.py b/synapse/config/server.py index 42d43ea7f6..00b414ea30 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -585,7 +585,7 @@ class ServerConfig(Config): " 'allow_public_rooms_over_federation' is set." ) - # Whether to support MSC4429 profile updates down legacy /sync + # Whether to support MSC4429 and MSC4262 Profile updates down sync self.include_profile_updates_in_sync = config.get( "include_profile_updates_in_sync", False, diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 5f2cec2366..3d052bc893 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -106,7 +106,9 @@ class ProfileHandler: self._worker_locks = hs.get_worker_locks_handler() # Profile updates stream - self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._is_events_writer = ( hs.get_instance_name() in hs.config.worker.writers.events ) @@ -767,6 +769,9 @@ class ProfileHandler: ) -> None: """Delete a field from a user's profile. + This should only be called for custom profile fields, + not displayname or avatar_url. + Preconditions: - This must NOT be called as part of deactivating the user, because we will notify modules about the change whilst claiming it is not related @@ -780,6 +785,8 @@ class ProfileHandler: field_name: The name of the profile field to remove. by_admin: Whether this change was made by an administrator. """ + assert field_name not in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL) + if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index b3342de778..7ee26079ed 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -25,20 +25,31 @@ from typing import ( from typing_extensions import TypeAlias, assert_never -from synapse.api.constants import AccountDataTypes, EduTypes, StickyEvent +from synapse.api.constants import ( + AccountDataTypes, + EduTypes, + EventTypes, + ProfileFields, + ProfileUpdateAction, + StickyEvent, +) from synapse.events.utils import FilteredEvent from synapse.handlers.receipts import ReceiptEventSource from synapse.logging.opentracing import trace from synapse.storage.databases.main.receipts import ReceiptInRoom from synapse.types import ( Absent, + AbsentType, DeviceListUpdates, + JsonDict, JsonMapping, + JsonValue, MultiWriterStreamToken, SlidingSyncStreamToken, StrCollection, StreamToken, ThreadSubscriptionsToken, + UserID, ) from synapse.types.handlers.sliding_sync import ( HaveSentRoomFlag, @@ -47,6 +58,7 @@ from synapse.types.handlers.sliding_sync import ( PerConnectionState, SlidingSyncConfig, SlidingSyncResult, + StateValues, ) from synapse.types.rest.client import SlidingSyncStickyEventsToken from synapse.util.async_helpers import ( @@ -80,6 +92,7 @@ class SlidingSyncExtensionHandler: self._storage_controllers = hs.get_storage_controllers() self._enable_thread_subscriptions = hs.config.experimental.msc4306_enabled self._enable_sticky_events = hs.config.experimental.msc4354_enabled + self._enable_profiles = hs.config.server.include_profile_updates_in_sync @trace async def get_extensions_response( @@ -197,6 +210,18 @@ class SlidingSyncExtensionHandler: from_token=from_token, ) + profiles_coro = None + if sync_config.extensions.profiles is not Absent and self._enable_profiles: + profiles_coro = self.get_profiles_extension_response( + sync_config=sync_config, + profiles_request=sync_config.extensions.profiles, + actual_room_ids=actual_room_ids, + to_token=to_token, + from_token=from_token, + actual_room_response_map=actual_room_response_map, + actual_lists=actual_lists, + ) + ( to_device_response, e2ee_response, @@ -205,6 +230,7 @@ class SlidingSyncExtensionHandler: typing_response, thread_subs_response, sticky_events_response, + profiles_response, ) = await gather_optional_coroutines( to_device_coro, e2ee_coro, @@ -213,6 +239,7 @@ class SlidingSyncExtensionHandler: typing_coro, thread_subs_coro, sticky_events_coro, + profiles_coro, ) return SlidingSyncResult.Extensions( @@ -223,6 +250,7 @@ class SlidingSyncExtensionHandler: typing=typing_response, thread_subscriptions=thread_subs_response, sticky_events=sticky_events_response, + profiles=profiles_response, ) def find_relevant_room_ids_for_extension( @@ -1055,3 +1083,379 @@ class SlidingSyncExtensionHandler: sticky_events_stream_id=sticky_events_to_id ), ) + + async def _get_profile_ids_for_profiles_extension( + self, + user_id: str, + actual_room_ids: set[str], + sync_config: SlidingSyncConfig, + actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], + actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], + ) -> tuple[set[str], set[str]]: + """ + Calculate target user profiles as candiates to include in the profile + extension sync response. + + This function looks at both the sync config and the already calculated + rooms response, and pieces together the full set of user IDs to include + profiles for, based on sync config rooms being lazy loading or not. + + For rooms with lazy loading, only profiles for those users who have sent events + into the timeline will be included, unless they would be included otherwise. + For other rooms, all members of the room will be included as candidates. + + Note, this does not collect user IDs from the profile updates stream. + + Args: + user_id: The full user ID syncing. + actual_room_ids: The actual room IDs in the the Sliding Sync response. + sync_config: The Sliding Sync config object. + actual_room_response_map: A calculated map of responses per room. + actual_lists: Sliding window API. A map of list key to list results in the + Sliding Sync response. + + Returns: + Tuple containing two sets: + - first including all found user IDs, + - second containing user IDs calculated via lazy configured rooms. + """ + lazy_profile_user_ids = set() + non_lazy_profile_user_ids = set() + + # Separate rooms into lazy and non-lazy based on sync config. + # Look at subscriptions first + lazy_rooms = ( + { + room_id + for room_id, room_config in sync_config.room_subscriptions.items() + if (EventTypes.Member, StateValues.LAZY) in room_config.required_state + } + if sync_config.room_subscriptions + else set() + ) + # Iterate lists to find lazy rooms + if sync_config.lists: + for list_name, list_data in sync_config.lists.items(): + if (EventTypes.Member, StateValues.LAZY) in list_data.required_state: + for op in actual_lists[list_name].ops: + lazy_rooms.update(op.room_ids) + + if lazy_rooms: + # For rooms configured as lazy, include users based on room response. + for room_id, room_data in actual_room_response_map.items(): + if room_id not in lazy_rooms: + continue + # Include users from timeline events + for timeline_event in room_data.timeline_events: + lazy_profile_user_ids.add(timeline_event.event.sender) + # Include users from required state + for state_event in room_data.required_state: + if state_event.type == EventTypes.Member: + lazy_profile_user_ids.add(state_event.state_key) + # Include heroes + if room_data.heroes: + for hero in room_data.heroes: + lazy_profile_user_ids.add(hero.user_id) + + non_lazy_rooms = actual_room_ids.difference(lazy_rooms) + # If we still have non-lazy rooms, get their members. + if non_lazy_rooms: + non_lazy_profile_user_ids = ( + # TODO we should consider adding a limit to how many profiles + # of room members we push down the line. However, this produces + # a problem for clients in that they won't know which users + # just don't have any profile information, and which users were limited + # out. If we had an endpoint to fetch a list of profiles at once, + # we could have a hard limit here and clients could fetch the missing + # profiles separately for non-lazy initial sync cases. + await self.store.get_local_users_who_share_room_with_user( + user_id, + limit_to_rooms=non_lazy_rooms, + ) + ) + + # Unify the two lists + profile_user_ids = lazy_profile_user_ids.union(non_lazy_profile_user_ids) + + # Return a tuple containing the full list of user IDs and the lazy subset. + return ( + profile_user_ids, + lazy_profile_user_ids, + ) + + async def _get_profiles_extension_initial_sync_response( + self, + user_id: UserID, + fields: set[str] | None, + profile_user_ids: set[str], + ) -> dict[str, JsonDict]: + """ + Build an initial sync response for the profiles extension. + + Args: + user_id: The syncing user UserID + fields: A set of fields to include in the response. + `None` means all fields. + profile_user_ids: Set of user IDs whose profiles are related to this sync response. + + Returns: + A dictionary (in API response format) mapping users to their + profile updates in an `updated` dictionary. + + { + "@user:example.org": { + "updated": { + "displayname": "Somebody", + "avatar_url": "mxc://example.org/123123123", + "org.example.field": "hiss", + ... + } + }, + ... + } + """ + response: dict[str, JsonDict] = {} + + # This doesn't return entries for the users with no profile data, + # which is good as we don't want to generate anything for users + # with no profile data in initial sync. + profile_data_by_user = await self.store.get_profile_data_for_users( + # Force our own user to be in the set, as we should + # always watch our own profile updates + profile_user_ids | {user_id.to_string()} + ) + + # Serialise the profile updates into the sync response format. + for profile_user_id, profile_data in profile_data_by_user.items(): + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] + # Include the fields the client asked for, or all, if not specified + if fields is not None: + per_user_updates = { + k: v for k, v in profile_data.items() if k in fields + } + else: + per_user_updates = profile_data + + if per_user_updates: + response[profile_user_id] = { + "updated": per_user_updates, + } + + return response + + async def get_profiles_extension_response( + self, + sync_config: SlidingSyncConfig, + profiles_request: SlidingSyncConfig.Extensions.ProfilesExtension, + actual_room_ids: set[str], + to_token: StreamToken, + from_token: SlidingSyncStreamToken | None, + actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], + actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], + ) -> SlidingSyncResult.Extensions.ProfilesExtension | None: + """ + Generate a response for the profiles extension. + + Args: + sync_config: The Sliding Sync config. + profiles_request: The profiles extension request. + actual_room_ids: The actual room IDs in the the Sliding Sync response. + to_token: The stream token to generate a response until. + from_token: The stream token to generate a response from. + actual_room_response_map: A calculated map of responses per room. + actual_lists: Sliding window API. A map of list key to list results in the + Sliding Sync response. + + Returns: + - A SlidingSyncResult.Extensions.ProfilesExtension object containing + all the users who have profile updates. + - None if the extension is disabled. + """ + if not profiles_request.enabled: + return None + + user_id = sync_config.user.to_string() + fields = ( + set(profiles_request.fields) + if profiles_request.fields is not Absent + else None + ) + + response: dict[str, JsonDict | None] = {} + + ( + profile_user_ids, + lazy_profile_user_ids, + ) = await self._get_profile_ids_for_profiles_extension( + user_id=user_id, + actual_room_ids=actual_room_ids, + sync_config=sync_config, + actual_room_response_map=actual_room_response_map, + actual_lists=actual_lists, + ) + + if from_token is None: + # Initial sync + return SlidingSyncResult.Extensions.ProfilesExtension( + users=await self._get_profiles_extension_initial_sync_response( + user_id=sync_config.user, + fields=fields, + profile_user_ids=profile_user_ids, + ), + ) + + # Incremental sync + updates = await self.store.get_profile_updates_for_user_and_fields( + from_id=from_token.stream_token.profile_updates_key, + to_id=to_token.profile_updates_key, + user_id=user_id, + field_names=fields, + ) + + # Set of users that just joined their first room that we share with them + joined_room_user_ids: set[str] = set() + # Set of tracked users that have updated their profile + updated_user_ids: set[str] = set() + # Set of tracked users that just left their last room that we share with them + left_room_user_ids: set[str] = set() + + # Process updates in stream order + # We need to be careful of users that have multiple types of updates + # within this sequence of stream rows. + for update in updates: + if update.action == ProfileUpdateAction.JOINED_ROOM: + joined_room_user_ids.add(update.user_id) + # If the user joins a shared room, that overrides + # the fact that they previously left the last shared room + left_room_user_ids.discard(update.user_id) + elif update.action == ProfileUpdateAction.UPDATE: + updated_user_ids.add(update.user_id) + elif update.action == ProfileUpdateAction.LEFT_ROOM: + left_room_user_ids.add(update.user_id) + # If the user leaves their last shared room, that overrides + # the fact that they previously joined a shared room + # and perhaps updated their profile whilst they were in it + joined_room_user_ids.discard(update.user_id) + updated_user_ids.discard(update.user_id) + + # Add the users who joined a shared room or updated their profile to the set of + # users we will serialise profiles for + profile_user_ids.update(joined_room_user_ids) + profile_user_ids.update(updated_user_ids) + + # Process left rooms + for other_user_id in left_room_user_ids: + # Return a null response to the client + # This tells the client that it will no longer receive updates for the user + response[other_user_id] = None + + updated_user_fields: dict[str, set[str]] = {} + # Set fields from updates + for update in updates: + if ( + update.action != ProfileUpdateAction.UPDATE + or not update.affected_fields + or update.user_id in left_room_user_ids + # Skip if not interested in this user + or update.user_id not in profile_user_ids + ): + continue + interesting_changed_fields: set[str] + if fields is not None: + interesting_changed_fields = set(update.affected_fields) & fields + else: + interesting_changed_fields = set(update.affected_fields) + + if not interesting_changed_fields: + # Skip the update as the client is not interested in these fields + continue + + updated_user_fields.setdefault(update.user_id, set()).update( + interesting_changed_fields + ) + + profile_data_by_user = await self.store.get_profile_data_for_users( + profile_user_ids, + ) + + # Serialise the profile updates into the sync response format. + for profile_user_id in profile_user_ids: + if profile_user_id in left_room_user_ids: + continue + profile_data = profile_data_by_user.get(profile_user_id) + if profile_data is None: + # We don't have profile data for this user + # (This is different from having an empty profile) + # Return a null in incremental sync, telling the client to + # remove all profile information for this user. + response[profile_user_id] = None + continue + + # Calculate which fields had updates + updated_fields: set[str] = updated_user_fields.get(profile_user_id, set()) + # Calculate the full available field list + user_fields = set(profile_data.keys()).union(updated_fields) + + # If the user joined the room or is included via lazy loading events, + # include all fields the client wants. This happens because when lazy + # a room, clients will not necessarily have the profile for the user that + # sent an event in the room, and thus we deliver all the fields. The same + # is true if another user joins the room - we need to deliver an initial + # state for clients to work on. + # For non-lazy-loaded users, include only updated fields. We assume clients + # with non-lazy loaded rooms have received the profiles for all the members + # in the room, and thus only need updates. + user_fields = ( + user_fields + if profile_user_id in joined_room_user_ids + or profile_user_id in lazy_profile_user_ids + else updated_fields + ) + # Filter down if the client only wants a subset + if fields: + user_fields = user_fields.intersection(fields) + + if not user_fields: + continue + + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} + per_user_removals: set[str] = set() + for field_name in user_fields: + # For custom fields the lack of a field means it will be `Absent`, + # for displayname/avatar_url it will be `None`, due to way we store + # things differently. + # FIXME: I intend to simplify this by pushing the special-case logic + # for these 'original' profile fields into the storage layer instead. + absent_type = ( + Absent + if field_name + not in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL) + else None + ) + field_value: JsonValue | dict[str, JsonValue] | AbsentType = ( + profile_data.get(field_name, absent_type) + ) + if ( + # If the field isn't found on the profile and it is present in + # `updated_fields`, that means an existing field has been removed. + # We need the check against `updated_fields` as some profile fields + # are `None` by default, for example each and every user created + # by Synapse will have `avatar_url: None`, and we don't want to + # constantly send that to the clients. + field_value is absent_type and field_name in updated_fields + ): + per_user_removals.add(field_name) + else: + per_user_updates[field_name] = cast(JsonValue, field_value) + + if per_user_updates or per_user_removals: + entry: dict[str, JsonValue | JsonDict] = {} + response[profile_user_id] = entry + if per_user_updates: + entry["updated"] = per_user_updates + if per_user_removals: + entry["removed"] = list(per_user_removals) + + return SlidingSyncResult.Extensions.ProfilesExtension( + users=response, + ) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 4e437c5e98..08002a6708 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -55,7 +55,13 @@ from synapse.http.servlet import ( from synapse.http.site import SynapseRequest from synapse.logging.opentracing import log_kv, set_tag, trace_with_opname from synapse.rest.admin.experimental_features import ExperimentalFeature -from synapse.types import JsonDict, Requester, SlidingSyncStreamToken, StreamToken +from synapse.types import ( + JsonDict, + JsonMapping, + Requester, + SlidingSyncStreamToken, + StreamToken, +) from synapse.types.rest.client import SlidingSyncBody from synapse.util.caches.lrucache import LruCache from synapse.util.cancellation import cancellable @@ -123,7 +129,9 @@ class SyncRestServlet(RestServlet): self._event_serializer = hs.get_event_client_serializer() self._msc2654_enabled = hs.config.experimental.msc2654_enabled self._msc3773_enabled = hs.config.experimental.msc3773_enabled - self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, @@ -352,7 +360,7 @@ class SyncRestServlet(RestServlet): if sync_result.to_device: response["to_device"] = {"events": sync_result.to_device} - if self._msc4429_enabled and sync_result.profile_updates: + if self._include_profile_updates_in_sync and sync_result.profile_updates: # FIXME: See issue https://github.com/element-hq/synapse/issues/19981 # for concerns around the current implementation of the profile # updates stream. @@ -1142,8 +1150,33 @@ class SlidingSyncRestServlet(RestServlet): requester, extensions.sticky_events, ref_rooms_results ) + if extensions.profiles: + serialized_extensions[ + "org.matrix.msc4262.profiles" + ] = await self._serialise_profiles( + extensions.profiles, + ) + return serialized_extensions + async def _serialise_profiles( + self, + profiles: SlidingSyncResult.Extensions.ProfilesExtension, + ) -> JsonMapping: + """ + Serialise the profiles extension response. + + Args: + profiles: The generated profiles response object. + + Returns: + A dictionary containing the response `users` with the + generated profile updates. + """ + return { + "users": profiles.users, + } + async def _serialise_sticky_events( self, requester: Requester, diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 8211612e2d..35f387576f 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -269,7 +269,9 @@ class PersistEventsStore: self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() self._msc4354_enabled = hs.config.experimental.msc4354_enabled - self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._ephemeral_messages_enabled = hs.config.server.enable_ephemeral_messages self.is_mine_id = hs.is_mine_id @@ -2121,7 +2123,7 @@ class PersistEventsStore: txn, {m for m in members_to_cache_bust if not self.hs.is_mine_id(m)} ) - if self._msc4429_enabled: + if self._include_profile_updates_in_sync: # Handle changes to the profile updates stream. # We've already done a bunch of work calculating the changes needed # for the sliding sync tables, so we may as well re-use that information diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 05faad5b26..d5e35fa7ca 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -86,7 +86,9 @@ class ProfileWorkerStore(SQLBaseStore): "populate_full_user_id_profiles", self.populate_full_user_id_profiles ) - self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._is_events_writer = self._instance_name in hs.config.worker.writers.events self._profile_updates_id_gen: MultiWriterIdGenerator = MultiWriterIdGenerator( db_conn=db_conn, @@ -403,6 +405,7 @@ class ProfileWorkerStore(SQLBaseStore): "get_updated_profile_updates", _get_updated_profile_updates_txn ) + # FIXME this function should be deleted, it's not used. async def get_profile_updates_for_fields( self, *, @@ -500,7 +503,7 @@ class ProfileWorkerStore(SQLBaseStore): from_id: int, to_id: int, user_id: str, - field_names: Set[str], + field_names: Set[str] | None, include_users: set[str] | None = None, ) -> list[ProfileUpdate]: """Get profile update markers for a user in a stream range. @@ -515,15 +518,16 @@ class ProfileWorkerStore(SQLBaseStore): to_id: The ending stream ID (inclusive). user_id: The full user ID to filter on. field_names: Set of field names to filter update actions against. + `None` means "include all fields". include_users: If given, only include updates for these user IDs. Returns: - A list of ProfileUpdates update rows. + A list of ProfileUpdate update rows, in stream order """ if from_id >= to_id: return [] - if len(field_names) == 0: + if field_names is not None and len(field_names) == 0: return [] if include_users is not None and len(include_users) == 0: @@ -533,22 +537,27 @@ class ProfileWorkerStore(SQLBaseStore): def _get_profile_updates_for_user_and_fields_txn( txn: LoggingTransaction, ) -> list[ProfileUpdate]: - wanted_field_in_elems_clause, wanted_field_in_elems_args = ( - make_in_list_sql_clause( + # Build a `field_clause` that matches updates containing the fields we are interested in + if field_names is None: + # We are interested in all fields, so match any update with fields + field_clause = "pu.affected_fields IS NOT NULL" + field_args: list[str] = [] + else: + wanted_field_in_elems_clause, field_args = make_in_list_sql_clause( txn.database_engine, "field_names.value", field_names ) - ) - if isinstance(txn.database_engine, PostgresEngine): - # Note that if we had a GIN index on `affected_fields`, this would defeat it. - # If we decide we want one, we should consider using the `?|` operator or its - # clearer-named `jsonb_exists_any` equivalent. - all_field_names_table_expression = "jsonb_array_elements_text(pu.affected_fields) AS field_names(value)" - else: - # json_each is a table-valued function that gives `value` as one of its column names - all_field_names_table_expression = ( - "json_each(pu.affected_fields) AS field_names" - ) + if isinstance(txn.database_engine, PostgresEngine): + # Note that if we had a GIN index on `affected_fields`, this would defeat it. + # If we decide we want one, we should consider using the `?|` operator or its + # clearer-named `jsonb_exists_any` equivalent. + all_field_names_table_expression = "jsonb_array_elements_text(pu.affected_fields) AS field_names(value)" + else: + # json_each is a table-valued function that gives `value` as one of its column names + all_field_names_table_expression = ( + "json_each(pu.affected_fields) AS field_names" + ) + field_clause = f"(EXISTS (SELECT 1 FROM {all_field_names_table_expression} WHERE {wanted_field_in_elems_clause}))" user_clause = "" user_args: list[str] = [] @@ -573,7 +582,7 @@ class ProfileWorkerStore(SQLBaseStore): AND puf.user_id = ? {user_clause} AND ( - (EXISTS (SELECT 1 FROM {all_field_names_table_expression} WHERE {wanted_field_in_elems_clause})) + {field_clause} OR pu.action != ? ) ORDER BY pu.stream_id ASC @@ -583,7 +592,7 @@ class ProfileWorkerStore(SQLBaseStore): to_id, user_id, *user_args, - *wanted_field_in_elems_args, + *field_args, ProfileUpdateAction.UPDATE.value, ), ) @@ -591,18 +600,20 @@ class ProfileWorkerStore(SQLBaseStore): updates: list[ProfileUpdate] = [] for stream_id, updated_user_id, action, affected_fields_dbjson in rows: + if affected_fields_dbjson is not None: + # Get the field names that were affected by this update + affected_fields = frozenset(db_to_json(affected_fields_dbjson)) + if field_names is not None: + # Only include the field names that we care about + affected_fields &= field_names + else: + affected_fields = None updates.append( ProfileUpdate( stream_id=stream_id, user_id=updated_user_id, action=action, - affected_fields=( - # Get the field names that were affected by this update - # and intersect with the field names we care about - frozenset(db_to_json(affected_fields_dbjson)) & field_names - ) - if affected_fields_dbjson is not None - else None, + affected_fields=affected_fields, ) ) @@ -755,7 +766,7 @@ class ProfileWorkerStore(SQLBaseStore): Returns: The profile updates stream ID that was created in this transaction """ - if self._msc4429_enabled: + if self._include_profile_updates_in_sync: assert self._is_events_writer self._check_profile_size(txn, user_id, field_name, new_value) @@ -818,7 +829,7 @@ class ProfileWorkerStore(SQLBaseStore): ), ) - if not self._msc4429_enabled: + if not self._include_profile_updates_in_sync: return None # Record updates in the profile updates stream @@ -847,7 +858,7 @@ class ProfileWorkerStore(SQLBaseStore): users profile should be pushed to the client, should they need it already even if the user hasn't actually joined the room. """ - if not self._msc4429_enabled: + if not self._include_profile_updates_in_sync: return assert self._is_events_writer @@ -884,7 +895,7 @@ class ProfileWorkerStore(SQLBaseStore): txn: Transaction to use user_id: User ID that made the profile update action: The profile update action, either `update`, `left_room` or - `joined_room` + `joined_room`. field_names: A list of fields that were set, if ProfileUpdateAction.UPDATE user_rooms: Optionally, a set of rooms that the update concerns. If not given, a database lookup will be done to fetch all the users rooms. @@ -895,7 +906,7 @@ class ProfileWorkerStore(SQLBaseStore): Returns: The latest stream ID created in this transaction """ - if not self._msc4429_enabled: + if not self._include_profile_updates_in_sync: return None if action == ProfileUpdateAction.UPDATE: @@ -1010,7 +1021,7 @@ class ProfileWorkerStore(SQLBaseStore): field_name: The name of the custom profile field. """ - if self._msc4429_enabled: + if self._include_profile_updates_in_sync: assert self._is_events_writer def delete_profile_field(txn: LoggingTransaction) -> int | None: @@ -1032,7 +1043,7 @@ class ProfileWorkerStore(SQLBaseStore): (f'$."{field_name}"', user_id.localpart), ) - if not self._msc4429_enabled: + if not self._include_profile_updates_in_sync: return None stream_id = self.record_profile_updates_txn( diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 499b114cb6..72ac8c7ed2 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -1041,12 +1041,26 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): return user_who_share_room - async def get_local_users_who_share_room_with_user(self, user_id: str) -> set[str]: - """Returns the set of local users who share a room with `user_id`. + async def get_local_users_who_share_room_with_user( + self, + user_id: str, + limit_to_rooms: set[str] | None = None, + ) -> set[str]: + """ + Returns the set of local users who share a room with `user_id`. This also includes the `user_id` themselves. + + Args: + user_id: The user ID to find the local users who share rooms. + limit_to_rooms: Optional set of rooms to limit to. + + Returns: + Set of local user ID's who share a room with the given user. """ room_ids = await self.get_rooms_for_user(user_id) + if limit_to_rooms is not None: + room_ids = room_ids.intersection(limit_to_rooms) user_who_share_room: set[str] = set() for room_id in room_ids: diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 3495dce866..0d9022116d 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -175,7 +175,7 @@ Changes in SCHEMA_VERSION = 93 Changes in SCHEMA_VERSION = 94 - Add `recheck` column (boolean, default true) to the `redactions` table. - MSC4242: Add state DAG tables. - - MSC4429: Track updates to user profile fields via a new stream. + - MSC4429/MSC4262: Track updates to user profile fields via a new stream. """ diff --git a/synapse/storage/schema/main/delta/94/07_profile_updates.sql b/synapse/storage/schema/main/delta/94/07_profile_updates.sql index 720f958e09..e053421be3 100644 --- a/synapse/storage/schema/main/delta/94/07_profile_updates.sql +++ b/synapse/storage/schema/main/delta/94/07_profile_updates.sql @@ -12,7 +12,7 @@ -- . -- Track updates to profile fields. --- For MSC4429 legacy /sync and others. +-- For MSC4429 and MSC4262 down sync and others. -- See https://github.com/element-hq/synapse/issues/19981 for potential future directions of this table. CREATE TABLE IF NOT EXISTS profile_updates ( stream_id BIGINT NOT NULL PRIMARY KEY, diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index d8c90c0fdd..f6d30460b5 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -442,6 +442,19 @@ class SlidingSyncResult: def __bool__(self) -> bool: return bool(self.room_id_to_sticky_events) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class ProfilesExtension: + """The Profile Updates extension (MSC4262) + + Attributes: + users: map (user_id -> [profile_updates]) + """ + + users: Mapping[str, JsonMapping | None] + + def __bool__(self) -> bool: + return bool(self.users) + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None @@ -449,6 +462,7 @@ class SlidingSyncResult: typing: TypingExtension | None = None thread_subscriptions: ThreadSubscriptionsExtension | None = None sticky_events: StickyEventsExtension | None = None + profiles: ProfilesExtension | None = None def __bool__(self) -> bool: """Are there any updates that should be returned immediately to @@ -461,6 +475,7 @@ class SlidingSyncResult: or self.typing or self.thread_subscriptions or self.sticky_events + or self.profiles ) next_pos: SlidingSyncStreamToken diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index a7cb4d8b08..353367c0d7 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -478,6 +478,18 @@ class SlidingSyncBody(RequestBodyModel): limit: NonNegativeStrictInt = 100 since: SlidingSyncStickyEventsToken | AbsentType = Absent + class ProfilesExtension(RequestBodyModel): + """The Profile Updates extension (MSC4262) + + Attributes: + enabled + fields: List of fields to filter upon (optional) + """ + + enabled: StrictBool = False + # Optionally filter on specific fields + fields: list[StrictStr] | AbsentType = Absent + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None @@ -489,6 +501,9 @@ class SlidingSyncBody(RequestBodyModel): sticky_events: StickyEventsExtension | AbsentType = Field( Absent, alias="org.matrix.msc4354.sticky_events" ) + profiles: ProfilesExtension | AbsentType = Field( + Absent, alias="org.matrix.msc4262.profiles" + ) conn_id: StrictStr | None = None lists: ( diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 5211bcc8e1..4ef1620a84 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -391,6 +391,7 @@ T4 = TypeVar("T4") T5 = TypeVar("T5") T6 = TypeVar("T6") T7 = TypeVar("T7") +T8 = TypeVar("T8") @overload @@ -544,6 +545,32 @@ async def gather_optional_coroutines( ]: ... +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + tuple[ + Coroutine[Any, Any, T1] | None, + Coroutine[Any, Any, T2] | None, + Coroutine[Any, Any, T3] | None, + Coroutine[Any, Any, T4] | None, + Coroutine[Any, Any, T5] | None, + Coroutine[Any, Any, T6] | None, + Coroutine[Any, Any, T7] | None, + Coroutine[Any, Any, T8] | None, + ] + ], +) -> tuple[ + T1 | None, + T2 | None, + T3 | None, + T4 | None, + T5 | None, + T6 | None, + T7 | None, + T8 | None, +]: ... + + async def gather_optional_coroutines( *coroutines: Unpack[tuple[Coroutine[Any, Any, T1] | None, ...]], ) -> tuple[T1 | None, ...]: diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index c53c04fbc8..590c7e62f5 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -199,13 +199,13 @@ class ProfileTestCase(unittest.HomeserverTestCase): ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], ] ) - def test_update_profile_does_not_update_stream_on_set_field_if_msc4429_not_enabled( + def test_update_profile_does_not_update_stream_on_set_field_if_include_profile_updates_in_sync_not_enabled( self, field_name: str, new_value: str, ) -> None: """Test that profile updates don't get recorded in the profile updates stream - if MSC4429 is not enabled.""" + if `include_profile_updates_in_sync` is not enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -230,13 +230,13 @@ class ProfileTestCase(unittest.HomeserverTestCase): ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], ] ) - def test_update_profile_does_not_notify_notifier_on_set_field_if_msc4429_not_enabled( + def test_update_profile_does_not_notify_notifier_on_set_field_if_include_profile_updates_in_sync_not_enabled( self, field_name: str, new_value: str, ) -> None: """Test that profile updates do not cause the profile updates stream notifier - to wake up if MSC4429 is not enabled.""" + to wake up if `include_profile_updates_in_sync` is not enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -265,7 +265,8 @@ class ProfileTestCase(unittest.HomeserverTestCase): self, field_name: str, new_value: str ) -> None: """Test that profile updates do not cause the profile updates stream notifier - to wake up if the user is not in any rooms, if MSC4429 is enabled.""" + to wake up if the user is not in any rooms, if `include_profile_updates_in_sync` + is enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -293,7 +294,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): self, field_name: str, new_value: str ) -> None: """Test that profile updates get recorded in the profile updates stream if - MSC4429 is enabled.""" + `include_profile_updates_in_sync` is enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -320,6 +321,8 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) fields_updates = self.get_success( + # FIXME this function should be deleted, it's not used. + # Adapt this test to use the right one. self.store.get_profile_updates_for_fields( from_id=1, to_id=2, @@ -361,7 +364,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): self, ) -> None: """Test that profiles updates get recorded in the 'per user' profile updates - stream tracking table, if MSC4429 is enabled.""" + stream tracking table, if `include_profile_updates_in_sync` is enabled.""" self.register_user("roger", "password") roger_token = self.login("roger", "password") self.register_user("millie", "password") @@ -501,7 +504,8 @@ class ProfileTestCase(unittest.HomeserverTestCase): self, ) -> None: """Test that previous profile update stream rows are removed for a user if - the user no longer shares rooms with another user, if MSC4429 is enabled. + the user no longer shares rooms with another user, if + `include_profile_updates_in_sync` is enabled. This test ensures that when a user leaves a room, we clear all old profile update rows of users who the user no longer shares rooms with, to avoid @@ -668,7 +672,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): new_value: str, ) -> None: """Test that profile updates wake up the profile updates stream on profile - field updates, if MSC4429 is enabled.""" + field updates, if `include_profile_updates_in_sync` is enabled.""" self.helper.create_room_as( room_creator=self.frank.to_string(), tok=self.frank_token, diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index bfb687a6c7..5771e60533 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1187,8 +1187,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): ) def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: - """Test that without MSC4429 enabled the initial sync response does not - contain any profile updates.""" + """Test that without `include_profile_updates_in_sync` enabled the initial sync + response does not contain any profile updates.""" self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.other_user), @@ -1212,8 +1212,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: - """Test that with MSC4429 enabled the initial sync response does not - contain any profile updates, if fields are not filtered for.""" + """Test that with `include_profile_updates_in_sync` enabled the initial sync + response does not contain any profile updates, if fields are not filtered for.""" self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.other_user), @@ -1240,9 +1240,9 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: - """Test that with MSC4429 enabled the initial sync response does - contain profile updates for users who share rooms, for the fields the - client requests. This response should include our syncing user.""" + """Test that with `include_profile_updates_in_sync` enabled the initial sync + response does contain profile updates for users who share rooms, for the fields + the client requests. This response should include our syncing user.""" self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.other_user), @@ -1304,8 +1304,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): def test_initial_sync_does_not_include_untracked_users_profile_updates( self, is_lazy: bool ) -> None: - """Test that with MSC4429 enabled the initial sync response does not - contain profile updates for users who do not share rooms.""" + """Test that with `include_profile_updates_in_sync` enabled the initial sync + response does not contain profile updates for users who do not share rooms.""" third_user = self.register_user("third_user", "password") self.get_success( self.profile_handler.set_field( @@ -1347,8 +1347,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( self, ) -> None: - """Test that with MSC4429 enabled the initial sync lazy loading response does - contain profile updates for events in the timeline. + """Test that with `include_profile_updates_in_sync` enabled the initial sync + lazy loading response does contain profile updates for events in the timeline. This test ensures lazy loading sync only returns profiles that we also have events for in the sync response. The second room in this test has the most @@ -1424,8 +1424,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): def test_incremental_sync_sends_down_profile_update_diffs( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync response does - contain profile update diffs.""" + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response does contain profile update diffs.""" requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( @@ -1499,9 +1499,9 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync lazy loading response - does contain profile updates even if the user would be filtered out by lazy - loading. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync lazy loading response does contain profile updates even if the user would + be filtered out by lazy loading. """ third_user = self.register_user("third_user", "password") third_tok = self.login("third_user", "password") @@ -1652,7 +1652,7 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): is_initial: bool, is_lazy: bool, ) -> None: - """Test that with MSC4429 enabled any sync response + """Test that with `include_profile_updates_in_sync` enabled any sync response doesn't contain federated users even if there are timeline events from them. """ # Join a federated user to the room, causing a membership event into @@ -1742,8 +1742,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): is_initial: bool, is_lazy: bool, ) -> None: - """Test that with MSC4429 enabled any sync response always contains the users - own updates. + """Test that with `include_profile_updates_in_sync` enabled any sync response + always contains the users own updates. This test is made with a user that is not in any rooms, to prove our code to collect interested users from the profile updates always collect the user. @@ -1830,8 +1830,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): is_initial: bool, is_lazy: bool, ) -> None: - """Test that with MSC4429 enabled a sync response correctly includes falsey - profile field values. + """Test that with `include_profile_updates_in_sync` enabled a sync response + correctly includes falsey profile field values. """ requester = create_requester(self.user) filter_json: dict[str, dict] = { @@ -1911,8 +1911,9 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_fields( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync lazy loading response - filters out unchanged profiles or fields we have recently sent to the client. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync lazy loading response filters out unchanged profiles or fields we have + recently sent to the client. """ requester = create_requester(self.user) self.get_success( @@ -2062,8 +2063,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync response - includes a 'null' for users who are no longer sharing rooms. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes a 'null' for users who are no longer sharing rooms. """ requester = create_requester(self.user) initial_result = self.get_success( @@ -2112,9 +2113,9 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joined( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync response - includes all the requested fields of a user who has joined a room with the - syncing user. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes all the requested fields of a user who has joined a room + with the syncing user. """ requester = create_requester(self.user) initial_result = self.get_success( @@ -2211,8 +2212,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): ) @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> None: - """Test that with MSC4429 enabled the incremental sync response includes - ones own profile updates.""" + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes ones own profile updates.""" requester = create_requester(self.user) filter_json: dict[str, dict] = { "org.matrix.msc4429.profile_fields": {"ids": ["m.status", "avatar_url"]} @@ -2275,8 +2276,8 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): eager_sync: bool, is_lazy: bool, ) -> None: - """Test that with MSC4429 enabled the incremental sync response - correctly handles multiple join / leave / join / leave in a row. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response correctly handles multiple join / leave / join / leave in a row. In the first variant we sync and check after each iteration of join/leave. In the second variant we only sync at the end of all the join/leaves. @@ -2440,9 +2441,9 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): value: str | bool | list | dict | int | float | None, new_value: str | bool | list | dict | int | float | None, ) -> None: - """Test that with MSC4429 enabled the incremental lazy sync response - includes all the profile update changes for the user, even if the profile - field has been recently sent and is in our lazy loading cache. + """Test that with `include_profile_updates_in_sync` enabled the incremental + lazy sync response includes all the profile update changes for the user, even + if the profile field has been recently sent and is in our lazy loading cache. Parameterize across different types of potential value types that profile field updates could have to ensure robustness. @@ -2559,9 +2560,9 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): def test_lazy_loading_cache_and_multiple_updates_to_the_same_field( self, ) -> None: - """Test that with MSC4429 enabled the incremental lazy sync response - includes an update to a field, even when the value changes back to a - value set and cached previously. + """Test that with `include_profile_updates_in_sync` enabled the incremental + lazy sync response includes an update to a field, even when the value changes + back to a value set and cached previously. """ requester = create_requester(self.user) filter_json = { diff --git a/tests/rest/client/sliding_sync/test_extension_profiles.py b/tests/rest/client/sliding_sync/test_extension_profiles.py new file mode 100644 index 0000000000..40d426fb87 --- /dev/null +++ b/tests/rest/client/sliding_sync/test_extension_profiles.py @@ -0,0 +1,1231 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import logging + +from parameterized import parameterized, parameterized_class + +from twisted.internet.testing import MemoryReactor + +import synapse.rest.admin +from synapse.api.constants import ProfileFields +from synapse.rest.client import knock, login, profile, room, sync +from synapse.server import HomeServer +from synapse.types import UserID, create_requester +from synapse.util.clock import Clock + +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.unittest import override_config + +logger = logging.getLogger(__name__) + + +# FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the +# foreground update for +# `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` (tracked by +# https://github.com/element-hq/synapse/issues/17623) +@parameterized_class( + ("use_new_tables",), + [ + (True,), + (False,), + ], + class_name_func=lambda cls, + num, + params_dict: f"{cls.__name__}_{'new' if params_dict['use_new_tables'] else 'fallback'}", +) +class SlidingSyncProfilesTestCase(SlidingSyncBase): + """Tests for the profile updates sliding sync extension""" + + servlets = [ + synapse.rest.admin.register_servlets, + knock.register_servlets, + login.register_servlets, + profile.register_servlets, + room.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.profile_handler = self.hs.get_profile_handler() + self.user = self.register_user("user", "password") + self.tok = self.login("user", "password") + self.other_user = self.register_user("other_user", "password") + self.other_tok = self.login("other_user", "password") + self.joined_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + super().prepare(reactor, clock, hs) + + @parameterized.expand( + [ + True, + False, + ] + ) + def test_no_data_when_not_enabled(self, is_initial: bool) -> None: + """ + Test that no profile extension response is returned + if the feature is not enabled. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + if not is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_no_data_initial_sync(self) -> None: + """ + Test that enabling the profiles extension works during an initial sync, + even if there is no-data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_no_data_incremental_sync(self) -> None: + """ + Test that enabling profiles extension works during an incremental sync, even + if there is no-data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make an incremental Sliding Sync request with the profiles extension enabled + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_fields_are_sent(self, is_initial: bool) -> None: + """ + Test that profile extension response returns field updates + in incremental and initial sync. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "room_subscriptions": { + self.joined_room: { + "required_state": [], + "timeline_limit": 10, + }, + }, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # We don't include room subscriptions, as we want to see updates coming + # through even without room subscriptions + del sync_body["room_subscriptions"] + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_field_then_deleted_does_not_error(self) -> None: + """ + Test that profile extension response does not crash if the user first + updates a field, then deletes it, and then the sync happens seeing both + the update and delete in the stream. + """ + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + # Update field + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="new value", + ) + ) + # Delete field + self.get_success( + self.profile_handler.delete_profile_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + ) + ) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "removed": [ + "field", + ], + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_fields_are_not_sent_if_not_requested( + self, is_initial: bool + ) -> None: + """ + Test that profile extension response doesn't return field updates we didn't + request in initial and incremental sync. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="anotherfield", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + # Nothing returned since we didn't ask for the updated field + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + if not is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="anotherfield", + new_value="value", + ) + ) + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # Nothing returned since we didn't ask for the updated field + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_fields_are_not_included_if_not_in_requested_rooms( + self, is_initial: bool + ) -> None: + """ + Test that profile extension response respects the room subscriptions, by: + * for initial sync returning updates for only those users in the given rooms + * for incremental sync returning all updates in shared rooms + """ + new_room = self.helper.create_room_as(self.user, tok=self.tok) + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "room_subscriptions": { + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + }, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + # Nothing returned since even though user and other_user share a room, + # we didn't ask for that room. + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + if not is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # Even though we only asked for a room other_user is not in, + # since these users share a room, updates are always sent via incremental + # sync. + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_all_fields_returned_if_no_fields_specified(self, is_initial: bool) -> None: + """ + Test that profile extension response returns all profile fields if we didn't + request any particular fields in initial and incremental sync. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + # We need to ensure a room is included to get things back in initial sync + "room_subscriptions": { + self.joined_room: { + "required_state": [], + "timeline_limit": 10, + }, + }, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + # As this is an initial sync, we get all profile fields + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "avatar_url": None, + "displayname": "other_user", + "field": "value", + } + }, + ) + + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # We don't include room subscriptions, as we want to see updates coming + # through even without room subscriptions + del sync_body["room_subscriptions"] + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # As this is an incremental sync, we only get actual updates back + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_null_profile_returned_if_user_left_all_rooms( + self, + request_fields: bool, + ) -> None: + """ + Test that profile extension response returns a null for the user in + incremental sync. + """ + # Make an initial Sliding Sync request with the profiles extension enabled + profiles_config: dict = { + "enabled": True, + } + if request_fields: + profiles_config["fields"] = ["field"] + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": profiles_config, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + self.helper.leave(self.joined_room, self.other_user, tok=self.other_tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # We should see a null profile + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_profile_returned_if_user_left_then_rejoined(self) -> None: + """ + Test that the profile extension response returns a profile, rather than a + null, for a user that left their last shared room and then rejoined it + within the same incremental sync window. + """ + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": {"enabled": True}, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + self.helper.leave(self.joined_room, self.other_user, tok=self.other_tok) + self.helper.join(self.joined_room, self.other_user, tok=self.other_tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # The rejoin overrides the leave, so we should see the full profile rather + # than a null profile. + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "displayname": "other_user", + # FIXME: This shouldn't be returned, but currently is + "avatar_url": None, + } + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_all_fields_returned_in_incremental_non_lazy_sync_if_someone_joined( + self, request_fields: bool + ) -> None: + """ + Test that profile extension response returns all profile fields in + incremental non-lazy sync, if someone joined the room.. + """ + # Make an initial Sliding Sync request with the profiles extension enabled + profiles_config: dict = { + "enabled": True, + } + if request_fields: + profiles_config["fields"] = ["displayname"] + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": profiles_config, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + third_user = self.register_user("third_user", "third_user") + third_tok = self.login(third_user, "third_user") + self.helper.join(self.joined_room, third_user, tok=third_tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + + expectation = { + "updated": { + "avatar_url": None, + "displayname": "third_user", + } + } + if request_fields: + expectation = { + "updated": { + "displayname": "third_user", + }, + } + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@third_user:test" + ], + expectation, + ) + + @parameterized.expand(["displayname", "avatar_url", "someotherfield"]) + @override_config({"include_profile_updates_in_sync": True}) + def test_removed_fields_get_sent_down_as_removed( + self, + field_name: str, + ) -> None: + """ + Test that we deliver clear/removed fields in the "removed" key in the response. + """ + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name=field_name, + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + # Delete the field + if field_name in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL): + self.get_success( + self.profile_handler.set_profile_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name=field_name, + new_value=None, + ) + ) + else: + self.get_success( + self.profile_handler.delete_profile_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name=field_name, + ) + ) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # We should see the removed field + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "removed": [ + field_name, + ], + }, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_key_only_present_if_updates(self) -> None: + """ + > The updated field SHOULD only be present if there are changes to existing fields on a user's profile. + """ + self.skipTest("Not yet implemented") + + @override_config({"include_profile_updates_in_sync": True}) + def test_rooms_subset_changing_includes_full_profile(self) -> None: + """ + > When a room enters this subset in this connection for the first time, all requested + > fields from profiles of users in that room MAY be sent down. This gives the client + > a base set of information for which future field updates can be applied on top of. + > The homeserver MAY omit some fields and profiles if it believes that the client has + > already received them, likewise repeat profiles MAY be sent down based on homeserver + > implementation. + """ + self.skipTest("Not yet implemented") + + @override_config({"include_profile_updates_in_sync": True}) + def test_fields_subset_changing_sends_down_field_even_if_not_changed(self) -> None: + """ + > Finally, if the list of fields expands to cover a new field ID, those fields should + > be sent down for all users that are within the current room subset. Future incremental + > updates will then include changes to this field. + """ + self.skipTest("Not yet implemented") + + @parameterized.expand( + [ + [True, True], + [True, False], + [False, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_down_full_profile_if_events_in_timeline( + self, + is_initial: bool, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading, only those members who have events in + the timeline get their profiles sent down in the sync response, for + rooms configured with lazy loading. + + Rooms without lazy loading should include all the members in initial sync, + none in incremental. + """ + # Create three users to fill the heroes + # Our heroes will thus be user, other_user and the three heroes here. + for i in range(3): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + fourth_user = self.register_user("fourth_user", "password") + fourth_tok = self.login("fourth_user", "password") + fifth_user = self.register_user("fifth_user", "password") + fifth_tok = self.login("fifth_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + self.helper.join( + room=self.joined_room, + user=fifth_user, + tok=fifth_tok, + ) + new_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=new_room, + user=fourth_user, + tok=fourth_tok, + ) + if is_initial: + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [], + "timeline_limit": 10, + }, + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [], + "timeline_limit": 10, + } + } + # We also need to specifically request the non-lazy room, otherwise + # our test to see if non-lazy members are also included will fail + sync_body["room_subscriptions"] = { + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + if is_initial: + if use_room_subsciptions: + sync_body["room_subscriptions"][self.joined_room]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + else: + sync_body["lists"]["foo-list"]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # Other user is a hero so should be included. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@other_user:test" + ) + ) + # Third user has events in the timeline, so should be here. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@third_user:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + # Fourth user is a member of a non-lazy configured room, so should be here. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fourth_user:test" + ) + ) + # Fifth user should be filtered out as they have no events in the room. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fifth_user:test" + ) + ) + + if not is_initial: + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + if use_room_subsciptions: + sync_body["room_subscriptions"][self.joined_room]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + else: + sync_body["lists"]["foo-list"]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # TODO check this if it's expected that heroes come down differently + # depending on if using a room subscription or a list + if use_room_subsciptions: + # Other user should be filtered out as heroes don't come down + # in incremental sync in the same way as initial sync, if the + # room is included via a room subscription. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"][ + "users" + ].get("@other_user:test") + ) + else: + # Other user should be included as heroes do come down + # in incremental sync in the same way as initial sync when the + # room is included in a list + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"][ + "users" + ].get("@other_user:test") + ) + # Third user has events in the timeline, so should be here. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@third_user:test" + ) + ) + # We are not included ourselves in incremental sync without updates. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + # Fourth user is a member of a non-lazy configured room, but had no updates, + # so shouldn't be here. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fourth_user:test" + ) + ) + # Fifth user should be excluded as they have no events. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fifth_user:test" + ) + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_full_profile_even_if_no_events_if_otherwise_included( + self, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading, if a user is in both a lazy loading room + and a non-lazy configured room, even if there are no events in the timeline, + their profile is sent down. + + This test only makes sense for initial sync, as for incremental we would + not expect to see users without timeline events if they had no profile updates. + """ + # Create some users to fill the heroes so they don't pollute the test. + for i in range(3): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + new_user = self.register_user("new_user", password="password") + new_tok = self.login("new_user", password="password") + new_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=self.joined_room, + user=new_user, + tok=new_tok, + ) + self.helper.join( + room=new_room, + user=new_user, + tok=new_tok, + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + }, + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + } + } + # We also need to specifically request the non-lazy room, otherwise + # our test to see if non-lazy members are also included will fail + sync_body["room_subscriptions"] = { + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # New user should be included as they are in a non-lazy room too, + # even though the lazy configured room had no events. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@new_user:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_full_profile_for_required_state_member_events( + self, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading for lazy rooms, even without timeline events, + we get profiles for users who have membership events in required_state. + + This test only makes sense for initial sync, as for incremental this would + happen via the `ProfileUpdateAction.JOINED_ROOM` events. + """ + # Create some users to fill the heroes so they don't pollute the test. + for i in range(3): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + new_user = self.register_user("new_user", password="password") + new_tok = self.login("new_user", password="password") + self.helper.join( + room=self.joined_room, + user=new_user, + tok=new_tok, + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [ + ["m.room.member", "$LAZY"], + ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [ + ["m.room.member", "$LAZY"], + ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + } + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # New user should be included as they joined the room and as such + # have membership events in required_state. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@new_user:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_full_profile_for_heroes( + self, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading for lazy rooms, even without timeline events or + required_state, we get profiles for room heroes. + + This test must ensure heroes don't get included in timeline_events + or required_state. + + This test only makes sense for initial sync, as for incremental sync + Synapse doesn't generate a room response without requesting state or + timeline events, thus no heroes either. + """ + # Create some users to fill the heroes + for i in range(4): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + not_hero = self.register_user("not_hero", "password") + not_hero_tok = self.login("not_hero", "password") + self.helper.join(self.joined_room, user=not_hero, tok=not_hero_tok) + + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + } + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # Other user should be included as they are a room hero + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@other_user:test" + ) + ) + # Not hero user should be excluded as they're not a hero + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@not_hero:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_repeat_of_sync_correctly_includes_profile_information_again(self) -> None: + """ + > Homeservers should only consider a profile field update "accepted" by a client + > once the client returns with a new /sync request with the next /sync token, + > NOT just after sending down the profile update. The client may never receive + > response due to network conditions, or a bug in the client implementation. + """ + self.skipTest("Not yet implemented") From d59856b9a79dbc52cd1740ae2235f0350dde6d28 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 25 Aug 2026 19:14:17 +0300 Subject: [PATCH 113/158] Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms (#20135) Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. Fixes https://github.com/element-hq/synapse/issues/20110 --- changelog.d/20135.bugfix | 1 + synapse/handlers/profile.py | 18 +++++++++++++++++ tests/handlers/test_profile.py | 36 +++++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20135.bugfix diff --git a/changelog.d/20135.bugfix b/changelog.d/20135.bugfix new file mode 100644 index 0000000000..1b51b21fa2 --- /dev/null +++ b/changelog.d/20135.bugfix @@ -0,0 +1 @@ +Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. \ No newline at end of file diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 3d052bc893..067a16262c 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -668,11 +668,20 @@ class ProfileHandler: if stream_id is not None: room_ids = await self.store.get_rooms_for_user(target_user.to_string()) if room_ids: + # Wake up the stream for the rooms involved self._notifier.on_new_event( StreamKeyType.PROFILE_UPDATES, stream_id, rooms=room_ids, ) + else: + # Wake up the stream for ourselves, as we might be updating our + # profile even if we don't have rooms + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + users=[target_user], + ) async def set_profile_field( self, @@ -807,11 +816,20 @@ class ProfileHandler: if stream_id: room_ids = await self.store.get_rooms_for_user(target_user.to_string()) if room_ids: + # Wake up the stream for the rooms involved self._notifier.on_new_event( StreamKeyType.PROFILE_UPDATES, stream_id, rooms=room_ids, ) + else: + # Wake up the stream for ourselves, as we might be updating our + # profile even if we don't have rooms + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + users=[target_user], + ) async def on_profile_query(self, args: JsonDict) -> JsonDict: """Handles federation profile query requests.""" diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 590c7e62f5..31e7e1c0a7 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -261,10 +261,10 @@ class ProfileTestCase(unittest.HomeserverTestCase): ] ) @override_config({"include_profile_updates_in_sync": True}) - def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_rooms( + def test_update_profile_does_notify_notifier_on_set_field_if_user_not_in_rooms( self, field_name: str, new_value: str ) -> None: - """Test that profile updates do not cause the profile updates stream notifier + """Test that profile updates does cause the profile updates stream notifier to wake up if the user is not in any rooms, if `include_profile_updates_in_sync` is enabled.""" self.get_success( @@ -280,7 +280,37 @@ class ProfileTestCase(unittest.HomeserverTestCase): for call in self.on_new_event.mock_calls if call.args[0] == StreamKeyType.PROFILE_UPDATES ] - self.assertEqual(len(calls_found), 0) + self.assertEqual(len(calls_found), 1) + + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_does_notify_notifier_on_delete_profile_field_if_user_not_in_rooms( + self, + ) -> None: + """Test that profile updates does cause the profile updates stream notifier + to wake up if the user is not in any rooms, if `include_profile_updates_in_sync` + is enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="field", + new_value="value", + ) + ) + self.on_new_event.reset_mock() + self.get_success( + self.handler.delete_profile_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="field", + ) + ) + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 1) @parameterized.expand( [ From 37b59059edf383b7ccd5586804b7c0455cf3a21f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:18:21 -0500 Subject: [PATCH 114/158] 1.160.0rc1 --- CHANGES.md | 35 +++++++++++++++++++++++++++++++ changelog.d/19875.doc | 1 - changelog.d/19974.feature | 1 - changelog.d/20003.feature | 2 -- changelog.d/20067.feature | 1 - changelog.d/20089.doc | 1 - changelog.d/20090.bugfix | 1 - changelog.d/20093.misc | 1 - changelog.d/20094.bugfix | 1 - changelog.d/20098.misc | 1 - changelog.d/20099.misc | 1 - changelog.d/20117.misc | 1 - changelog.d/20120.misc | 1 - changelog.d/20124.misc | 1 - changelog.d/20128.misc | 1 - changelog.d/20129.misc | 1 - changelog.d/20131.misc | 1 - changelog.d/20131.misc.1 | 1 - changelog.d/20135.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 22 files changed, 43 insertions(+), 21 deletions(-) delete mode 100644 changelog.d/19875.doc delete mode 100644 changelog.d/19974.feature delete mode 100644 changelog.d/20003.feature delete mode 100644 changelog.d/20067.feature delete mode 100644 changelog.d/20089.doc delete mode 100644 changelog.d/20090.bugfix delete mode 100644 changelog.d/20093.misc delete mode 100644 changelog.d/20094.bugfix delete mode 100644 changelog.d/20098.misc delete mode 100644 changelog.d/20099.misc delete mode 100644 changelog.d/20117.misc delete mode 100644 changelog.d/20120.misc delete mode 100644 changelog.d/20124.misc delete mode 100644 changelog.d/20128.misc delete mode 100644 changelog.d/20129.misc delete mode 100644 changelog.d/20131.misc delete mode 100644 changelog.d/20131.misc.1 delete mode 100644 changelog.d/20135.bugfix diff --git a/CHANGES.md b/CHANGES.md index 51c41cddb7..3fc546caea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,38 @@ +# Synapse 1.160.0rc1 (2026-08-25) + +## Features + +- Add experimental support for MSC4502: Targeted and unrestricted room member queries. ([\#19974](https://github.com/element-hq/synapse/issues/19974)) +- Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). + Currently defaults to not enabled, and is limited to local users only for the sync results. ([\#20003](https://github.com/element-hq/synapse/issues/20003)) +- Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. ([\#20067](https://github.com/element-hq/synapse/issues/20067)) + +## Bugfixes + +- Fix a bug where presence updates could stop being sent to clients (the presence stream position becoming stuck) if a `/sync` request was cancelled while a presence write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. ([\#20090](https://github.com/element-hq/synapse/issues/20090)) +- Thumbnail WebP images that use transparency as PNG rather than JPEG, to preserve transparency during thumbnailing. Contributed by @catfromplan9. ([\#20094](https://github.com/element-hq/synapse/issues/20094)) +- Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. ([\#20135](https://github.com/element-hq/synapse/issues/20135)) + +## Improved Documentation + +- Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo, a contribution from the JaxLUG, the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) +- Fix the documentation on the `federation_domain_whitelist` config option. ([\#20089](https://github.com/element-hq/synapse/issues/20089)) + +## Internal Changes + +- Update release script to check more often for actions being completed so you don't have to wait around as much. ([\#20093](https://github.com/element-hq/synapse/issues/20093)) +- Speed up the conversion of device list changes into outbound federation pokes, and add a metric for how far behind the conversion is. ([\#20098](https://github.com/element-hq/synapse/issues/20098)) +- Fix the schema diff CI not using `faketime` for SQLite. ([\#20099](https://github.com/element-hq/synapse/issues/20099)) +- Fix the schema diff CI breaking when the Rust module was changed. ([\#20117](https://github.com/element-hq/synapse/issues/20117), [\#20129](https://github.com/element-hq/synapse/issues/20129)) +- Reduce database CPU usage when marking device list changes as sent over federation. ([\#20120](https://github.com/element-hq/synapse/issues/20120)) +- Fix cache `__len__` of Sliding Sync `PerConnectionState` ignoring account data entries. ([\#20124](https://github.com/element-hq/synapse/issues/20124)) +- Update Synapse repo link in inconsistent stream error. ([\#20128](https://github.com/element-hq/synapse/issues/20128)) +- Update rustls-webpki to address GHSA-82j2-j2ch-gfr8. ([\#20131](https://github.com/element-hq/synapse/issues/20131)) +- Update pyo3 to address GHSA-36hh-v3qg-5jq4 and GHSA-chgr-c6px-7xpp. ([\#20131](https://github.com/element-hq/synapse/issues/20131)) + + + + # Synapse 1.159.0 (2026-08-18) No significant changes since 1.159.0rc1. diff --git a/changelog.d/19875.doc b/changelog.d/19875.doc deleted file mode 100644 index 6ab67a08a2..0000000000 --- a/changelog.d/19875.doc +++ /dev/null @@ -1 +0,0 @@ -Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo, a contribution from the JaxLUG, the Jacksonville Linux Users Group Inc.. diff --git a/changelog.d/19974.feature b/changelog.d/19974.feature deleted file mode 100644 index 4a9290ea76..0000000000 --- a/changelog.d/19974.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for MSC4502: Targeted and unrestricted room member queries. diff --git a/changelog.d/20003.feature b/changelog.d/20003.feature deleted file mode 100644 index add6b613c8..0000000000 --- a/changelog.d/20003.feature +++ /dev/null @@ -1,2 +0,0 @@ -Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). -Currently defaults to not enabled, and is limited to local users only for the sync results. \ No newline at end of file diff --git a/changelog.d/20067.feature b/changelog.d/20067.feature deleted file mode 100644 index 123d4df745..0000000000 --- a/changelog.d/20067.feature +++ /dev/null @@ -1 +0,0 @@ -Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. diff --git a/changelog.d/20089.doc b/changelog.d/20089.doc deleted file mode 100644 index e72db0dbde..0000000000 --- a/changelog.d/20089.doc +++ /dev/null @@ -1 +0,0 @@ -Fix the documentation on the `federation_domain_whitelist` config option. \ No newline at end of file diff --git a/changelog.d/20090.bugfix b/changelog.d/20090.bugfix deleted file mode 100644 index 69b8035a44..0000000000 --- a/changelog.d/20090.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where presence updates could stop being sent to clients (the presence stream position becoming stuck) if a `/sync` request was cancelled while a presence write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. diff --git a/changelog.d/20093.misc b/changelog.d/20093.misc deleted file mode 100644 index 60c745eb47..0000000000 --- a/changelog.d/20093.misc +++ /dev/null @@ -1 +0,0 @@ -Update release script to check more often for actions being completed so you don't have to wait around as much. diff --git a/changelog.d/20094.bugfix b/changelog.d/20094.bugfix deleted file mode 100644 index ebea9aaa04..0000000000 --- a/changelog.d/20094.bugfix +++ /dev/null @@ -1 +0,0 @@ -Thumbnail WebP images that use transparency as PNG rather than JPEG, to preserve transparency during thumbnailing. Contributed by @catfromplan9. diff --git a/changelog.d/20098.misc b/changelog.d/20098.misc deleted file mode 100644 index 0504be872d..0000000000 --- a/changelog.d/20098.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up the conversion of device list changes into outbound federation pokes, and add a metric for how far behind the conversion is. diff --git a/changelog.d/20099.misc b/changelog.d/20099.misc deleted file mode 100644 index f17b66915b..0000000000 --- a/changelog.d/20099.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the schema diff CI not using `faketime` for SQLite. \ No newline at end of file diff --git a/changelog.d/20117.misc b/changelog.d/20117.misc deleted file mode 100644 index 92d421117e..0000000000 --- a/changelog.d/20117.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the schema diff CI breaking when the Rust module was changed. \ No newline at end of file diff --git a/changelog.d/20120.misc b/changelog.d/20120.misc deleted file mode 100644 index d59db030c6..0000000000 --- a/changelog.d/20120.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce database CPU usage when marking device list changes as sent over federation. diff --git a/changelog.d/20124.misc b/changelog.d/20124.misc deleted file mode 100644 index d92a12ce08..0000000000 --- a/changelog.d/20124.misc +++ /dev/null @@ -1 +0,0 @@ -Fix cache `__len__` of Sliding Sync `PerConnectionState` ignoring account data entries. \ No newline at end of file diff --git a/changelog.d/20128.misc b/changelog.d/20128.misc deleted file mode 100644 index 1361eb0e5e..0000000000 --- a/changelog.d/20128.misc +++ /dev/null @@ -1 +0,0 @@ -Update Synapse repo link in inconsistent stream error. diff --git a/changelog.d/20129.misc b/changelog.d/20129.misc deleted file mode 100644 index 92d421117e..0000000000 --- a/changelog.d/20129.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the schema diff CI breaking when the Rust module was changed. \ No newline at end of file diff --git a/changelog.d/20131.misc b/changelog.d/20131.misc deleted file mode 100644 index a90fd7d4ad..0000000000 --- a/changelog.d/20131.misc +++ /dev/null @@ -1 +0,0 @@ -Update rustls-webpki to address GHSA-82j2-j2ch-gfr8. diff --git a/changelog.d/20131.misc.1 b/changelog.d/20131.misc.1 deleted file mode 100644 index c0c211ed8c..0000000000 --- a/changelog.d/20131.misc.1 +++ /dev/null @@ -1 +0,0 @@ -Update pyo3 to address GHSA-36hh-v3qg-5jq4 and GHSA-chgr-c6px-7xpp. diff --git a/changelog.d/20135.bugfix b/changelog.d/20135.bugfix deleted file mode 100644 index 1b51b21fa2..0000000000 --- a/changelog.d/20135.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 8a6da09864..a0325ac3bc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.160.0~rc1) stable; urgency=medium + + * New synapse release 1.160.0rc1. + + -- Synapse Packaging team Tue, 25 Aug 2026 16:17:54 +0000 + matrix-synapse-py3 (1.159.0) stable; urgency=medium * New synapse release 1.159.0. diff --git a/pyproject.toml b/pyproject.toml index 8e84e6d37d..751b2805a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.159.0" +version = "1.160.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index a8d1719288..59791a4a1f 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.159/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.160/synapse-config.schema.json type: object properties: modules: From 5c0e07221415d964b1a34b6f15ba421c30f8573f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:19:31 -0500 Subject: [PATCH 115/158] Link MSC --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3fc546caea..270a8eb013 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ ## Features -- Add experimental support for MSC4502: Targeted and unrestricted room member queries. ([\#19974](https://github.com/element-hq/synapse/issues/19974)) +- Add experimental support for [MSC4502](https://github.com/matrix-org/matrix-spec-proposals/pull/4502): Targeted and unrestricted room member queries. ([\#19974](https://github.com/element-hq/synapse/issues/19974)) - Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). Currently defaults to not enabled, and is limited to local users only for the sync results. ([\#20003](https://github.com/element-hq/synapse/issues/20003)) - Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. ([\#20067](https://github.com/element-hq/synapse/issues/20067)) From 2f2ee47d0a0c2b02b068ea3ce85e0ce5cd774b43 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:20:18 -0500 Subject: [PATCH 116/158] Remove negative phrasing --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 270a8eb013..d0e85623cc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ - Add experimental support for [MSC4502](https://github.com/matrix-org/matrix-spec-proposals/pull/4502): Targeted and unrestricted room member queries. ([\#19974](https://github.com/element-hq/synapse/issues/19974)) - Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). - Currently defaults to not enabled, and is limited to local users only for the sync results. ([\#20003](https://github.com/element-hq/synapse/issues/20003)) + Currently defaults to disabled, and is limited to local users only for the sync results. ([\#20003](https://github.com/element-hq/synapse/issues/20003)) - Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. ([\#20067](https://github.com/element-hq/synapse/issues/20067)) ## Bugfixes From 4172c8284198ac83b8fdf631d5fde4d1a0664b13 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:22:06 -0500 Subject: [PATCH 117/158] Standardize attribution --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d0e85623cc..c265a0a839 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ ## Improved Documentation -- Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo, a contribution from the JaxLUG, the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) +- Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo. Contributed by JaxLUG from the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) - Fix the documentation on the `federation_domain_whitelist` config option. ([\#20089](https://github.com/element-hq/synapse/issues/20089)) ## Internal Changes From 44188b5ceb54fb3a81cb2bcefc030fae4730ac79 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:25:26 -0500 Subject: [PATCH 118/158] Link GitHub security advisories --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c265a0a839..edfe161d22 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,8 +27,8 @@ - Reduce database CPU usage when marking device list changes as sent over federation. ([\#20120](https://github.com/element-hq/synapse/issues/20120)) - Fix cache `__len__` of Sliding Sync `PerConnectionState` ignoring account data entries. ([\#20124](https://github.com/element-hq/synapse/issues/20124)) - Update Synapse repo link in inconsistent stream error. ([\#20128](https://github.com/element-hq/synapse/issues/20128)) -- Update rustls-webpki to address GHSA-82j2-j2ch-gfr8. ([\#20131](https://github.com/element-hq/synapse/issues/20131)) -- Update pyo3 to address GHSA-36hh-v3qg-5jq4 and GHSA-chgr-c6px-7xpp. ([\#20131](https://github.com/element-hq/synapse/issues/20131)) +- Update rustls-webpki to address [GHSA-82j2-j2ch-gfr8](https://github.com/advisories/GHSA-82j2-j2ch-gfr8). ([\#20131](https://github.com/element-hq/synapse/issues/20131)) +- Update pyo3 to address [GHSA-36hh-v3qg-5jq4](https://github.com/advisories/GHSA-36hh-v3qg-5jq4) and [GHSA-chgr-c6px-7xpp](https://github.com/advisories/GHSA-chgr-c6px-7xpp). ([\#20131](https://github.com/element-hq/synapse/issues/20131)) From 9806ac4dbbcaf2a62c3c25651c0a2b9b4637916c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:36:35 -0500 Subject: [PATCH 119/158] Generic stream ID fix --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index edfe161d22..dd9f64ed03 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,7 +9,7 @@ ## Bugfixes -- Fix a bug where presence updates could stop being sent to clients (the presence stream position becoming stuck) if a `/sync` request was cancelled while a presence write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. ([\#20090](https://github.com/element-hq/synapse/issues/20090)) +- Fix a bug where stream positions (presence, to-device message, etc.) could stop being sent to clients if a request was cancelled while a write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. ([\#20090](https://github.com/element-hq/synapse/issues/20090)) - Thumbnail WebP images that use transparency as PNG rather than JPEG, to preserve transparency during thumbnailing. Contributed by @catfromplan9. ([\#20094](https://github.com/element-hq/synapse/issues/20094)) - Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. ([\#20135](https://github.com/element-hq/synapse/issues/20135)) From d27933f121a877313beff4b0008c51c327cb721a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:38:07 -0500 Subject: [PATCH 120/158] Remove specific example source that doesn't read well --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index dd9f64ed03..670a61782e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ ## Improved Documentation -- Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo. Contributed by JaxLUG from the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) +- Document lighttpd reverse proxy configuration example. Contributed by JaxLUG from the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) - Fix the documentation on the `federation_domain_whitelist` config option. ([\#20089](https://github.com/element-hq/synapse/issues/20089)) ## Internal Changes From 781d08df263e640d85ca5c8b5ff514433e019c6f Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 26 Aug 2026 14:27:28 +0100 Subject: [PATCH 121/158] Fix the `/profile` endpoint giving 500 when the profile does not exist at all. (#20149) Drive-by fix for a problem noticed when doing some other work. This only triggers for 'custom'/generic fields, not the built-in `displayname` and `avatar_url`. Not seeing an open issue for it. Return a 404, not 500, when looking up a profile field on a missing profile --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/20149.bugfix | 1 + synapse/storage/databases/main/profile.py | 16 ++++++++++++---- tests/storage/test_profile.py | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 changelog.d/20149.bugfix diff --git a/changelog.d/20149.bugfix b/changelog.d/20149.bugfix new file mode 100644 index 0000000000..2440618acf --- /dev/null +++ b/changelog.d/20149.bugfix @@ -0,0 +1 @@ +Fix the `/profile` endpoint, when querying custom fields, returning a 500 error instead of 404 when the profile does not exist at all. \ No newline at end of file diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index d5e35fa7ca..fe48636ffb 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -291,11 +291,14 @@ class ProfileWorkerStore(SQLBaseStore): (field_path, field_name, user_id.localpart), ) + row = txn.fetchone() + if row is None: + # The user has no profile at all. + raise StoreError(404, "No row found") + # Test exists first since value being None is used for both # missing and a null JSON value. - exists, value = cast( - tuple[bool, JsonValue | dict[str, JsonValue]], txn.fetchone() - ) + exists, value = cast(tuple[bool, JsonValue | dict[str, JsonValue]], row) if not exists: raise StoreError(404, "No row found") return value @@ -310,9 +313,14 @@ class ProfileWorkerStore(SQLBaseStore): (field_path, field_path, user_id.localpart), ) + row = txn.fetchone() + if row is None: + # The user has no profile at all. + raise StoreError(404, "No row found") + # If value_type is None, then the value did not exist. value_type, value = cast( - tuple[str | None, JsonValue | dict[str, JsonValue]], txn.fetchone() + tuple[str | None, JsonValue | dict[str, JsonValue]], row ) if not value_type: raise StoreError(404, "No row found") diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index b65af28962..0fc40794c7 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -19,9 +19,12 @@ # # +from http import HTTPStatus + from twisted.internet.testing import MemoryReactor from synapse.api.constants import ProfileFields +from synapse.api.errors import StoreError from synapse.server import HomeServer from synapse.storage.database import LoggingTransaction from synapse.storage.engines import PostgresEngine @@ -95,6 +98,20 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): self.get_success(self.store.get_profile_avatar_url(self.u_frank)) ) + def test_get_profile_field_without_profile(self) -> None: + """ + Getting a custom profile field for a user that has no row in the + `profiles` table at all should raise a 404. + + Regression test (we previously would trigger an unhandled exception). + Can happen for users whose profile was erased upon deactivation. + """ + f = self.get_failure( + self.store.get_profile_field(self.u_frank, "org.example.field"), + StoreError, + ) + self.assertEqual(f.value.code, HTTPStatus.NOT_FOUND) + def test_profiles_bg_migration(self) -> None: """ Test background job that copies entries from column user_id to full_user_id, adding From 4245bdc6f048e8682751fd8092666184bb4033fb Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 27 Aug 2026 09:29:50 +0300 Subject: [PATCH 122/158] Fix sending down removed custom profile fields in legacy sync (#20147) When a client (correctly) calls the `DELETE` endpoint to remove a custom profile field (like Element X does with `m.status`), we incorrectly don't include it in the sync response in legacy sync. This was due to the fact that we cleaned up the sent fields down to what fields the profile currently has. Always ensure any fields in `ProfileUpdateAction.UPDATE` are sent down, as `null` values for profile fields which have been deleted. Fixes an issue where clearing a user status from Element X does not reflect in the user status being cleared on Element Web. Note, target is the v1.160.0 release branch due to customer commitments, and this fixes web and mobile clients not working together correctly. ### Pull Request Checklist * [ ] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20147.bugfix | 1 + synapse/handlers/sync.py | 37 +++++++++++-- tests/handlers/test_sync.py | 100 +++++++++++++++++++++++++++++++++++- 3 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20147.bugfix diff --git a/changelog.d/20147.bugfix b/changelog.d/20147.bugfix new file mode 100644 index 0000000000..74768dc63e --- /dev/null +++ b/changelog.d/20147.bugfix @@ -0,0 +1 @@ +Fix sending custom profile field removals to legacy sync clients when the field is deleted using the profile field delete endpoint. \ No newline at end of file diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 943105415a..f8a766feee 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2420,8 +2420,22 @@ class SyncHandler: if include_users and other_user_id in include_users: # Include all the fields the client asked for, as this user # has events in a lazy loaded sync response, except for - # fields we've recently sent in a previous lazy loaded sync response - fields = set(profile_data.keys()).intersection(profile_fields) + # fields we've recently sent in a previous lazy loaded sync response. + # We must include _updated_ fields even if the profile doesn't have + # this field. The value will be sent down as `None`. We must do + # this as currently legacy sync delivers field removals by + # delivering a null value to clients, and if a field is completely + # deleted, we can't otherwise do that. The fact this field has + # a `ProfileUpdateAction.UPDATE` is enough to tell us it should + # be sent down. + # TODO once removals are sent down in a dedicated key instead of + # null values, the `.union(updated_user_fields.get(other_user_id, []))` + # part here can be removed. + fields = ( + set(profile_data.keys()) + .union(updated_user_fields.get(other_user_id, [])) + .intersection(profile_fields) + ) for field_name in fields: cache_key = ( sync_config.user.to_string(), @@ -2469,9 +2483,24 @@ class SyncHandler: if other_user_id in joined_room_user_ids else set(updated_user_fields.get(other_user_id, [])) ) - fields = set(profile_data.keys()).intersection(fields) + # We must include _updated_ fields even if the profile doesn't have + # this field. The value will be sent down as `None`. We must do + # this as currently legacy sync delivers field removals by + # delivering a null value to clients, and if a field is completely + # deleted, we can't otherwise do that. The fact this field has + # a `ProfileUpdateAction.UPDATE` is enough to tell us it should + # be sent down. + # TODO once removals are sent down in a dedicated key instead of + # null values, the `.union(updated_user_fields.get(other_user_id, []))` + # part here can be removed. + fields = ( + set(profile_data.keys()) + .union(updated_user_fields.get(other_user_id, [])) + .intersection(fields) + ) + # fields.update(set(updated_user_fields.get(other_user_id, []))) for field_name in fields: - per_user_updates[field_name] = profile_data[field_name] + per_user_updates[field_name] = profile_data.get(field_name) if per_user_updates: profile_updates[other_user_id] = per_user_updates diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 5771e60533..b732c501d9 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -26,7 +26,7 @@ from parameterized import parameterized, parameterized_class from twisted.internet import defer from twisted.internet.testing import MemoryReactor -from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules +from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules, ProfileFields from synapse.api.errors import Codes, ResourceLimitError from synapse.api.filtering import FilterCollection, Filtering from synapse.api.room_versions import RoomVersion, RoomVersions @@ -2109,6 +2109,104 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): incremental_result.profile_updates["@other_user:test"], ) + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_deleted_fields(self, is_lazy: bool) -> None: + """ + Tests that, with `include_profile_updates_in_sync` enabled, + an incremental sync returns deleted fields as a `null` value, both for + `displayname` (stored as its own column) and for + generic custom fields (stored as JSON). + """ + # Set up a user with `displayname` and `m.status` profile fields + requester = create_requester(self.user) + other_requester = create_requester(self.other_user) + other_user = UserID.from_string(self.other_user) + filter_json: dict = { + "org.matrix.msc4429.profile_fields": {"ids": ["displayname", "m.status"]}, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + sync_config = generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ) + self.get_success( + self.profile_handler.set_field( + target_user=other_user, + requester=other_requester, + field_name=ProfileFields.DISPLAYNAME, + new_value="Bob", + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=other_user, + requester=other_requester, + field_name="m.status", + new_value={"text": "On holiday"}, + ) + ) + + # Do an initial sync after the point of those fields being set + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=sync_config, + request_key=generate_request_key(), + ) + ) + self.assertEqual( + initial_result.profile_updates["@other_user:test"], + {"displayname": "Bob", "m.status": {"text": "On holiday"}}, + ) + + # Delete the displayname and the `m.status` profile fields + self.get_success( + self.profile_handler.set_field( + target_user=other_user, + requester=other_requester, + field_name=ProfileFields.DISPLAYNAME, + new_value="", + ) + ) + self.get_success( + self.profile_handler.delete_profile_field( + target_user=other_user, + requester=other_requester, + field_name="m.status", + ) + ) + + # Do an incremental sync. + # Expect the deletion of both fields to be communicated in it. + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=sync_config, + request_key=generate_request_key(), + ) + ) + self.assertEqual( + incremental_result.profile_updates, + # We currently represent deleted fields as `null`, even though + # it's ambiguous (TODO MSC change) + {"@other_user:test": {"displayname": None, "m.status": None}}, + ) + @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joined( self, From c246d81dec17bbb0d52e4d1b284e80b57b632d69 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Thu, 27 Aug 2026 13:21:47 +0200 Subject: [PATCH 123/158] Add support for specifying the SFU WebSocket URL together with configured LiveKit transports (#20146) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Signed-off-by: Johannes Marbach --- changelog.d/20146.feature | 1 + docs/upgrade.md | 14 ++++ .../configuration/config_documentation.md | 9 ++- schema/synapse-config.schema.yaml | 21 +++++- synapse/config/matrixrtc.py | 26 +++++-- tests/rest/client/test_matrixrtc.py | 71 ++++++++++++++++++- 6 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 changelog.d/20146.feature diff --git a/changelog.d/20146.feature b/changelog.d/20146.feature new file mode 100644 index 0000000000..262ae828f8 --- /dev/null +++ b/changelog.d/20146.feature @@ -0,0 +1 @@ +Deprecate `livekit_service_url` and add support for specifying the SFU WebSocket URL for configured LiveKit transports. Please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11600). diff --git a/docs/upgrade.md b/docs/upgrade.md index a3c529f1f1..9eac9c5ebc 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -118,6 +118,20 @@ stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.161.0 + +## Deprecation of `matrix_rtc.livekit_service_url` + +When configuring the MatrixRTC LiveKit transport, the `livekit_service_url` is now +deprecated but should continue to be listed to ensure backwards compatibility with +older clients. A new sibling `url` config property is added that should be set to +your SFU's WebSocket URL. Clients that support `url` will use the Client-Server API +to (indirectly) interact with the LiveKit authorization service. The service needs +to be set up as an application service in order to support these endpoints. See +https://github.com/element-hq/lk-jwt-service and +https://element-hq.github.io/synapse/v1.161/usage/configuration/config_documentation.html#matrix_rtc +for further details. + # Upgrading to v1.159.0 ## Change of signing key expiry date for the Debian/Ubuntu package repository (2026) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 00a31ff447..876b74d5d3 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2662,13 +2662,20 @@ This setting has the following sub-options: * `type` (string): The type of transport to use to connect to the selective forwarding unit (SFU). - * `livekit_service_url` (string): The base URL of the LiveKit service. Should only be used with LiveKit-based transports. + * `url` (string): The WebSocket URL of the LiveKit SFU. If type is "livekit", either this or `livekit_service_url` is required. + + Clients that support `url` will use the Client-Server API to (indirectly) interact with the LiveKit authorization service. The service needs to be set up as an application service in order to support these endpoints. See https://github.com/element-hq/lk-jwt-service for further details. + + * `livekit_service_url` (string): Deprecated. The HTTP URL of the LiveKit authorization service. If type is "livekit", either this or `url` is required. + + Clients that don't support `url` will use `livekit_service_url` to directly interact with the LiveKit authorization service. This mode of operation is deprecated and should only be used for backwards compatibility. Example configuration: ```yaml matrix_rtc: transports: - type: livekit + url: wss://livekit.example.com livekit_service_url: https://matrix-rtc.example.com/livekit/jwt ``` --- diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 59791a4a1f..cfa2cd4a41 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -3022,10 +3022,28 @@ properties: type: string description: The type of transport to use to connect to the selective forwarding unit (SFU). example: livekit + url: + type: string + description: >- + The WebSocket URL of the LiveKit SFU. If type is "livekit", either this or `livekit_service_url` is + required. + + + Clients that support `url` will use the Client-Server API to (indirectly) interact with the LiveKit + authorization service. The service needs to be set up as an application service in order to support + these endpoints. See https://github.com/element-hq/lk-jwt-service for further details. + example: + wss://livekit.example.com livekit_service_url: type: string description: >- - The base URL of the LiveKit service. Should only be used with LiveKit-based transports. + Deprecated. The HTTP URL of the LiveKit authorization service. If type is "livekit", either this or `url` is + required. + + + Clients that don't support `url` will use `livekit_service_url` to directly interact with the LiveKit + authorization service. This mode of operation is deprecated and should only be used for backwards + compatibility. example: https://matrix-rtc.example.com/livekit/jwt description: A list of transport types and arguments to use for MatrixRTC connections. default: [] @@ -3033,6 +3051,7 @@ properties: examples: - transports: - type: livekit + url: wss://livekit.example.com livekit_service_url: https://matrix-rtc.example.com/livekit/jwt enable_registration: type: boolean diff --git a/synapse/config/matrixrtc.py b/synapse/config/matrixrtc.py index 84c245e286..896317fca8 100644 --- a/synapse/config/matrixrtc.py +++ b/synapse/config/matrixrtc.py @@ -17,7 +17,7 @@ from typing import Any -from pydantic import Field, StrictStr, ValidationError, model_validator +from pydantic import Field, StrictStr, ValidationError, field_validator, model_validator from typing_extensions import Self from synapse.types import JsonDict @@ -29,20 +29,34 @@ from ._base import Config, ConfigError class TransportConfigModel(ParseModel): type: StrictStr + url: StrictStr | None = Field(default=None) + """An optional WebSocket URL pointing to the LiveKit SFU. If type is "livekit", either this or livekit_service_url is required.""" + livekit_service_url: StrictStr | None = Field(default=None) - """An optional livekit service URL. Only required if type is "livekit".""" + """Deprecated. An optional HTTP URL pointing to the LiveKit authorization service. If type is "livekit", either this or url is required.""" @model_validator(mode="after") - def validate_livekit_service_url(self) -> Self: - if self.type == "livekit" and not self.livekit_service_url: + def validate_livekit_transport(self) -> Self: + if self.type == "livekit" and not self.url and not self.livekit_service_url: raise ValueError( - "You must set a `livekit_service_url` when using the 'livekit' transport." + "You must set either `url` or `livekit_service_url` when using the 'livekit' transport." ) return self class MatrixRtcConfigModel(ParseModel): - transports: list = [] + transports: list[dict[str, Any]] = [] + + @field_validator("transports") + @classmethod + def validate_transports( + cls, transports: list[dict[str, Any]] + ) -> list[dict[str, Any]]: + """Validate each transport by attempting to construct a `TransportConfigModel` + from it, raising a `ValidationError` if construction fails.""" + for transport in transports: + TransportConfigModel(**transport) + return transports class MatrixRtcConfig(Config): diff --git a/tests/rest/client/test_matrixrtc.py b/tests/rest/client/test_matrixrtc.py index f2bf1596be..63c7632a88 100644 --- a/tests/rest/client/test_matrixrtc.py +++ b/tests/rest/client/test_matrixrtc.py @@ -17,8 +17,13 @@ """Tests REST events for /rtc/endpoints path.""" +import unittest as stdlib_unittest + +from pydantic import ValidationError + from twisted.internet.testing import MemoryReactor +from synapse.config.matrixrtc import TransportConfigModel from synapse.rest import admin from synapse.rest.client import login, matrixrtc, register, room, versions from synapse.server import HomeServer @@ -30,7 +35,16 @@ from tests.unittest import HomeserverTestCase, override_config PATH_PREFIX = "/_matrix/client/unstable/org.matrix.msc4143" RTC_ENDPOINT = {"type": "focusA", "required_field": "theField"} -LIVEKIT_ENDPOINT = { +LIVEKIT_TRANSPORT = { + "type": "livekit", + "url": "wss://livekit.example.com", +} +BACKWARDS_COMPATIBLE_LIVEKIT_TRANSPORT = { + "type": "livekit", + "url": "wss://livekit.example.com", + "livekit_service_url": "https://livekit.example.com", +} +LEGACY_LIVEKIT_TRANSPORT = { "type": "livekit", "livekit_service_url": "https://livekit.example.com", } @@ -96,7 +110,7 @@ class MatrixRtcTestCase(HomeserverTestCase): @override_config( { "experimental_features": {"msc4143_enabled": True}, - "matrix_rtc": {"transports": [LIVEKIT_ENDPOINT]}, + "matrix_rtc": {"transports": [LIVEKIT_TRANSPORT]}, } ) def test_matrixrtc_endpoint_livekit_transport(self) -> None: @@ -104,7 +118,38 @@ class MatrixRtcTestCase(HomeserverTestCase): "GET", f"{PATH_PREFIX}/rtc/transports", access_token=self._alice_tok ) self.assertEqual(200, channel.code, channel.json_body) - self.assert_dict({"rtc_transports": [LIVEKIT_ENDPOINT]}, channel.json_body) + self.assert_dict({"rtc_transports": [LIVEKIT_TRANSPORT]}, channel.json_body) + + @override_config( + { + "experimental_features": {"msc4143_enabled": True}, + "matrix_rtc": {"transports": [BACKWARDS_COMPATIBLE_LIVEKIT_TRANSPORT]}, + } + ) + def test_matrixrtc_endpoint_backwards_compatible_livekit_transport(self) -> None: + channel = self.make_request( + "GET", f"{PATH_PREFIX}/rtc/transports", access_token=self._alice_tok + ) + self.assertEqual(200, channel.code, channel.json_body) + self.assert_dict( + {"rtc_transports": [BACKWARDS_COMPATIBLE_LIVEKIT_TRANSPORT]}, + channel.json_body, + ) + + @override_config( + { + "experimental_features": {"msc4143_enabled": True}, + "matrix_rtc": {"transports": [LEGACY_LIVEKIT_TRANSPORT]}, + } + ) + def test_matrixrtc_endpoint_legacy_livekit_transport(self) -> None: + channel = self.make_request( + "GET", f"{PATH_PREFIX}/rtc/transports", access_token=self._alice_tok + ) + self.assertEqual(200, channel.code, channel.json_body) + self.assert_dict( + {"rtc_transports": [LEGACY_LIVEKIT_TRANSPORT]}, channel.json_body + ) class MatrixRtcVersionsTestCase(HomeserverTestCase): @@ -150,3 +195,23 @@ class MatrixRtcVersionsTestCase(HomeserverTestCase): channel = self.make_request("GET", "/_matrix/client/versions") self.assertEqual(channel.code, 200, channel.result) self.assertTrue(channel.json_body["unstable_features"]["org.matrix.msc4143"]) + + +class TransportConfigModelTestCase(stdlib_unittest.TestCase): + """Tests validation of the `TransportConfigModel` pydantic model.""" + + def test_livekit_transport_requires_url_or_livekit_service_url(self) -> None: + with self.assertRaises(ValidationError): + TransportConfigModel(type="livekit") + + def test_livekit_transport_with_only_url(self) -> None: + TransportConfigModel(type="livekit", url="wss://livekit.example.com") + + def test_livekit_transport_with_only_livekit_service_url(self) -> None: + TransportConfigModel( + type="livekit", livekit_service_url="https://livekit.example.com" + ) + + def test_invalid_field_type(self) -> None: + with self.assertRaises(ValidationError): + TransportConfigModel(type="livekit", url=1234) # type: ignore[arg-type] From d9f3a9d4fa87c5f0b11b8acc950800fa3a73e538 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 27 Aug 2026 14:42:41 +0100 Subject: [PATCH 124/158] Expand `synapse.module_api.__all__` to include more of the publicly exposed types/helpers (#20107) I *think* the intention is that `__all__` can be used to access all the types that are exposed via the module API. However, some of them are missing. This PR adds those that are directly referenced but were not exposed. There are some others that are indirectly referenced that I've not included (e.g. `synapse.spam_checker_api.RegistrationBehaviour`). --- changelog.d/20107.misc | 1 + synapse/module_api/__init__.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 changelog.d/20107.misc diff --git a/changelog.d/20107.misc b/changelog.d/20107.misc new file mode 100644 index 0000000000..fb7a845252 --- /dev/null +++ b/changelog.d/20107.misc @@ -0,0 +1 @@ +Declare types that already appear in the module API's public signatures (such as `Requester`, `SynapseRequest` and `UserInfo`) in `synapse.module_api.__all__`. diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 1be0e9fd91..58ccddc792 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -194,14 +194,21 @@ __all__ = [ "run_in_background", "run_as_background_process", "cached", + "CachedFunction", "NOT_SPAM", "UserID", + "DomainSpecificString", "DatabasePool", "LoggingTransaction", "DirectServeHtmlResource", "DirectServeJsonResource", + "SimpleHttpClient", + "SynapseRequest", "ModuleApi", + "AccountDataManager", + "PublicRoomListManager", "PRESENCE_ALL_USERS", + "UserPresenceState", "LoginResponse", "JsonDict", "JsonMapping", @@ -209,7 +216,13 @@ __all__ = [ "StateMap", "ProfileInfo", "RoomAlias", + "RoomID", + "Requester", + "UserInfo", + "UserIpAndAgent", "UserProfile", + "Absent", + "AbsentType", "RatelimitOverride", "MediaUploadLimit", ] From bf68c63605214452af56e5165a22210c311ced11 Mon Sep 17 00:00:00 2001 From: V02460 Date: Thu, 27 Aug 2026 15:55:41 +0200 Subject: [PATCH 125/158] Make `federation_domain_whitelist` nullable in config schema (#20140) Aligns the config schema of `federation_domain_whitelist` more closely with its semantics and its actual implementation in Synapse. This is done by making it nullable. - all behavior of the option is now achievable by setting the config option to an actual value - aligned behavior of the property between unset and null - provide a definitive default value This also fixes the formatting problem with separating the paragraphs in the documentation. --- changelog.d/20140.doc | 1 + docs/usage/configuration/config_documentation.md | 11 ++++++++--- schema/synapse-config.schema.yaml | 8 ++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20140.doc diff --git a/changelog.d/20140.doc b/changelog.d/20140.doc new file mode 100644 index 0000000000..9d6432631f --- /dev/null +++ b/changelog.d/20140.doc @@ -0,0 +1 @@ +Make `federation_domain_whitelist` nullable in config schema. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 876b74d5d3..f725ee1a67 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1287,10 +1287,15 @@ Options related to federation. --- ### `federation_domain_whitelist` -*(array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. +*(null|array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. + If specified as an empty list (`[]`), federation will be denied with all servers. Specifying an empty list (`[]`) here is the recommended way of disabling federation. -If not specified, the default is to allow federation with all servers. -Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to establish a "private federation", where a group of servers all whitelist each other and have the same whitelist. There is no default for this option. + +If unset or null, allows federation with all servers. + +Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to establish a "private federation", where a group of servers all whitelist each other and have the same whitelist. + +Defaults to `null`. Example configuration: ```yaml diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index cfa2cd4a41..9169677cd5 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1575,17 +1575,20 @@ properties: - myCA2.pem - myCA3.pem federation_domain_whitelist: - type: array + type: ["null", "array"] description: >- Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. + If specified as an empty list (`[]`), federation will be denied with all servers. Specifying an empty list (`[]`) here is the recommended way of disabling federation. - If not specified, the default is to allow federation with all servers. + + If unset or null, allows federation with all servers. + Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to @@ -1593,6 +1596,7 @@ properties: each other and have the same whitelist. items: type: string + default: null examples: - - lon.example.com - nyc.example.com From 83193b378db01fa842e2b30f08508273bc9384e7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 27 Aug 2026 15:57:00 +0100 Subject: [PATCH 126/158] Fix retrying failed event persistence (#20148) If we failed to persist an event and then retried, this would fail due to a conflict error trying to reinsert the state groups that we are persisting. This blocks all new events being persisted in the room until the instance is restarted (which clears entries from the table matching the instance name). This is because we always mark the [latest persisted state group in the room as being persisted](https://github.com/element-hq/synapse/blob/781d08df263e640d85ca5c8b5ff514433e019c6f/synapse/storage/databases/state/deletion.py#L242-L243). Also add an `inserted_ts` field which we use to detect stale entries, e.g. from instances that have been taken offline permanently. These aren't a problem beyond blocking deletion of the associated state groups Introduced in https://github.com/element-hq/synapse/pull/18107 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/20148.bugfix | 1 + synapse/storage/databases/state/deletion.py | 59 +++++++++++++++---- synapse/storage/schema/__init__.py | 1 + ...groups_persisting_inserted_ts.sql.postgres | 22 +++++++ ...e_groups_persisting_inserted_ts.sql.sqlite | 31 ++++++++++ tests/storage/test_state_deletion.py | 52 +++++++++++++++- 6 files changed, 153 insertions(+), 13 deletions(-) create mode 100644 changelog.d/20148.bugfix create mode 100644 synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.postgres create mode 100644 synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.sqlite diff --git a/changelog.d/20148.bugfix b/changelog.d/20148.bugfix new file mode 100644 index 0000000000..2983484208 --- /dev/null +++ b/changelog.d/20148.bugfix @@ -0,0 +1 @@ +Fix a bug where, until Synapse was restarted, new events in a room would fail to be persisted if the database went down while an event was being persisted in the room. Bug introduced in v1.124.0. diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py index 23150e8626..9f08f570d8 100644 --- a/synapse/storage/databases/state/deletion.py +++ b/synapse/storage/databases/state/deletion.py @@ -14,6 +14,7 @@ import contextlib +import logging from typing import ( TYPE_CHECKING, AbstractSet, @@ -30,11 +31,14 @@ from synapse.storage.database import ( make_in_list_sql_clause, ) from synapse.storage.engines import PostgresEngine +from synapse.util.duration import Duration from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer +logger = logging.getLogger(__name__) + class StateDeletionDataStore: """Manages deletion of state groups in a safe manner. @@ -81,6 +85,12 @@ class StateDeletionDataStore: # event will fail to persist (as well as any event in the same batch). DELAY_BEFORE_DELETION_MS = 10 * 60 * 1000 + # How old a row in `state_groups_persisting` has to be before we assume the + # persist that wrote it has gone away. This should be much longer than any + # persist can take. If we clear the row of a live persist, then its state + # groups can be deleted while it is still using them. + STALE_PERSISTING_DURATION = Duration(days=7) + def __init__( self, database: DatabasePool, @@ -91,17 +101,35 @@ class StateDeletionDataStore: self.db_pool = database self._instance_name = hs.get_instance_name() - with db_conn.cursor(txn_name="_clear_existing_persising") as txn: - self._clear_existing_persising(txn) + with db_conn.cursor(txn_name="_clear_existing_persisting") as txn: + self._clear_existing_persisting(txn) - def _clear_existing_persising(self, txn: LoggingTransaction) -> None: + def _clear_existing_persisting(self, txn: LoggingTransaction) -> None: """On startup we clear any entries in `state_groups_persisting` that - match our instance name, in case of a previous unclean shutdown""" + match our instance name (or are very old), in case of a previous unclean + shutdown.""" - self.db_pool.simple_delete_txn( - txn, - table="state_groups_persisting", - keyvalues={"instance_name": self._instance_name}, + # Delete any rows that are very old, or that match our instance name. We + # clear all stale rows, even if they don't match our instance name, as + # we don't know if the instance that created them is still running. + cutoff = self._clock.time_msec() - self.STALE_PERSISTING_DURATION.as_millis() + sql = """ + DELETE FROM state_groups_persisting + WHERE inserted_ts < ? OR (instance_name = ?) + RETURNING state_group + """ + txn.execute(sql, (cutoff, self._instance_name)) + + # Two instances can each have a stale row for the same state group. + state_groups = {state_group for (state_group,) in txn} + + if not state_groups: + return + + logger.info( + "Cleared %d stale state groups from state_groups_persisting: %s", + len(state_groups), + shortstr(state_groups), ) async def check_state_groups_and_bump_deletion( @@ -277,11 +305,20 @@ class StateDeletionDataStore: f"state groups have been deleted: {shortstr(missing_state_groups)}" ) - self.db_pool.simple_insert_many_txn( + # There is a unique key on (state_group, instance_name) so we need to + # handle the case where we have already marked a state group as being + # persisted. This can happen if we fail to persist an event and then + # retry. + now = self._clock.time_msec() + self.db_pool.simple_upsert_many_txn( txn, table="state_groups_persisting", - keys=("state_group", "instance_name"), - values=[(state_group, self._instance_name) for state_group in state_groups], + key_names=("state_group", "instance_name"), + key_values=[ + (state_group, self._instance_name) for state_group in state_groups + ], + value_names=("inserted_ts",), + value_values=[(now,) for _ in state_groups], ) def _finish_persisting_txn( diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 0d9022116d..926121d8e1 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -176,6 +176,7 @@ Changes in SCHEMA_VERSION = 94 - Add `recheck` column (boolean, default true) to the `redactions` table. - MSC4242: Add state DAG tables. - MSC4429/MSC4262: Track updates to user profile fields via a new stream. + - Add an `inserted_ts` column to the `state_groups_persisting` table. """ diff --git a/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.postgres b/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.postgres new file mode 100644 index 0000000000..4e5c0eea96 --- /dev/null +++ b/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.postgres @@ -0,0 +1,22 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Record when we marked a state group as being persisted, so that we can clear +-- out rows left behind by a persist that never finished. +-- +-- The rows already in the table are stamped with the time this runs, so they get +-- the same grace period as a new row before we treat them as stale. The default +-- covers instances that haven't been upgraded yet, which insert without the +-- column, and is what lets the column be NOT NULL during a rolling upgrade. +ALTER TABLE state_groups_persisting + ADD COLUMN inserted_ts BIGINT NOT NULL DEFAULT extract(epoch from now()) * 1000; diff --git a/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.sqlite b/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.sqlite new file mode 100644 index 0000000000..7720a814a5 --- /dev/null +++ b/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.sqlite @@ -0,0 +1,31 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Record when we marked a state group as being persisted, so that we can clear +-- out rows left behind by a persist that never finished. +-- +-- On SQLite we must be in monolith mode, so every row already in the table was +-- written by this process before it restarted, and is dead. Zero marks them as +-- stale straight away, which is what we want. Contrast Postgres, where such a +-- row may belong to a worker that is still running and so gets a grace period. +-- +-- Note that this also correctly handles the case of Synapse version rolling +-- back, as new rows will be written with a 0 inserted_ts, *but* since its +-- running with the old code nothing deletes those row. On upgrade the restart +-- will mean any rows are automatically stale as above. +-- +-- The default only applies to an insert that omits the column, which our own +-- code never does. SQLite only accepts a constant default in ADD COLUMN, so it +-- could not be the current time in any case. +ALTER TABLE state_groups_persisting + ADD COLUMN inserted_ts BIGINT NOT NULL DEFAULT 0; diff --git a/tests/storage/test_state_deletion.py b/tests/storage/test_state_deletion.py index d4079c372e..066d69c93d 100644 --- a/tests/storage/test_state_deletion.py +++ b/tests/storage/test_state_deletion.py @@ -14,12 +14,15 @@ import logging +from collections.abc import Collection +from unittest.mock import patch from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer +from synapse.storage.database import LoggingTransaction from synapse.util.clock import Clock from tests.test_utils.event_injection import create_event @@ -48,8 +51,20 @@ class StateDeletionStoreTestCase(HomeserverTestCase): self.purge_events._delete_state_loop_call.stop() self.user_id = self.register_user("test", "password") - tok = self.login("test", "password") - self.room_id = self.helper.create_room_as(self.user_id, tok=tok) + self.tok = self.login("test", "password") + self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok) + + def get_persisting_marker_rows(self) -> list[tuple[int, str, int]]: + """Return the contents of the `state_groups_persisting` table.""" + + return self.get_success( + self.state_deletion_store.db_pool.simple_select_list( + table="state_groups_persisting", + keyvalues=None, + retcols=("state_group", "instance_name", "inserted_ts"), + desc="get_persisting_marker_rows", + ) + ) def check_if_can_be_deleted(self, state_group: int) -> bool: """Check if the state group is pending deletion.""" @@ -107,6 +122,39 @@ class StateDeletionStoreTestCase(HomeserverTestCase): self.get_success(ctx_mgr.__aenter__()) self.get_success(ctx_mgr.__aexit__(Exception, Exception("test"), None)) + def test_retry_send_after_failed_clean_up(self) -> None: + """Test that we can retry sending an event after the clean up of the + `state_groups_persisting` rows failed, as it does when the database goes + away while we're persisting.""" + + fail_clean_up = True + orig_finish_persisting_txn = self.state_deletion_store._finish_persisting_txn + + def _finish_persisting_txn( + txn: LoggingTransaction, state_groups: Collection[int], error: bool + ) -> None: + if fail_clean_up: + raise Exception("Database has gone away") + + orig_finish_persisting_txn(txn, state_groups, error) + + with patch.object( + self.state_deletion_store, + "_finish_persisting_txn", + new=_finish_persisting_txn, + ): + self.helper.send(self.room_id, body="first", tok=self.tok, expect_code=500) + + # The rows are still in place, as the transaction that would have + # removed them was rolled back. + self.assertNotEqual(self.get_persisting_marker_rows(), []) + + # The database has come back, so the retry should now go through. + fail_clean_up = False + self.helper.send(self.room_id, body="second", tok=self.tok) + + self.assertEqual(self.get_persisting_marker_rows(), []) + def test_existing_pending_deletion_is_cleared(self) -> None: """Test that the pending deletion flag gets cleared when the state group gets persisted.""" From 305faa2ab58cb2df4fba98a3204073a9bf19288f Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 27 Aug 2026 11:53:03 -0400 Subject: [PATCH 127/158] MSC3866: Apply missed filter on /users response (#20152) For the Admin API /users endpoint, the response is meant to exclude the approval flag when MSC3866 is enabled, but the filter that applies the exclusion never actually got used. Fix things so that it does. --- changelog.d/20152.bugfix | 1 + synapse/rest/admin/users.py | 14 +++++++++----- tests/rest/admin/test_user.py | 4 ++++ 3 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20152.bugfix diff --git a/changelog.d/20152.bugfix b/changelog.d/20152.bugfix new file mode 100644 index 0000000000..4233499d33 --- /dev/null +++ b/changelog.d/20152.bugfix @@ -0,0 +1 @@ +When not using MSC3866, omit the approval flag from the response of `GET /_synapse/admin/v2/users`. diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index ddb4483789..43dab16598 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -49,6 +49,7 @@ from synapse.rest.admin._base import ( assert_user_is_admin, ) from synapse.rest.client._base import client_patterns +from synapse.storage.databases.main import UserPaginateResponse from synapse.storage.databases.main.registration import ExternalIDReuseException from synapse.storage.databases.main.stats import UserSortOrder from synapse.types import JsonDict, JsonMapping, TaskStatus, UserID @@ -181,13 +182,16 @@ class UsersRestServletV2(RestServlet): ) # If support for MSC3866 is not enabled, don't show the approval flag. - filter = None + users_filter = None if not self._msc3866_enabled: + users_filter = attr.filters.exclude( + attr.fields(UserPaginateResponse).approved + ) - def _filter(a: attr.Attribute) -> bool: - return a.name != "approved" - - ret = {"users": [attr.asdict(u, filter=filter) for u in users], "total": total} + ret = { + "users": [attr.asdict(u, filter=users_filter) for u in users], + "total": total, + } if (start + limit) < total: ret["next_token"] = str(start + len(users)) diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index e2e133a85e..84bb7e319b 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -1424,6 +1424,10 @@ class UsersListTestCase(unittest.HomeserverTestCase): self.assertIn("avatar_url", u) self.assertIn("creation_ts", u) self.assertIn("last_seen_ts", u) + if self.hs.config.experimental.msc3866.enabled: + self.assertIn("approved", u) + else: + self.assertNotIn("approved", u) def _create_users(self, number_users: int) -> None: """ From c3bec609364dfd7372ccacb335c4856e45dd65be Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 28 Aug 2026 02:05:59 -0400 Subject: [PATCH 128/158] MSC4140: support getting a single delayed event (#19926) See https://github.com/matrix-org/matrix-spec-proposals/blob/toger5/expiring-events-keep-alive/proposals/4140-delayed-events-futures.md#getting-a-single-delayed-event Co-authored-by: Olivier 'reivilibre' --- changelog.d/19926.feature | 1 + synapse/handlers/delayed_events.py | 28 ++++- synapse/rest/client/delayed_events.py | 31 ++++- .../storage/databases/main/delayed_events.py | 84 ++++++++++--- tests/rest/client/test_delayed_events.py | 114 ++++++++++++++++++ 5 files changed, 238 insertions(+), 20 deletions(-) create mode 100644 changelog.d/19926.feature diff --git a/changelog.d/19926.feature b/changelog.d/19926.feature new file mode 100644 index 0000000000..88f3111079 --- /dev/null +++ b/changelog.d/19926.feature @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Add an endpoint for getting a single delayed event. diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 13d6a54de2..cfaf16a30c 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -31,6 +31,8 @@ from synapse.replication.http.delayed_events import ( ) from synapse.storage.databases.main.delayed_events import ( DelayedEventDetails, + DelayedEventResponse, + DelayedEventResponseLegacyCompat, EventType, StateKey, Timestamp, @@ -549,8 +551,30 @@ class DelayedEventsHandler: else: self._next_delayed_event_call.reset(delay_duration.as_secs()) - async def get_all_for_user(self, requester: Requester) -> list[JsonDict]: - """Return all pending delayed events requested by the given user.""" + async def get_for_user( + self, requester: Requester, delay_id: str + ) -> DelayedEventResponse: + """ + Return the specified pending delayed event requested by the given user. + + Raises: + NotFoundError: if no matching delayed event could be found. + """ + await self._delayed_event_mgmt_ratelimiter.ratelimit(requester) + return await self._store.get_delayed_event_for_user( + delay_id, + requester.user.localpart, + ) + + async def get_all_for_user( + self, requester: Requester + ) -> list[DelayedEventResponseLegacyCompat]: + """ + Return all pending delayed events owned by the given user. + Includes fields from earlier revisions of MSC4140 for + compatibility with clients that still expect them. + """ + # TODO: Remove legacy fields once stable await self._delayed_event_mgmt_ratelimiter.ratelimit(requester) return await self._store.get_all_delayed_events_for_user( requester.user.localpart diff --git a/synapse/rest/client/delayed_events.py b/synapse/rest/client/delayed_events.py index 7afecffe2d..07ca07b7b9 100644 --- a/synapse/rest/client/delayed_events.py +++ b/synapse/rest/client/delayed_events.py @@ -134,6 +134,28 @@ class SendDelayedEventServlet(RestServlet): return 200, {} +class DelayedEventServlet(RestServlet): + PATTERNS = client_patterns( + r"/org\.matrix\.msc4140/delayed_events/(?P[^/]+)$", + releases=(), + ) + CATEGORY = "Delayed event management requests" + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self.delayed_events_handler = hs.get_delayed_events_handler() + + async def on_GET( + self, request: SynapseRequest, delay_id: str + ) -> tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + delayed_event = await self.delayed_events_handler.get_for_user( + requester, delay_id + ) + return 200, delayed_event.asdict() + + class DelayedEventsServlet(RestServlet): PATTERNS = client_patterns( r"/org\.matrix\.msc4140/delayed_events$", @@ -150,9 +172,11 @@ class DelayedEventsServlet(RestServlet): requester = await self.auth.get_user_by_req(request) # TODO: Support Pagination stream API ("from" query parameter) delayed_events = await self.delayed_events_handler.get_all_for_user(requester) - - ret = {"delayed_events": delayed_events} - return 200, ret + return 200, { + "delayed_events": [ + delayed_event.asdict() for delayed_event in delayed_events + ] + } def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: @@ -162,4 +186,5 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: CancelDelayedEventServlet(hs).register(http_server) SendDelayedEventServlet(hs).register(http_server) RestartDelayedEventServlet(hs).register(http_server) + DelayedEventServlet(hs).register(http_server) DelayedEventsServlet(hs).register(http_server) diff --git a/synapse/storage/databases/main/delayed_events.py b/synapse/storage/databases/main/delayed_events.py index bb512611e4..35f78e3f97 100644 --- a/synapse/storage/databases/main/delayed_events.py +++ b/synapse/storage/databases/main/delayed_events.py @@ -64,6 +64,33 @@ class DelayedEventDetails(EventDetails): user_localpart: UserLocalpart +@attr.s(slots=True, frozen=True, auto_attribs=True) +class DelayedEventResponse: + """The representation of a delayed event in API format.""" + + delay_id: str + room_id: str + type: str + state_key: str | None + delay_ms: int + delayed_since_ts: int + content: JsonDict = attr.ib(converter=db_to_json) + + def asdict(self) -> JsonDict: + return attr.asdict(self, filter=lambda _attr, v: v is not None) + + +# TODO: Remove this class once the response format is stable +class DelayedEventResponseLegacyCompat(DelayedEventResponse): + """For backwards compatibility with field names from earlier revisions of MSC4140.""" + + def asdict(self) -> JsonDict: + return super().asdict() | { + "delay": self.delay_ms, + "running_since": self.delayed_since_ts, + } + + class DelayedEventsStore(SQLBaseStore): def __init__( self, @@ -290,11 +317,49 @@ class DelayedEventsStore(SQLBaseStore): _get_count_of_delayed_events, ) + async def get_delayed_event_for_user( + self, + delay_id: str, + user_localpart: str, + ) -> DelayedEventResponse: + """ + Returns the specified pending delayed event owned by the given user. + + Raises: + NotFoundError: if there is no matching delayed event. + """ + row = await self.db_pool.simple_select_one( + table="delayed_events", + keyvalues={ + "delay_id": delay_id, + "user_localpart": user_localpart, + "is_processed": False, + }, + retcols=( + "room_id", + "event_type", + "state_key", + "delay", + "send_ts - delay", + "content", + ), + allow_none=True, + desc="get_delayed_event_for_user", + ) + if row is None: + raise NotFoundError("Delayed event not found") + return DelayedEventResponse(delay_id, *row) + async def get_all_delayed_events_for_user( self, user_localpart: str, - ) -> list[JsonDict]: - """Returns all pending delayed events owned by the given user.""" + ) -> list[DelayedEventResponseLegacyCompat]: + """ + Return all pending delayed events owned by the given user. + Includes fields from earlier revisions of MSC4140 for + compatibility with clients that still expect them. + """ + # TODO: Remove legacy fields once stable # TODO: Support Pagination stream API ("next_batch" field) rows = await self.db_pool.execute( "get_all_delayed_events_for_user", @@ -305,7 +370,7 @@ class DelayedEventsStore(SQLBaseStore): event_type, state_key, delay, - send_ts, + send_ts - delay, content FROM delayed_events WHERE user_localpart = ? AND NOT is_processed @@ -313,18 +378,7 @@ class DelayedEventsStore(SQLBaseStore): """, user_localpart, ) - return [ - { - "delay_id": DelayID(row[0]), - "room_id": str(RoomID.from_string(row[1])), - "type": EventType(row[2]), - **({"state_key": StateKey(row[3])} if row[3] is not None else {}), - "delay": Delay(row[4]), - "running_since": Timestamp(row[5] - row[4]), - "content": db_to_json(row[6]), - } - for row in rows - ] + return [DelayedEventResponseLegacyCompat(*row) for row in rows] async def process_timeout_delayed_events( self, current_ts: Timestamp, reprocess_events: bool = False diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index 75d716244a..3af7d13858 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -127,6 +127,120 @@ class DelayedEventsTestCase(HomeserverTestCase): def test_delayed_events_empty_on_startup(self) -> None: self.assertListEqual([], self._get_delayed_events()) + def test_delayed_event_lookup(self) -> None: + # Schedule a message event + delay_ms = 100000 + content: JsonDict = {"message": "hello"} + delayed_since_ts = self.hs.get_clock().time_msec() + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, delay_ms), + content, + self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delay_id = channel.json_body["delay_id"] + + # Test that the scheduled delayed event can be retrieved + channel = self.make_request( + "GET", + f"{PATH_PREFIX}/{delay_id}", + access_token=self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + + # Assert the stored properties of the delayed event + event = channel.json_body + self.assertDictEqual( + event, + { + "delay_id": delay_id, + "room_id": self.room_id, + "type": _EVENT_TYPE, + "delay_ms": delay_ms, + "delayed_since_ts": delayed_since_ts, + "content": content, + }, + ) + + # Test that a non-existent delayed event cannot be found + channel = self.make_request( + "GET", + f"{PATH_PREFIX}/{delay_id}-fake", + access_token=self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + + # Test that other users cannot access this delayed event + channel = self.make_request( + "GET", + f"{PATH_PREFIX}/{delay_id}", + access_token=self.user2_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + + # Now schedule a state event. + # Do it in this test, as opposed to a new one, to confirm that the correct delayed event + # is retrieved when multiple delayed events have been scheduled. + delay_ms += 2000 + state_key = "" + state_event_type = _EVENT_TYPE + "_state" + content = {"state_message": "greetings"} + delayed_since_ts = self.hs.get_clock().time_msec() + channel = self.make_request( + "PUT", + _get_path_for_delayed_state( + self.room_id, state_event_type, state_key, delay_ms + ), + content, + self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delay_id_2 = channel.json_body["delay_id"] + + # Test that the new delayed event has a different ID from the previous one + self.assertNotEqual(delay_id, delay_id_2) + + # Test that the scheduled delayed event can be retrieved + channel = self.make_request( + "GET", + f"{PATH_PREFIX}/{delay_id_2}", + access_token=self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + + # Assert the stored properties of the delayed event + state_event = channel.json_body + self.assertDictEqual( + state_event, + { + "delay_id": delay_id_2, + "room_id": self.room_id, + "type": state_event_type, + "state_key": state_key, + "delay_ms": delay_ms, + "delayed_since_ts": delayed_since_ts, + "content": content, + }, + ) + + # Test that the list lookup retrieves the same items (with legacy fields included) + self.assertEqual( + self._get_delayed_events(), + [ + event + | { + "delay": event["delay_ms"], + "running_since": event["delayed_since_ts"], + }, + state_event + | { + "delay": state_event["delay_ms"], + "running_since": state_event["delayed_since_ts"], + }, + ], + ) + def test_delayed_state_events_are_sent_on_timeout(self) -> None: state_key = "to_send_on_timeout" From 57d6da409cfbcdfa6156c9df9d2746d5bd5fb5d6 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Fri, 28 Aug 2026 14:45:54 +0200 Subject: [PATCH 129/158] Add experimental support for letting application services proxy namespaces in the C-S and S-S API as per MSC4512 (#19972) This implements the proxying part of [MSC4512](https://github.com/matrix-org/matrix-spec-proposals/pull/4512) and is a stopgap towards https://github.com/element-hq/voip-internal/issues/641. It introduces a new configuration property `io.element.msc4512.proxy` that allows application services to claim namespaces in the C-S and S-S API. For requests underneath a claimed namespace, Synapse first authorizes the request and then reverse-proxies it to the application services. For now, the only allowed namespace that can be claimed is `unstable/io.element.msc4195/rtc/livekit`. This pull request can be reviewed by commits. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Signed-off-by: Johannes Marbach --- changelog.d/19972.feature | 1 + synapse/appservice/__init__.py | 27 ++ synapse/config/appservice.py | 39 ++ synapse/config/experimental.py | 3 + .../federation/transport/server/__init__.py | 4 + .../transport/server/appservice_proxy.py | 115 +++++ synapse/http/appservice_proxy.py | 194 ++++++++ synapse/http/proxy.py | 22 +- synapse/rest/__init__.py | 2 + synapse/rest/client/appservice_proxy.py | 82 ++++ tests/appservice/test_appservice.py | 66 +++ .../transport/server/test_appservice_proxy.py | 349 ++++++++++++++ tests/http/test_appservice_proxy.py | 47 ++ tests/rest/client/test_appservice_proxy.py | 442 ++++++++++++++++++ tests/storage/test_appservice.py | 234 +++++++++- 15 files changed, 1614 insertions(+), 13 deletions(-) create mode 100644 changelog.d/19972.feature create mode 100644 synapse/federation/transport/server/appservice_proxy.py create mode 100644 synapse/http/appservice_proxy.py create mode 100644 synapse/rest/client/appservice_proxy.py create mode 100644 tests/federation/transport/server/test_appservice_proxy.py create mode 100644 tests/http/test_appservice_proxy.py create mode 100644 tests/rest/client/test_appservice_proxy.py diff --git a/changelog.d/19972.feature b/changelog.d/19972.feature new file mode 100644 index 0000000000..56e3b27136 --- /dev/null +++ b/changelog.d/19972.feature @@ -0,0 +1 @@ +Add experimental support for letting application services proxy namespaces in the C-S and S-S API as per MSC4512. diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 19fff7f009..20f7a410b6 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -97,6 +97,11 @@ class ApplicationService: # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] + # Prefixes are applied after the version segment(s) (either /vX/ or /unstable/foo/): + # - /_matrix/client/(unstable/[^/]+|v[^/]+)/{prefix}/.* + # - /_matrix/federation/(unstable/[^/]+|v[^/]+)/{prefix}/.* + ALLOWED_PROXY_PREFIXES = {"rtc/livekit"} + def __init__( self, token: str, @@ -113,11 +118,19 @@ class ApplicationService: msc3202_transaction_extensions: bool = False, msc4190_device_management: bool = False, scopes: Iterable[str] = frozenset(), + proxy_prefix: str | None = None, + proxy_url: str | None = None, ): self.token = token self.url = ( url.rstrip("/") if isinstance(url, str) else None ) # url must not end with a slash + self.proxy_url = ( + proxy_url.rstrip("/") if isinstance(proxy_url, str) else None + ) # proxy_url must not end with a slash + self.proxy_prefix = ( + proxy_prefix.rstrip("/") if isinstance(proxy_prefix, str) else None + ) # proxy_prefix must not end with a slash self.hs_token = hs_token # The full Matrix ID for this application service's sender. self.sender = sender @@ -143,6 +156,14 @@ class ApplicationService: if "|" in self.id: raise Exception("application service ID cannot contain '|' character") + if (self.proxy_prefix is None) != (self.proxy_url is None): + raise KeyError("proxy_url and proxy_prefix must always be set together") + if proxy_prefix is not None: + if not proxy_prefix or not self.proxy_url: + raise ValueError("proxy_prefix and proxy_url must be non-empty strings") + if not self._is_proxy_prefix_allowed(proxy_prefix): + raise ValueError(f"cannot claim reserved proxy prefix {proxy_prefix!r}") + # .protocols is a publicly visible field if protocols: self.protocols = set(protocols) @@ -206,6 +227,12 @@ class ApplicationService: return namespace.exclusive return False + def _is_proxy_prefix_allowed(self, prefix: str) -> bool: + return any( + prefix == allowed or prefix.startswith(allowed + "/") + for allowed in ApplicationService.ALLOWED_PROXY_PREFIXES + ) + @cached(num_args=1, cache_context=True) async def _matches_user_in_member_list( self, diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 4e61ef694f..1868985f80 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -68,6 +68,7 @@ def load_appservices( # Dicts of value -> filename seen_as_tokens: dict[str, str] = {} seen_ids: dict[str, str] = {} + seen_proxy_prefixes: dict[str, str] = {} appservices = [] @@ -93,6 +94,17 @@ def load_appservices( ) ) seen_as_tokens[appservice.token] = config_file + if appservice.proxy_prefix is not None: + for seen_prefix, seen_file in seen_proxy_prefixes.items(): + if _proxy_prefixes_overlap( + appservice.proxy_prefix, seen_prefix + ): + raise ConfigError( + "io.element.msc4512.proxy_prefix values must not overlap across " + "application services: " + f"{appservice.proxy_prefix} (files: {config_file}, {seen_file})" + ) + seen_proxy_prefixes[appservice.proxy_prefix] = config_file logger.info("Loaded application service: %s", appservice) appservices.append(appservice) except Exception as e: @@ -102,6 +114,15 @@ def load_appservices( return appservices +def _proxy_prefixes_overlap(prefix_a: str, prefix_b: str) -> bool: + """Returns whether two proxy prefixes overlap by sharing a common path prefix.""" + return ( + prefix_a == prefix_b + or prefix_a.startswith(prefix_b + "/") + or prefix_b.startswith(prefix_a + "/") + ) + + def _load_appservice( hostname: str, as_info: JsonDict, config_filename: str ) -> ApplicationService: @@ -207,6 +228,22 @@ def _load_appservice( "The `io.element.msc4502.scopes` option should be a list of strings if specified." ) + # Opt-in setting to enable proxying C-S and S-S API endpoints. + # When set, Synapse will reverse-proxy requests under the prefix to the appservice: + proxy_prefix = as_info.get("io.element.msc4512.proxy_prefix") + if proxy_prefix is not None: + if not isinstance(proxy_prefix, str) or not proxy_prefix: + raise ValueError( + "The `io.element.msc4512.proxy_prefix` option should be a non-empty string." + ) + + proxy_url = as_info.get("io.element.msc4512.proxy_url") + if proxy_url is not None: + if not isinstance(proxy_url, str) or not proxy_url: + raise ValueError( + "The `io.element.msc4512.proxy_url` option should be a non-empty string." + ) + return ApplicationService( token=as_info["as_token"], url=as_info["url"], @@ -222,4 +259,6 @@ def _load_appservice( msc3202_transaction_extensions=msc3202_transaction_extensions, msc4190_device_management=msc4190_enabled, scopes=scopes, + proxy_prefix=proxy_prefix, + proxy_url=proxy_url, ) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 97dc803ab7..1c2f021322 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -312,3 +312,6 @@ class ExperimentalConfig(Config): # MSC4491: Invite reasons in room creation self.msc4491_enabled: bool = experimental.get("msc4491_enabled", False) + + # MSC4512: Delegating parts of the C-S and S-S API to application services + self.msc4512_enabled: bool = experimental.get("msc4512_enabled", False) diff --git a/synapse/federation/transport/server/__init__.py b/synapse/federation/transport/server/__init__.py index 0eff49cf73..70fc55123b 100644 --- a/synapse/federation/transport/server/__init__.py +++ b/synapse/federation/transport/server/__init__.py @@ -24,6 +24,7 @@ import logging from typing import TYPE_CHECKING, Iterable, Literal from synapse.api.errors import FederationDeniedError, SynapseError +from synapse.federation.transport.server import appservice_proxy from synapse.federation.transport.server._base import ( Authenticator, BaseFederationServlet, @@ -340,3 +341,6 @@ def register_servlets( ratelimiter=ratelimiter, server_name=hs.hostname, ).register(resource) + + if "federation" in servlet_groups: + appservice_proxy.register_servlets(hs, resource, authenticator, ratelimiter) diff --git a/synapse/federation/transport/server/appservice_proxy.py b/synapse/federation/transport/server/appservice_proxy.py new file mode 100644 index 0000000000..7ececd8fff --- /dev/null +++ b/synapse/federation/transport/server/appservice_proxy.py @@ -0,0 +1,115 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +import re +from http import HTTPStatus +from io import BytesIO +from typing import TYPE_CHECKING + +from synapse.api.errors import Codes, SynapseError +from synapse.appservice import ApplicationService +from synapse.federation.transport.server._base import Authenticator +from synapse.http import QuieterFileBodyProducer +from synapse.http.appservice_proxy import proxy_request_to_appservice +from synapse.http.server import HttpServer, ServletCallback +from synapse.http.site import SynapseRequest +from synapse.util.json import json_decoder +from synapse.util.ratelimitutils import FederationRateLimiter + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +def _make_proxy_callback( + hs: "HomeServer", + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + appservice: ApplicationService, +) -> ServletCallback: + """Builds a servlet callback that authenticates an incoming federation request, + rate-limits it by origin, and forwards it to the given application service's + proxy URL. + """ + + async def _proxy(request: SynapseRequest, **kwargs: str) -> None: + body_producer = None + content = None + + if request.method in (b"PUT", b"POST"): + raw_body = request.content.read() # type: ignore[union-attr] + try: + content = json_decoder.decode(raw_body.decode("utf-8")) + except Exception: + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Content not JSON.", Codes.NOT_JSON + ) + body_producer = QuieterFileBodyProducer(BytesIO(raw_body)) + + origin = await authenticator.authenticate_request(request, content) + + # Apply the same per-origin rate limiting that every other federation endpoint gets. + with ratelimiter.ratelimit(origin) as d: + await d + if request._disconnected: + logger.warning( + "client disconnected before we started processing request" + ) + return + + await proxy_request_to_appservice( + request, + hs, + appservice, + body_producer, + extra_request_headers={b"X-Matrix-Origin": origin.encode("ascii")}, + ) + + return _proxy + + +def register_servlets( + hs: "HomeServer", + resource: HttpServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, +) -> None: + """Registers blanket reverse-proxy routes for each application service that has + configured a proxy prefix. This forwards requests under + /_matrix/federation///* (where is either "vN" or "unstable") + to the same path under the application service's proxy URL after verifying request + authentication. + """ + if not hs.config.experimental.msc4512_enabled: + return + + for appservice in hs.get_datastores().main.get_app_services(): + if appservice.proxy_prefix is None or appservice.proxy_url is None: + continue + + pattern = re.compile( + r"^/_matrix/federation/(?:unstable/[^/]+|v[^/]+)/%s(/.*)?$" + % (re.escape(appservice.proxy_prefix),) + ) + callback = _make_proxy_callback(hs, authenticator, ratelimiter, appservice) + + for method in ("GET", "POST", "PUT", "DELETE"): + resource.register_paths( + method, + (pattern,), + callback, + "ApplicationServiceFederationProxy", + ) diff --git a/synapse/http/appservice_proxy.py b/synapse/http/appservice_proxy.py new file mode 100644 index 0000000000..714dfcc0c1 --- /dev/null +++ b/synapse/http/appservice_proxy.py @@ -0,0 +1,194 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING, Optional, cast +from urllib.parse import parse_qs, unquote_to_bytes, urlencode, urlsplit + +from twisted.python import failure +from twisted.web.http_headers import Headers +from twisted.web.iweb import IBodyProducer, IResponse + +from synapse.api.errors import Codes, SynapseError +from synapse.appservice import ApplicationService +from synapse.http.proxy import ( + HOP_BY_HOP_HEADERS_LOWERCASE, + _ProxyResponseBody, + parse_connection_header_value, +) +from synapse.http.server import return_json_error, set_cors_headers +from synapse.http.site import SynapseRequest +from synapse.logging.context import make_deferred_yieldable, run_in_background +from synapse.util.async_helpers import timeout_deferred + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +def has_dot_segments(path: bytes) -> bool: + """Whether the given request path contains any "." or ".." segments. + + The path is percent-decoded before it is split, since `%2e%2e` and `..` are + equivalent to anything that resolves the path, and `%2f` hides a separator that + would otherwise not be seen. Note that a single decode is deliberate: it matches + the single decode that route arguments get in `JsonResource._async_render`, so + `%252e%252e` is left alone rather than being treated as a dot segment. + + The caller is expected to reject such a path rather than rewrite it. Synapse + routes on the raw path, and federation request signatures cover the raw URI, so + normalising a path in place would break both. + """ + return any( + segment in (b".", b"..") for segment in unquote_to_bytes(path).split(b"/") + ) + + +def strip_access_token_from_uri(uri: bytes) -> bytes: + """Remove any `access_token` query parameter from the given request URI. + + Clients are not supposed to authenticate with an `access_token` query + parameter, but some might do so anyway. Since the URI is forwarded to the + application service verbatim, strip it out just in case, so that it isn't + inadvertently leaked to the application service. + """ + split_uri = urlsplit(uri) + if not split_uri.query: + return uri + + args = parse_qs(split_uri.query, keep_blank_values=True) + for key in list(args.keys()): + if key.lower() == b"access_token": + del args[key] + + if not args: + return split_uri.path + + return split_uri.path + b"?" + urlencode(args, doseq=True).encode("ascii") + + +async def proxy_request_to_appservice( + request: SynapseRequest, + hs: "HomeServer", + appservice: ApplicationService, + body_producer: Optional[IBodyProducer], + extra_request_headers: dict[bytes, bytes] | None = None, +) -> None: + """Forward the given request to an application service's proxy URL and stream + the response back to the original caller unchanged. + + Args: + request: The inbound request to forward. + hs: The homeserver. + appservice: The application service to forward the request to. Must have + `proxy_url` and `hs_token` set. + body_producer: A producer for the request body to forward, or None if the + request has no body to forward. + extra_request_headers: Additional headers to set on the outbound request, + beyond those copied from the original request. + """ + assert appservice.proxy_url is not None + assert appservice.hs_token is not None + + request_path = request.uri.split(b"?", 1)[0] + if has_dot_segments(request_path): + return_json_error( + failure.Failure( + SynapseError( + HTTPStatus.BAD_REQUEST, + "Request path is not allowed", + Codes.INVALID_PARAM, + ) + ), + request, + None, + ) + return + + target_uri = appservice.proxy_url.encode("ascii") + strip_access_token_from_uri( + request.uri + ) + + # Only forward the bare minimum of request headers an application service could + # plausibly need. + headers = Headers() + for header_name, header_values in request.requestHeaders.getAllRawHeaders(): + if header_name.decode("ascii").lower() in { + "content-type", + "accept", + "accept-language", + }: + headers.setRawHeaders(header_name, header_values) + + headers.setRawHeaders( + b"Authorization", [b"Bearer " + appservice.hs_token.encode("ascii")] + ) + + if extra_request_headers: + for header_name, header_value in extra_request_headers.items(): + headers.setRawHeaders(header_name, [header_value]) + + agent = hs.get_proxied_http_client().agent + request_deferred = run_in_background( + agent.request, + request.method, + target_uri, + headers=headers, + bodyProducer=body_producer, + ) + request_deferred = timeout_deferred( + deferred=request_deferred, + timeout=30, # Give the application service at most 30s to respond. + clock=hs.get_clock(), + ) + + try: + response = await make_deferred_yieldable(request_deferred) + except Exception: + logger.warning( + "Error proxying request to application service %s at %s", + appservice.id, + target_uri, + exc_info=True, + ) + return_json_error(failure.Failure(), request, None) + return + + _send_response(request, response) + + +def _send_response(request: SynapseRequest, response: IResponse) -> None: + response_headers = cast(Headers, response.headers) + + request.setResponseCode(response.code) + set_cors_headers(request) + + # We strip the "hop-by-hop" headers as defined by RFC2616. + headers_to_strip = set(HOP_BY_HOP_HEADERS_LOWERCASE) + + # The `Connection` header can define additional headers that should not be + # copied over. + connection_header = response_headers.getRawHeaders(b"connection") + headers_to_strip |= parse_connection_header_value( + connection_header[0] if connection_header else None + ) + + for header_name, header_values in response_headers.getAllRawHeaders(): + if header_name.decode("ascii").lower() in headers_to_strip: + continue + request.responseHeaders.setRawHeaders(header_name, header_values) + + response.deliverBody(_ProxyResponseBody(request)) diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index b3a2f84f29..ad5c6d5d91 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -51,16 +51,18 @@ logger = logging.getLogger(__name__) # "Hop-by-hop" headers (as opposed to "end-to-end" headers) as defined by RFC2616 # section 13.5.1 and referenced in RFC9110 section 7.6.1. These are meant to only be # consumed by the immediate recipient and not be forwarded on. -HOP_BY_HOP_HEADERS_LOWERCASE = { - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailers", - "transfer-encoding", - "upgrade", -} +HOP_BY_HOP_HEADERS_LOWERCASE = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + } +) assert all(header.lower() == header for header in HOP_BY_HOP_HEADERS_LOWERCASE) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index c8ede662aa..1f705aee23 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -28,6 +28,7 @@ from synapse.rest.client import ( account_data, account_validity, appservice_ping, + appservice_proxy, auth, auth_metadata, capabilities, @@ -130,6 +131,7 @@ CLIENT_SERVLET_FUNCTIONS: tuple[RegisterServletsFunc, ...] = ( auth_metadata.register_servlets, thread_subscriptions.register_servlets, room_membership.register_servlets, + appservice_proxy.register_servlets, ) SERVLET_GROUPS: dict[str, Iterable[RegisterServletsFunc]] = { diff --git a/synapse/rest/client/appservice_proxy.py b/synapse/rest/client/appservice_proxy.py new file mode 100644 index 0000000000..aac1f465d3 --- /dev/null +++ b/synapse/rest/client/appservice_proxy.py @@ -0,0 +1,82 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +import re +from typing import TYPE_CHECKING + +from synapse.api.ratelimiting import RequestRatelimiter +from synapse.appservice import ApplicationService +from synapse.http import QuieterFileBodyProducer +from synapse.http.appservice_proxy import proxy_request_to_appservice +from synapse.http.server import HttpServer, ServletCallback +from synapse.http.site import SynapseRequest + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +def _make_proxy_callback( + hs: "HomeServer", + ratelimiter: RequestRatelimiter, + appservice: ApplicationService, +) -> ServletCallback: + async def _proxy(request: SynapseRequest, **kwargs: str) -> None: + requester = await hs.get_auth().get_user_by_req(request) + + await ratelimiter.ratelimit(requester) + + await proxy_request_to_appservice( + request, + hs, + appservice, + QuieterFileBodyProducer(request.content), + extra_request_headers={ + b"X-Matrix-User-Identifier": requester.user.to_string().encode("ascii") + }, + ) + + return _proxy + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + """Registers blanket reverse-proxy routes for each application service that has + configured a proxy prefix. This forwards requests under + /_matrix/client///* (where is either "vN" or "unstable") + to the same path under the application service's proxy URL after verifying request + authentication. + """ + if not hs.config.experimental.msc4512_enabled: + return + + ratelimiter = hs.get_request_ratelimiter() + for appservice in hs.get_datastores().main.get_app_services(): + if appservice.proxy_prefix is None: + continue + + pattern = re.compile( + r"^/_matrix/client/(?:unstable/[^/]+|v[^/]+)/%s(/.*)?$" + % (re.escape(appservice.proxy_prefix),) + ) + callback = _make_proxy_callback(hs, ratelimiter, appservice) + + for method in ("GET", "POST", "PUT", "DELETE"): + http_server.register_paths( + method, + (pattern,), + callback, + "ApplicationServiceClientProxy", + ) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 3f124d9a2c..94bfdd3fba 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -291,3 +291,69 @@ class ApplicationServiceScopesTestCase(unittest.TestCase): token="some_token", scopes=["does:not:exist"], ) + + +class ApplicationServiceProxyPrefixTestCase(unittest.TestCase): + """Tests the proxying configuration for application services from MSC4512.""" + + def _make_service(self, **kwargs: Any) -> ApplicationService: + kwargs.setdefault("id", "unique_identifier") + kwargs.setdefault("sender", UserID.from_string("@as:test")) + kwargs.setdefault("token", "some_token") + return ApplicationService(**kwargs) + + def test_proxy_prefix_without_proxy_url_raises(self) -> None: + with self.assertRaises(KeyError): + self._make_service(proxy_url=None, proxy_prefix="rtc/livekit") + + def test_proxy_prefix_with_empty_proxy_url_raises(self) -> None: + with self.assertRaises(ValueError): + self._make_service(proxy_url="", proxy_prefix="rtc/livekit") + + def test_proxy_url_without_proxy_prefix_raises(self) -> None: + with self.assertRaises(KeyError): + self._make_service(proxy_url="http://proxy.example.com") + + def test_proxy_url_with_empty_proxy_prefix_raises(self) -> None: + with self.assertRaises(ValueError): + self._make_service(proxy_url="http://proxy.example.com", proxy_prefix="") + + def test_proxy_prefix_with_proxy_url_is_stored(self) -> None: + service = self._make_service( + proxy_url="http://proxy.example.com", + proxy_prefix="rtc/livekit", + ) + self.assertEqual(service.proxy_prefix, "rtc/livekit") + self.assertEqual(service.proxy_url, "http://proxy.example.com") + + def test_proxy_url_trailing_slash_is_stripped(self) -> None: + service = self._make_service( + proxy_url="http://proxy.example.com/", + proxy_prefix="rtc/livekit", + ) + self.assertEqual(service.proxy_url, "http://proxy.example.com") + + def test_nested_proxy_prefix_is_allowed(self) -> None: + service = self._make_service( + proxy_url="http://proxy.example.com", + proxy_prefix="rtc/livekit/foo", + ) + self.assertEqual(service.proxy_prefix, "rtc/livekit/foo") + + def test_disallowed_proxy_prefix_raises(self) -> None: + with self.assertRaises(ValueError): + self._make_service( + proxy_url="http://proxy.example.com", proxy_prefix="not/allowed" + ) + + def test_no_proxy_prefix_defaults_to_none(self) -> None: + service = self._make_service() + self.assertIsNone(service.proxy_prefix) + self.assertIsNone(service.proxy_url) + + def test_trailing_slash_on_proxy_prefix_is_stripped(self) -> None: + service = self._make_service( + proxy_url="http://proxy.example.com", + proxy_prefix="rtc/livekit/foo/", + ) + self.assertEqual(service.proxy_prefix, "rtc/livekit/foo") diff --git a/tests/federation/transport/server/test_appservice_proxy.py b/tests/federation/transport/server/test_appservice_proxy.py new file mode 100644 index 0000000000..957e1c3dae --- /dev/null +++ b/tests/federation/transport/server/test_appservice_proxy.py @@ -0,0 +1,349 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import tempfile +from unittest.mock import Mock + +import yaml + +from twisted.internet import defer +from twisted.internet.testing import MemoryReactor +from twisted.web.http_headers import Headers + +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock + +from tests import unittest +from tests.test_utils import FakeResponse + +APPSERVICE_URL = "http://appservice.example.com" +APPSERVICE_PREFIX = "rtc/livekit" +VERSIONED_PREFIX = f"v1/{APPSERVICE_PREFIX}" + + +class ApplicationServiceFederationProxyTestCase(unittest.FederatingHomeserverTestCase): + """Tests the proxying of federation requests to application services from MSC4512.""" + + def default_config(self) -> JsonDict: + config = super().default_config() + _, path = tempfile.mkstemp(prefix="as_fed_proxy_config") + with open(path, "w") as f: + yaml.dump( + { + "id": "proxy_as", + "url": None, + "as_token": "as_token", + "hs_token": "hs_token", + "sender_localpart": "proxy_bot", + "namespaces": {}, + "io.element.msc4512.proxy_prefix": APPSERVICE_PREFIX, + "io.element.msc4512.proxy_url": APPSERVICE_URL, + }, + f, + ) + config["app_service_config_files"] = [path] + config.setdefault("experimental_features", {}).setdefault( + "msc4512_enabled", True + ) + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self.agent = Mock() + hs.get_proxied_http_client().agent = self.agent + + def test_signed_get_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"ok": True}) + ) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"ok": True}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/federation/{VERSIONED_PREFIX}/some/path".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-Origin"), + [self.OTHER_SERVER_NAME.encode("ascii")], + ) + + def test_signed_get_is_proxied_at_root_path(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"ok": True}) + ) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}" + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"ok": True}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/federation/{VERSIONED_PREFIX}".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-Origin"), + [self.OTHER_SERVER_NAME.encode("ascii")], + ) + + def test_signed_post_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"ok": True}) + ) + ) + + content = {"key": "value"} + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + content=content, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"ok": True}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"POST") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/federation/{VERSIONED_PREFIX}/some/path".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-Origin"), + [self.OTHER_SERVER_NAME.encode("ascii")], + ) + self.assertEqual( + kwargs["headers"].getRawHeaders(b"Content-Type"), [b"application/json"] + ) + + body_producer = kwargs["bodyProducer"] + self.assertGreater(body_producer.length, 0) + + def test_headers_outside_the_allowlist_not_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_signed_federation_request( + "GET", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + custom_headers=[("Connection", "close"), ("X-Forward", "forward")], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertIsNone(headers.getRawHeaders(b"Connection")) + self.assertIsNone(headers.getRawHeaders(b"X-Forward")) + + def test_allowlisted_headers_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_signed_federation_request( + "GET", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + custom_headers=[ + ("Accept", "application/json"), + ("Accept-Language", "en-US"), + ], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Accept"), [b"application/json"]) + self.assertEqual(headers.getRawHeaders(b"Accept-Language"), [b"en-US"]) + + def test_host_and_content_length_headers_not_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_signed_federation_request( + "POST", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + content={"key": "value"}, + custom_headers=[("Host", "original-client-facing-host.example")], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertIsNone(headers.getRawHeaders(b"Host")) + self.assertIsNone(headers.getRawHeaders(b"Content-Length")) + + def test_response_headers_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse( + code=200, + body=b"hello", + headers=Headers({"X-Forward": ["forward"]}), + ) + ) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.result["body"], b"hello") + self.assertEqual(channel.headers.getRawHeaders(b"X-Forward"), [b"forward"]) + + def test_unsigned_get_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + shorthand=False, + ) + + self.assertEqual(channel.code, 401) + self.agent.request.assert_not_called() + + @unittest.override_config({"rc_federation": {"reject_limit": -1}}) + def test_rate_limited_request_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 429) + self.agent.request.assert_not_called() + + def test_non_existing_path_under_proxy_prefix_is_rejected(self) -> None: + self.agent.request = Mock(return_value=defer.fail(Exception("boom"))) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 500) + self.agent.request.assert_called() + + def test_path_with_dot_segment_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/../path" + ) + + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + self.agent.request.assert_not_called() + + def test_path_with_encoded_dot_segment_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/%2e%2e/path" + ) + + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + self.agent.request.assert_not_called() + + def test_unregistered_prefix_is_rejected(self) -> None: + channel = self.make_signed_federation_request( + "GET", "/_matrix/federation/not_a_registered_prefix/some/path" + ) + + self.assertEqual(channel.code, 404) + + def test_unregistered_prefix_with_suffix_is_rejected(self) -> None: + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}-2" + ) + + self.assertEqual(channel.code, 404) + + def test_missing_version_segment_is_rejected(self) -> None: + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{APPSERVICE_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 404) + self.agent.request.assert_not_called() + + def test_unstable_version_segment_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"ok": True}) + ) + ) + + path = ( + f"/_matrix/federation/unstable/org.example.msc9999/" + f"{APPSERVICE_PREFIX}/some/path" + ) + channel = self.make_signed_federation_request("GET", path) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"ok": True}) + + ((method, uri), _kwargs) = self.agent.request.call_args + self.assertEqual(method, b"GET") + self.assertEqual(uri, f"{APPSERVICE_URL}{path}".encode()) + + @unittest.override_config({"experimental_features": {"msc4512_enabled": False}}) + def test_proxy_route_not_registered_when_msc4512_disabled(self) -> None: + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 404) + self.agent.request.assert_not_called() diff --git a/tests/http/test_appservice_proxy.py b/tests/http/test_appservice_proxy.py new file mode 100644 index 0000000000..de1ba3cae9 --- /dev/null +++ b/tests/http/test_appservice_proxy.py @@ -0,0 +1,47 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from synapse.http.appservice_proxy import has_dot_segments + +from tests import unittest + + +class HasDotSegmentsTestCase(unittest.TestCase): + def test_plain_path_has_no_dot_segments(self) -> None: + self.assertFalse(has_dot_segments(b"/some/path")) + self.assertFalse(has_dot_segments(b"/some/path.txt")) + self.assertFalse(has_dot_segments(b"/some/...path")) + + def test_dot_segment_is_detected(self) -> None: + self.assertTrue(has_dot_segments(b"/some/./path")) + self.assertTrue(has_dot_segments(b"/./some/path")) + self.assertTrue(has_dot_segments(b"/some/path/.")) + + def test_dot_dot_segment_is_detected(self) -> None: + self.assertTrue(has_dot_segments(b"/some/../path")) + self.assertTrue(has_dot_segments(b"/../some/path")) + self.assertTrue(has_dot_segments(b"/some/path/..")) + + def test_percent_encoded_dot_segments_are_detected(self) -> None: + self.assertTrue(has_dot_segments(b"/some/%2e%2e/path")) + self.assertTrue(has_dot_segments(b"/some/%2e/path")) + self.assertTrue(has_dot_segments(b"/some/%2E%2E/path")) + + def test_percent_encoded_separator_is_detected(self) -> None: + self.assertTrue(has_dot_segments(b"/some%2f../path")) + + def test_double_encoded_dot_segments_are_not_detected(self) -> None: + # Only a single decode is performed, matching the single decode that route + # arguments get elsewhere, so a double-encoded segment is left alone. + self.assertFalse(has_dot_segments(b"/some/%252e%252e/path")) diff --git a/tests/rest/client/test_appservice_proxy.py b/tests/rest/client/test_appservice_proxy.py new file mode 100644 index 0000000000..d389fa7d00 --- /dev/null +++ b/tests/rest/client/test_appservice_proxy.py @@ -0,0 +1,442 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import os +import tempfile +from unittest.mock import Mock + +import yaml + +from twisted.internet import defer +from twisted.internet.testing import MemoryReactor +from twisted.web.http_headers import Headers + +from synapse.rest import admin +from synapse.rest.client import appservice_proxy, login +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock +from synapse.util.json import json_encoder + +from tests import unittest +from tests.test_utils import FakeResponse + +APPSERVICE_URL = "http://appservice.example.com" +APPSERVICE_PREFIX = "rtc/livekit" +VERSIONED_PREFIX = f"v1/{APPSERVICE_PREFIX}" + + +class ApplicationServiceClientProxyTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + appservice_proxy.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + with tempfile.NamedTemporaryFile( + mode="w", prefix="as_proxy_config", delete=False + ) as f: + self.addCleanup(os.remove, f.name) + yaml.dump( + { + "id": "proxy_as", + "url": None, + "as_token": "as_token", + "hs_token": "hs_token", + "sender_localpart": "proxy_bot", + "namespaces": {}, + "io.element.msc4512.proxy_prefix": APPSERVICE_PREFIX, + "io.element.msc4512.proxy_url": APPSERVICE_URL, + }, + f, + ) + config["app_service_config_files"] = [f.name] + config.setdefault("experimental_features", {}).setdefault( + "msc4512_enabled", True + ) + return config + + def prepare(self, _reactor: MemoryReactor, _clock: Clock, hs: HomeServer) -> None: + self.agent = Mock() + hs.get_proxied_http_client().agent = self.agent + + self.user_id = self.register_user("proxy_user", "password") + self.access_token = self.login("proxy_user", "password") + + def test_get_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path?foo=bar", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/client/{VERSIONED_PREFIX}/some/path?foo=bar".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-User-Identifier"), + [self.user_id.encode("ascii")], + ) + + def test_access_token_query_param_is_stripped(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path?access_token={self.access_token}&foo=bar", + shorthand=False, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), _kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/client/{VERSIONED_PREFIX}/some/path?foo=bar".encode(), + ) + + def test_get_is_proxied_at_root_path(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/client/{VERSIONED_PREFIX}".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-User-Identifier"), + [self.user_id.encode("ascii")], + ) + + def test_post_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + channel = self.make_request( + "POST", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + content={"key": "value"}, + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"POST") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/client/{VERSIONED_PREFIX}/some/path".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-User-Identifier"), + [self.user_id.encode("ascii")], + ) + self.assertEqual(headers.getRawHeaders(b"Content-Type"), [b"application/json"]) + + body_producer = kwargs["bodyProducer"] + expected_body = json_encoder.encode({"key": "value"}).encode("utf8") + self.assertEqual(body_producer.length, len(expected_body)) + + def test_headers_outside_the_allowlist_not_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + custom_headers=[("Connection", "close"), ("X-Forward", "forward")], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertIsNone(headers.getRawHeaders(b"Connection")) + self.assertIsNone(headers.getRawHeaders(b"X-Forward")) + + def test_allowlisted_headers_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + custom_headers=[ + ("Accept", "application/json"), + ("Accept-Language", "en-US"), + ], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Accept"), [b"application/json"]) + self.assertEqual(headers.getRawHeaders(b"Accept-Language"), [b"en-US"]) + + def test_host_and_content_length_headers_not_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_request( + "POST", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + content={"key": "value"}, + shorthand=False, + access_token=self.access_token, + custom_headers=[("Host", "original-client-facing-host.example")], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertIsNone(headers.getRawHeaders(b"Host")) + self.assertIsNone(headers.getRawHeaders(b"Content-Length")) + + def test_response_headers_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse( + code=200, + body=b"hello", + headers=Headers({"X-Forward": ["forward"]}), + ) + ) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.result["body"], b"hello") + self.assertEqual(channel.headers.getRawHeaders(b"X-Forward"), [b"forward"]) + + def test_response_cors_headers_set(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.headers.getRawHeaders(b"Access-Control-Allow-Origin"), [b"*"] + ) + + def test_non_existing_path_under_proxy_prefix_is_rejected(self) -> None: + self.agent.request = Mock(return_value=defer.fail(Exception("boom"))) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/does/not/exist", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 500) + self.assertEqual(channel.json_body["errcode"], "M_UNKNOWN") + self.agent.request.assert_called() + + def test_unauthenticated_get_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + ) + + self.assertEqual(channel.code, 401) + self.agent.request.assert_not_called() + + @unittest.override_config({"rc_message": {"burst_count": 0}}) + def test_rate_limited_request_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 429) + self.agent.request.assert_not_called() + + def test_path_with_dot_segment_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/../path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + self.agent.request.assert_not_called() + + def test_path_with_encoded_dot_segment_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/%2e%2e/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + self.agent.request.assert_not_called() + + def test_unregistered_prefix_is_rejected(self) -> None: + channel = self.make_request( + "GET", + "/_matrix/client/not-a-prefix", + shorthand=False, + ) + + self.assertEqual(channel.code, 404) + + def test_unregistered_prefix_with_suffix_is_rejected(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}-2", + shorthand=False, + ) + + self.assertEqual(channel.code, 404) + + def test_missing_version_segment_is_rejected(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/{APPSERVICE_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 404) + self.agent.request.assert_not_called() + + def test_unstable_msc_version_segment_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + path = f"/_matrix/client/unstable/org.example.msc9999/{APPSERVICE_PREFIX}/some/path" + channel = self.make_request( + "GET", + path, + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), _kwargs) = self.agent.request.call_args + self.assertEqual(method, b"GET") + self.assertEqual(uri, f"{APPSERVICE_URL}{path}".encode()) + + @unittest.override_config({"experimental_features": {"msc4512_enabled": False}}) + def test_proxy_route_not_registered_when_msc4512_disabled(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 404) + self.agent.request.assert_not_called() diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 8430e54b92..1e097306e4 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -42,7 +42,7 @@ from synapse.storage.databases.main.appservice import ( ApplicationServiceStore, ApplicationServiceTransactionStore, ) -from synapse.types import DeviceListUpdates, JsonDict +from synapse.types import DeviceListUpdates from synapse.util.clock import Clock from tests import unittest @@ -483,8 +483,8 @@ class TestTransactionStore(ApplicationServiceTransactionStore, ApplicationServic class ApplicationServiceStoreConfigTestCase(unittest.HomeserverTestCase): - def _write_config(self, suffix: str, **kwargs: Any) -> str: - vals: JsonDict = { + def _write_config(self, suffix: str, **kwargs: str | list[str] | None) -> str: + vals: dict[str, Any] = { "id": "id" + suffix, "url": "url" + suffix, "as_token": "as_token" + suffix, @@ -636,3 +636,231 @@ class ApplicationServiceStoreConfigTestCase(unittest.HomeserverTestCase): ), self.hs, ) + + def test_proxy_prefix_works(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + store = ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + (appservice,) = store.get_app_services() + self.assertEqual(appservice.proxy_prefix, "rtc/livekit") + self.assertEqual(appservice.proxy_url, "http://proxy") + + def test_proxy_prefix_requires_proxy_url(self) -> None: + f1 = self._write_config( + suffix="1", + **{"io.element.msc4512.proxy_prefix": "rtc/livekit"}, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(KeyError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_proxy_url_requires_proxy_prefix(self) -> None: + f1 = self._write_config( + suffix="1", + **{"io.element.msc4512.proxy_url": "http://proxy"}, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(KeyError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_proxy_prefix_requires_non_empty_proxy_url(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ValueError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_proxy_url_requires_non_empty_proxy_prefix(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ValueError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_proxy_prefix_does_not_allow_reserved_values(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "not/allowed", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ValueError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_duplicate_proxy_prefix(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + f2 = self._write_config( + suffix="2", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "http://proxy2", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1, f2] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ConfigError) as cm: + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + e = cm.exception + self.assertIn(f1, str(e)) + self.assertIn(f2, str(e)) + self.assertIn("io.element.msc4512.proxy_prefix", str(e)) + + def test_overlapping_proxy_prefix(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + f2 = self._write_config( + suffix="2", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit/foobar", + "io.element.msc4512.proxy_url": "http://proxy2", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1, f2] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ConfigError) as cm: + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + e = cm.exception + self.assertIn(f1, str(e)) + self.assertIn(f2, str(e)) + self.assertIn("io.element.msc4512.proxy_prefix", str(e)) From a92a21600cc5ecfd0ca223b168bc02dac636138e Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 28 Aug 2026 08:53:30 -0400 Subject: [PATCH 130/158] MSC4140: update error for exceeding delay maximum (#20156) Use the HTTP code & errcode specified in the latest revision of the MSC (The changes were added by https://github.com/matrix-org/matrix-spec-proposals/pull/4140/changes/0c7193aaa07c9a2634dccd7cf3bd83e2733a806e) --- changelog.d/20156.misc | 1 + synapse/api/errors.py | 2 ++ synapse/handlers/delayed_events.py | 4 ++-- tests/rest/client/test_rooms.py | 4 ++-- 4 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changelog.d/20156.misc diff --git a/changelog.d/20156.misc b/changelog.d/20156.misc new file mode 100644 index 0000000000..116a238d8f --- /dev/null +++ b/changelog.d/20156.misc @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Update the error response for requesting to schedule a delayed event with a delay that exceeds the server-enforced maximum delay. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0c2b41225a..78fd8c3a60 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -137,6 +137,8 @@ class Codes(str, Enum): PROFILE_TOO_LARGE = "M_PROFILE_TOO_LARGE" KEY_TOO_LARGE = "M_KEY_TOO_LARGE" + DELAY_TOO_LARGE = "ORG.MATRIX.MSC4140_DELAY_TOO_LARGE" + # Part of MSC4155/MSC4380 INVITE_BLOCKED = "M_INVITE_BLOCKED" diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index cfaf16a30c..ac2da15112 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -373,9 +373,9 @@ class DelayedEventsHandler: requested_delay = delay.as_millis() max_delay = self._config.server.max_event_delay_duration.as_millis() raise SynapseError( - HTTPStatus.FORBIDDEN, + HTTPStatus.BAD_REQUEST, f"The requested delay ({requested_delay}ms) exceeds the allowed maximum ({max_delay}ms)", - Codes.FORBIDDEN, + Codes.DELAY_TOO_LARGE, ) self._event_creation_handler.validator.validate_builder( diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index a6f5043c92..78c960a537 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2567,9 +2567,9 @@ class RoomDelayedEventTestCase(RoomBase): ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) + self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) self.assertEqual( - Codes.FORBIDDEN, + "ORG.MATRIX.MSC4140_DELAY_TOO_LARGE", channel.json_body.get("errcode"), channel.json_body, ) From 6460a96d6f7abf54dde02c1a7e34a428e898e99f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 28 Aug 2026 18:20:32 +0300 Subject: [PATCH 131/158] Return unable to authorise join if there are unrecognized restricted join rules (#20132) Rooms may use custom restricted join `allow` rules for things like pre-validating joins before they're allowed into the room graph. However, currently any Synapse with invite permissions that doesn't know about the custom rule will throw a 403, which cancels the entire join process. By returning `M_UNABLE_TO_AUTHORISE_JOIN` instead, the origin server can keep trying other servers to join through. For reference, continuwuity already implements this correctly: https://forgejo.ellis.link/continuwuation/continuwuity/src/commit/5ceed2c2bb4ad80160cd1b7365afd8fe388261eb/src/api/server/make_join.rs#L255-L256 --- changelog.d/20132.bugfix | 1 + synapse/handlers/event_auth.py | 34 +++++++++++++++++++++++--------- synapse/handlers/room_member.py | 12 ++++++++--- synapse/handlers/room_summary.py | 16 ++++++++------- 4 files changed, 44 insertions(+), 19 deletions(-) create mode 100644 changelog.d/20132.bugfix diff --git a/changelog.d/20132.bugfix b/changelog.d/20132.bugfix new file mode 100644 index 0000000000..542596af63 --- /dev/null +++ b/changelog.d/20132.bugfix @@ -0,0 +1 @@ +Fixed joining rooms with unrecognized restricted join rules being rejected outright instead of returning the proper `M_UNABLE_TO_AUTHORISE_JOIN` error code. Contributed by @tulir @ Beeper. diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index a9a6990581..8c49d3e06f 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -234,11 +234,21 @@ class EventAuthHandler: # Get the rooms which allow access to this room and check if the user is # in any of them. - allowed_rooms = await self.get_rooms_that_allow_join(state_ids) + allowed_rooms, has_unknown_rules = await self.get_rooms_that_allow_join( + state_ids + ) if not await self.is_user_in_rooms(allowed_rooms, user_id): - # If this is a remote request, the user might be in an allowed room - # that we do not know about. - if not self._is_mine_id(user_id): + # If there are unknown allow rules, there could be other servers + # that are able to authorise the join. Alternatively, if this is + # a remote request, the user might be in an allowed room that we + # do not know about. + if has_unknown_rules: + raise SynapseError( + 400, + "Unrecognized restricted join rules found.", + Codes.UNABLE_AUTHORISE_JOIN, + ) + elif not self._is_mine_id(user_id): for room_id in allowed_rooms: if not await self._store.is_host_joined(room_id, self._server_name): raise SynapseError( @@ -289,7 +299,7 @@ class EventAuthHandler: async def get_rooms_that_allow_join( self, state_ids: StateMap[str] - ) -> StrCollection: + ) -> tuple[StrCollection, bool]: """ Generate a list of rooms in which membership allows access to a room. @@ -297,12 +307,16 @@ class EventAuthHandler: state_ids: The current state of the room the user wishes to join Returns: - A collection of room IDs. Membership in any of the rooms in the list grants the ability to join the target room. + A tuple of a collection of room IDs and a boolean indicating whether + there are any unknown allow rules. Membership in any of the rooms in + the list grants the ability to join the target room. If unknown rules + are present, failure to authorise the join should not be treated as + a permanent error. """ # If there's no join rule, then it defaults to invite (so this doesn't apply). join_rules_event_id = state_ids.get((EventTypes.JoinRules, ""), None) if not join_rules_event_id: - return () + return (), False # If the join rule is not restricted, this doesn't apply. join_rules_event = await self._store.get_event(join_rules_event_id) @@ -310,16 +324,18 @@ class EventAuthHandler: # If allowed is of the wrong form, then only allow invited users. allow_list = join_rules_event.content.get("allow", []) if not isinstance(allow_list, list): - return () + return (), False # Pull out the other room IDs, invalid data gets filtered. result = [] + has_unknown_rules = False for allow in allow_list: if not isinstance(allow, dict): continue # If the type is unexpected, skip it. if allow.get("type") != RestrictedJoinRuleTypes.ROOM_MEMBERSHIP: + has_unknown_rules = True continue room_id = allow.get("room_id") @@ -328,7 +344,7 @@ class EventAuthHandler: result.append(room_id) - return result + return result, has_unknown_rules async def is_user_in_rooms(self, room_ids: StrCollection, user_id: str) -> bool: """ diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 4f3a63f8bc..b73e7114c0 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1379,9 +1379,15 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): return True, list(servers_that_can_issue_invite) # Ensure the member should be allowed access via membership in a room. - await self.event_auth_handler.check_restricted_join_rules( - state_before_join, room_version, user_id, previous_membership - ) + try: + await self.event_auth_handler.check_restricted_join_rules( + state_before_join, room_version, user_id, previous_membership + ) + except SynapseError as e: + if e.errcode == Codes.UNABLE_AUTHORISE_JOIN: + servers_that_can_issue_invite.discard(self.hs.hostname) + return True, list(servers_that_can_issue_invite) + raise # If this is going to be a local join, additional information must # be included in the event content in order to efficiently validate diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 0bc6021fa7..4afd449292 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -685,9 +685,10 @@ class RoomSummaryHandler: if await self._event_auth_handler.has_restricted_join_rules( state_ids, room_version ): - allowed_rooms = ( - await self._event_auth_handler.get_rooms_that_allow_join(state_ids) - ) + ( + allowed_rooms, + _, + ) = await self._event_auth_handler.get_rooms_that_allow_join(state_ids) if await self._event_auth_handler.is_user_in_rooms( allowed_rooms, requester ): @@ -707,9 +708,10 @@ class RoomSummaryHandler: if await self._event_auth_handler.has_restricted_join_rules( state_ids, room_version ): - allowed_rooms = ( - await self._event_auth_handler.get_rooms_that_allow_join(state_ids) - ) + ( + allowed_rooms, + _, + ) = await self._event_auth_handler.get_rooms_that_allow_join(state_ids) for space_id in allowed_rooms: if await self._event_auth_handler.is_host_in_room(space_id, origin): return True @@ -822,7 +824,7 @@ class RoomSummaryHandler: if room_version and await self._event_auth_handler.has_restricted_join_rules( join_rules_state_ids, room_version ): - allowed_rooms = await self._event_auth_handler.get_rooms_that_allow_join( + allowed_rooms, _ = await self._event_auth_handler.get_rooms_that_allow_join( join_rules_state_ids ) if allowed_rooms: From 30355721f64c6e7654e7dd755ad2a679fd479b9c Mon Sep 17 00:00:00 2001 From: Soyoung Kim Date: Fri, 28 Aug 2026 23:34:27 +0200 Subject: [PATCH 132/158] Fix `public_chat` preset and deepmerge when it is needed (#20050) --- changelog.d/20050.bugfix | 1 + synapse/handlers/room.py | 58 ++++++++++++++++++++++++++------- tests/rest/client/test_rooms.py | 16 +++++---- 3 files changed, 57 insertions(+), 18 deletions(-) create mode 100644 changelog.d/20050.bugfix diff --git a/changelog.d/20050.bugfix b/changelog.d/20050.bugfix new file mode 100644 index 0000000000..d7a0025039 --- /dev/null +++ b/changelog.d/20050.bugfix @@ -0,0 +1 @@ +Fix `m.call.invite` state events not being given the proper power level on rooms created with the `public_chat` preset. Contributed by @famedly @itsoyou. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 9029cda1f5..0837325e89 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -21,6 +21,7 @@ """Contains functions for performing actions on rooms.""" +import copy import itertools import logging import math @@ -165,7 +166,7 @@ class RoomCreationHandler: "history_visibility": HistoryVisibility.SHARED, "original_invitees_have_ops": False, "guest_can_join": False, - "power_level_content_override": {EventTypes.CallInvite: 50}, + "power_level_content_override": {"events": {EventTypes.CallInvite: 50}}, }, } @@ -1529,6 +1530,8 @@ class RoomCreationHandler: alias for the room power_level_content_override: The power level content to override in the default power level event. + `power_level_content_override` doesn't apply when `initial_state` has + a power level state event content (i.e. `EventTypes.PowerLevels`). creator_join_profile: Set to override the displayname and avatar for the creating user in this room. @@ -1608,7 +1611,7 @@ class RoomCreationHandler: prev_state_events = [new_event.event_id] return new_event, new_unpersisted_context - preset_config, config = self._room_preset_config(room_config) + preset_name, preset_config = self._room_preset_config(room_config) if creation_event_with_context is None: # MSC2175 removes the creator field from the create event. @@ -1674,6 +1677,7 @@ class RoomCreationHandler: events_to_send = [] # We treat the power levels override specially as this needs to be one # of the first events that get sent into a room. + # If the `initial_state` has `EventTypes.PowerLevels` content, use it. pl_content = initial_state.pop((EventTypes.PowerLevels, ""), None) if pl_content is not None: power_event, power_context = await create_event( @@ -1681,6 +1685,8 @@ class RoomCreationHandler: ) events_to_send.append((power_event, power_context)) else: + # If the `initial_state` does not have `EventTypes.PowerLevels` content, + # use the default power level content. # Please update the docs for `default_power_level_content_override` when # updating the `events` dict below power_level_content: JsonDict = { @@ -1713,25 +1719,27 @@ class RoomCreationHandler: # have set these users as additional_creators, hence don't set the PL for creators as # that is invalid. if ( - config["original_invitees_have_ops"] + preset_config["original_invitees_have_ops"] and not room_version.msc4289_creator_power_enabled ): for invitee in invite_list: power_level_content["users"][invitee] = 100 - # If the user supplied a preset name e.g. "private_chat", - # we apply that preset - power_level_content.update(config["power_level_content_override"]) + # If the user supplied a preset name e.g. "private_chat", apply that + # preset's power level content. + power_level_content = self._deepmerge_power_level_content( + power_level_content, preset_config["power_level_content_override"] + ) # If the server config contains default_power_level_content_override, # and that contains information for this room preset, apply it. if self._default_power_level_content_override: - override = self._default_power_level_content_override.get(preset_config) + override = self._default_power_level_content_override.get(preset_name) if override is not None: power_level_content.update(override) # Finally, if the user supplied specific permissions for this room, - # apply those. + # apply those. Supplied room override wins over server config. if power_level_content_override: power_level_content.update(power_level_content_override) pl_event, pl_context = await create_event( @@ -1750,7 +1758,7 @@ class RoomCreationHandler: if (EventTypes.JoinRules, "") not in initial_state: join_rules_event, join_rules_context = await create_event( EventTypes.JoinRules, - {"join_rule": config["join_rules"]}, + {"join_rule": preset_config["join_rules"]}, True, ) events_to_send.append((join_rules_event, join_rules_context)) @@ -1758,12 +1766,12 @@ class RoomCreationHandler: if (EventTypes.RoomHistoryVisibility, "") not in initial_state: visibility_event, visibility_context = await create_event( EventTypes.RoomHistoryVisibility, - {"history_visibility": config["history_visibility"]}, + {"history_visibility": preset_config["history_visibility"]}, True, ) events_to_send.append((visibility_event, visibility_context)) - if config["guest_can_join"]: + if preset_config["guest_can_join"]: if (EventTypes.GuestAccess, "") not in initial_state: guest_access_event, guest_access_context = await create_event( EventTypes.GuestAccess, @@ -1778,7 +1786,7 @@ class RoomCreationHandler: ) events_to_send.append((event, context)) - if config["encrypted"] and not ignore_forced_encryption: + if preset_config["encrypted"] and not ignore_forced_encryption: encryption_event, encryption_context = await create_event( EventTypes.RoomEncryption, {"algorithm": RoomEncryptionAlgorithms.DEFAULT}, @@ -1868,6 +1876,32 @@ class RoomCreationHandler: f"You cannot create an encrypted room. user_level ({room_admin_level}) < send_level ({encryption_level})", ) + def _deepmerge_power_level_content( + self, power_level_content: JsonDict, override: JsonDict + ) -> JsonDict: + """Deep-merge `override` into `power_level_content`. + + Nested dicts (e.g. events, users) are merged recursively. All other values from + `override` replace those in `power_level_content`. + + Args: + power_level_content: The base power level content to update. + override: Values to merge on top of `power_level_content`. + + Returns: + The updated power level content. + """ + for key, value in override.items(): + existing = power_level_content.get(key) + if isinstance(existing, dict) and isinstance(value, dict): + power_level_content[key] = self._deepmerge_power_level_content( + dict(existing), value + ) + else: + # Copy so we don't accidentally modify the preset config. + power_level_content[key] = copy.deepcopy(value) + return power_level_content + def _room_preset_config(self, room_config: JsonDict) -> tuple[str, dict]: # The spec says rooms should default to private visibility if # `visibility` is not specified. diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 78c960a537..d0df8809c3 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -507,7 +507,8 @@ class RoomPermissionsTestCase(RoomBase): ) ) assert pl_event is not None - self.assertEqual(50, pl_event.content.get("m.call.invite")) + self.assertEqual(50, pl_event.content.get("events", {}).get("m.call.invite")) + self.assertEqual(50, pl_event.content.get("events", {}).get("m.room.name")) private_pl_event = self.get_success( self.store_controllers.state.get_current_state_event( @@ -515,7 +516,9 @@ class RoomPermissionsTestCase(RoomBase): ) ) assert private_pl_event is not None - self.assertEqual(None, private_pl_event.content.get("m.call.invite")) + self.assertEqual( + None, private_pl_event.content.get("events", {}).get("m.call.invite") + ) class RoomStateTestCase(RoomBase): @@ -1930,7 +1933,8 @@ class RoomPowerLevelOverridesTestCase(RoomBase): def test_default_power_levels_with_room_override(self) -> None: """ Create a room, providing power level overrides. - Confirm that the room's power levels reflect the overrides. + When the `power_level_content_override` was provided, it should replace the + default power levels. See https://github.com/matrix-org/matrix-spec/issues/492 - currently we overwrite each key of power_level_content_override @@ -1959,9 +1963,9 @@ class RoomPowerLevelOverridesTestCase(RoomBase): ) def test_power_levels_with_server_override(self) -> None: """ - With a server configured to modify the room-level defaults, - Create a room, without providing any extra power level overrides. - Confirm that the room's power levels reflect the server-level overrides. + With a server configured `default_power_level_content_override`, creating a room + without `power_level_content_override` should result in the server-level overrides + being applied. Similar to https://github.com/matrix-org/matrix-spec/issues/492, we overwrite each key of power_level_content_override completely. From 955d0cf70d4ff2a2a754619eb9223cfd5be1e6b5 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 31 Aug 2026 17:25:54 -0500 Subject: [PATCH 133/158] 1.160.0rc2 --- CHANGES.md | 8 ++++++++ changelog.d/20147.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/20147.bugfix diff --git a/CHANGES.md b/CHANGES.md index 670a61782e..d6d9b22f2f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +# Synapse 1.160.0rc2 (2026-08-31) + +## Bugfixes + +- Fix sending custom profile field removals to legacy sync clients when the field is deleted using the profile field delete endpoint. ([\#20147](https://github.com/element-hq/synapse/issues/20147)) + + + # Synapse 1.160.0rc1 (2026-08-25) ## Features diff --git a/changelog.d/20147.bugfix b/changelog.d/20147.bugfix deleted file mode 100644 index 74768dc63e..0000000000 --- a/changelog.d/20147.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix sending custom profile field removals to legacy sync clients when the field is deleted using the profile field delete endpoint. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index a0325ac3bc..63c2d5fcad 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.160.0~rc2) stable; urgency=medium + + * New synapse release 1.160.0rc2. + + -- Synapse Packaging team Mon, 31 Aug 2026 22:25:15 +0000 + matrix-synapse-py3 (1.160.0~rc1) stable; urgency=medium * New synapse release 1.160.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 751b2805a5..69a493ee09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.160.0rc1" +version = "1.160.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 89363f45d217fe5805a9e7b348fae702ba71af4d Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Tue, 1 Sep 2026 05:13:56 -0400 Subject: [PATCH 134/158] Relax validation of signature upload for master keys, and allow updates (#19915) Co-authored-by: Andrew Morgan --- changelog.d/19915.feature | 1 + synapse/handlers/e2e_keys.py | 34 ++++--- synapse/storage/database.py | 1 + .../storage/databases/main/end_to_end_keys.py | 97 ++++++++++++++++--- .../09_e2e_cross_signing_signatures_index.sql | 21 ++++ ...signing_signatures_add_key_id_to_index.sql | 18 ++++ tests/handlers/test_e2e_keys.py | 97 +++++++++++++++++++ tests/storage/test_end_to_end_keys.py | 96 ++++++++++++++++++ 8 files changed, 339 insertions(+), 26 deletions(-) create mode 100644 changelog.d/19915.feature create mode 100644 synapse/storage/schema/main/delta/94/09_e2e_cross_signing_signatures_index.sql create mode 100644 synapse/storage/schema/main/delta/94/10_e2e_cross_signing_signatures_add_key_id_to_index.sql diff --git a/changelog.d/19915.feature b/changelog.d/19915.feature new file mode 100644 index 0000000000..39446ae6a1 --- /dev/null +++ b/changelog.d/19915.feature @@ -0,0 +1 @@ +Don't validate signatures with unknown algorithms for master keys, and allow updates to signatures. \ No newline at end of file diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 64f705a3da..6284f75319 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -1272,21 +1272,27 @@ class E2eKeysHandler: master_key_signature_list = [] sigs = signed_master_key["signatures"] for signing_key_id, signature in sigs[user_id].items(): - _, signing_device_id = signing_key_id.split(":", 1) - if ( - signing_device_id not in devices - or signing_key_id not in devices[signing_device_id]["keys"] - ): - # signed by an unknown device, or the - # device does not have the key - raise SynapseError(400, "Invalid signature", Codes.INVALID_SIGNATURE) + algorithm, signing_device_id = signing_key_id.split(":", 1) + # we only check the signature for known algorithms + if algorithm == "ed25519": + if ( + signing_device_id not in devices + or signing_key_id not in devices[signing_device_id]["keys"] + ): + # signed by an unknown device, or the + # device does not have the key + raise SynapseError( + 400, "Invalid signature", Codes.INVALID_SIGNATURE + ) - # get the key and check the signature - pubkey = devices[signing_device_id]["keys"][signing_key_id] - verify_key = decode_verify_key_bytes(signing_key_id, decode_base64(pubkey)) - _check_device_signature( - user_id, verify_key, signed_master_key, stored_master_key - ) + # get the key and check the signature + pubkey = devices[signing_device_id]["keys"][signing_key_id] + verify_key = decode_verify_key_bytes( + signing_key_id, decode_base64(pubkey) + ) + _check_device_signature( + user_id, verify_key, signed_master_key, stored_master_key + ) master_key_signature_list.append( SignatureListItem(signing_key_id, user_id, master_key_id, signature) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 023014276b..1de0cc5fe0 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -120,6 +120,7 @@ UNIQUE_INDEX_BACKGROUND_UPDATES = { "event_push_summary": "event_push_summary_unique_index2", "receipts_linearized": "receipts_linearized_unique_index", "receipts_graph": "receipts_graph_unique_index", + "e2e_cross_signing_signatures": "e2e_cross_signing_signatures_add_key_id_to_index", } diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index c93ebd3dda..3075d6a05e 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -78,6 +78,10 @@ class DeviceKeyLookupResult: class EndToEndKeyBackgroundStore(SQLBaseStore): + CROSS_SIGNING_KEYS_REMOVE_DUPLICATES_NAME = ( + "e2e_cross_signing_signatures_remove_duplicates" + ) + def __init__( self, database: DatabasePool, @@ -101,6 +105,80 @@ class EndToEndKeyBackgroundStore(SQLBaseStore): columns=("user_id", "device_id", "algorithm", "ts_added_ms"), ) + self.db_pool.updates.register_background_update_handler( + self.CROSS_SIGNING_KEYS_REMOVE_DUPLICATES_NAME, + self._background_cross_signing_signatures_remove_duplicates, + ) + + self.db_pool.updates.register_background_index_update( + update_name="e2e_cross_signing_signatures_add_key_id_to_index", + index_name="e2e_cross_signing_signatures_idx3", + table="e2e_cross_signing_signatures", + columns=("user_id", "target_user_id", "target_device_id", "key_id"), + unique=True, + replaces_index="e2e_cross_signing_signatures2_idx", + ) + + async def _background_cross_signing_signatures_remove_duplicates( + self, progress: dict, batch_size: int + ) -> int: + """Removes duplicate cross-signing signatures so that we can add a + unique index on `(user_id, target_user_id, target_device_id, key_id)` to + `e2e_cross_signing_signatures`. + """ + + def _remove_duplicate_signatures_txn(txn: LoggingTransaction) -> int: + sql = """ + SELECT user_id, key_id, target_user_id, target_device_id, MAX(signature) + FROM e2e_cross_signing_signatures + GROUP BY user_id, key_id, target_user_id, target_device_id + HAVING COUNT(*) > 1 + LIMIT ? + """ + txn.execute(sql, (batch_size,)) + duplicate_keys = cast(list[tuple[str, str, str, str, str]], list(txn)) + + for ( + user_id, + key_id, + target_user_id, + target_device_id, + signature, + ) in duplicate_keys: + sql = """ + DELETE FROM e2e_cross_signing_signatures + WHERE + user_id = ? AND + key_id = ? AND + target_user_id = ? AND + target_device_id = ? + """ + txn.execute(sql, (user_id, key_id, target_user_id, target_device_id)) + + sql = """ + INSERT INTO e2e_cross_signing_signatures + (user_id, key_id, target_user_id, target_device_id, signature) + VALUES + (?, ?, ?, ?, ?) + """ + txn.execute( + sql, (user_id, key_id, target_user_id, target_device_id, signature) + ) + + return len(duplicate_keys) + + number_deleted = await self.db_pool.runInteraction( + self.CROSS_SIGNING_KEYS_REMOVE_DUPLICATES_NAME, + _remove_duplicate_signatures_txn, + ) + + if number_deleted < batch_size: + await self.db_pool.updates._end_background_update( + self.CROSS_SIGNING_KEYS_REMOVE_DUPLICATES_NAME + ) + + return number_deleted + class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorkerStore): def __init__( @@ -1808,7 +1886,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker ) async def store_e2e_cross_signing_signatures( - self, user_id: str, signatures: "Iterable[SignatureListItem]" + self, user_id: str, signatures: "list[SignatureListItem]" ) -> None: """Stores cross-signing signatures. @@ -1819,28 +1897,23 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker def _store_e2e_cross_signing_signatures( txn: LoggingTransaction, - signatures: "Iterable[SignatureListItem]", + signatures: "list[SignatureListItem]", ) -> None: - self.db_pool.simple_insert_many_txn( + self.db_pool.simple_upsert_many_txn( txn, "e2e_cross_signing_signatures", - keys=( - "user_id", - "key_id", - "target_user_id", - "target_device_id", - "signature", - ), - values=[ + key_names=("user_id", "key_id", "target_user_id", "target_device_id"), + key_values=[ ( user_id, item.signing_key_id, item.target_user_id, item.target_device_id, - item.signature, ) for item in signatures ], + value_names=("signature",), + value_values=[(item.signature,) for item in signatures], ) to_invalidate = [ diff --git a/synapse/storage/schema/main/delta/94/09_e2e_cross_signing_signatures_index.sql b/synapse/storage/schema/main/delta/94/09_e2e_cross_signing_signatures_index.sql new file mode 100644 index 0000000000..cde6ca1500 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/09_e2e_cross_signing_signatures_index.sql @@ -0,0 +1,21 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Remove any rows `e2e_cross_signing_signatures` that have duplicate cross-signing signatures. +-- Ensures that rows are unique across `(user_id, target_user_id, target_device_id, key_id)`, so that +-- we can create an index on those columns. See +-- `./10_e2e_cross_signing_signatures_add_key_id_to_index.sql`, which adds said +-- index. + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9409, 'e2e_cross_signing_signatures_remove_duplicates', '{}'); diff --git a/synapse/storage/schema/main/delta/94/10_e2e_cross_signing_signatures_add_key_id_to_index.sql b/synapse/storage/schema/main/delta/94/10_e2e_cross_signing_signatures_add_key_id_to_index.sql new file mode 100644 index 0000000000..65b5df6c86 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/10_e2e_cross_signing_signatures_add_key_id_to_index.sql @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Adds the `key_id` to the e2e_cross_signing_signatures index, since the +-- ("user_id", "key_id", "target_user_id", "target_device_id") should be +-- unique. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9410, 'e2e_cross_signing_signatures_add_key_id_to_index', '{}'); \ No newline at end of file diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index a4f9d55a13..4790c339d9 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -817,6 +817,103 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): self.assertDictEqual(devices["device_keys"][local_user]["abc"], device_key_1) self.assertDictEqual(devices["device_keys"][local_user]["def"], device_key_2) + def test_update_signature_master_key(self) -> None: + """should be able to update a signature on the Master signing key with an unknown algorithm""" + local_user = "@boris:" + self.hs.hostname + master_key: JsonDict = { + # private key: HvQBbU+hc2Zr+JP1sE0XwBe1pfZZEYtJNPJLZJtS+F8 + "user_id": local_user, + "usage": ["master"], + "keys": { + "ed25519:EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ": "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ" + }, + "signatures": {local_user: {"unknown:abcdefg": "abcdefg"}}, + } + self_signing_key = { + # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0 + "user_id": local_user, + "usage": ["self_signing"], + "keys": { + "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk": "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk" + }, + } + master_signing_key = key.decode_signing_key_base64( + "ed25519", + "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ", + "HvQBbU+hc2Zr+JP1sE0XwBe1pfZZEYtJNPJLZJtS+F8", + ) + sign.sign_json(self_signing_key, local_user, master_signing_key) + self.get_success( + self.handler.upload_signing_keys_for_user( + local_user, + {"master_key": master_key, "self_signing_key": self_signing_key}, + ) + ) + + device_key: JsonDict = { + "user_id": local_user, + "device_id": "abc", + "algorithms": [ + "m.olm.curve25519-aes-sha2", + RoomEncryptionAlgorithms.MEGOLM_V1_AES_SHA2, + ], + "keys": { + "ed25519:abc": "base64+ed25519+key", + "curve25519:abc": "base64+curve25519+key", + }, + "signatures": {local_user: {"ed25519:abc": "base64+signature"}}, + } + self.get_success( + self.handler.upload_keys_for_user( + local_user, "abc", {"device_keys": device_key} + ) + ) + + # Update the signature and upload it. + master_key["signatures"][local_user]["unknown:abcdefg"] = "ABCDEFG" + self.get_success( + self.handler.upload_signatures_for_device_keys( + local_user, + { + local_user: { + "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ": master_key + } + }, + ) + ) + + devices = self.get_success( + self.handler.query_devices( + {"device_keys": {local_user: []}}, 0, local_user, "device123" + ) + ) + if "unsigned" in devices["master_keys"][local_user]: + del devices["master_keys"][local_user]["unsigned"] + self.assertDictEqual(devices["master_keys"][local_user], master_key) + + # Update the signature again and upload it. + master_key["signatures"][local_user]["unknown:abcdefg"] = "AbCdEfG" + self.get_success( + self.handler.upload_signatures_for_device_keys( + local_user, + { + local_user: { + "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ": master_key + } + }, + ) + ) + + # Assert that we were able to update the signature. + devices = self.get_success( + self.handler.query_devices( + {"device_keys": {local_user: []}}, 0, local_user, "device123" + ) + ) + if "unsigned" in devices["master_keys"][local_user]: + del devices["master_keys"][local_user]["unsigned"] + self.assertDictEqual(devices["master_keys"][local_user], master_key) + def test_self_signing_key_doesnt_show_up_as_device(self) -> None: """signing keys should be hidden when fetching a user's devices""" local_user = "@boris:" + self.hs.hostname diff --git a/tests/storage/test_end_to_end_keys.py b/tests/storage/test_end_to_end_keys.py index 24fdb0bf6d..537393f3fc 100644 --- a/tests/storage/test_end_to_end_keys.py +++ b/tests/storage/test_end_to_end_keys.py @@ -22,6 +22,7 @@ from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer +from synapse.storage.database import LoggingTransaction from synapse.util.clock import Clock from tests.unittest import HomeserverTestCase @@ -118,3 +119,98 @@ class EndToEndKeyStoreTestCase(HomeserverTestCase): self.assertIn("user2", res) self.assertNotIn("device1", res["user2"]) self.assertIn("device2", res["user2"]) + + def test_bg_signatures_migration(self) -> None: + updater = self.hs.get_datastores().main.db_pool.updates + + # drop the constraint so we can insert duplicate signatures + def f(txn: LoggingTransaction) -> None: + txn.execute("DROP INDEX e2e_cross_signing_signatures_idx3") + + self.get_success(self.store.db_pool.runInteraction("", f)) + + # save multiple copies of the same key in the database + for _i in range(2): + self.get_success( + self.store.db_pool.simple_insert( + "e2e_cross_signing_signatures", + { + "user_id": "@alice:example.org", + "key_id": "ed25519:abcdefg", + "target_user_id": "@alice:example.org", + "target_device_id": "hijklmnop", + "signature": "some+signature", + }, + ) + ) + + for _i in range(2): + self.get_success( + self.store.db_pool.simple_insert( + "e2e_cross_signing_signatures", + { + "user_id": "@alice:example.org", + "key_id": "ed25519:hijklmnop", + "target_user_id": "@alice:example.org", + "target_device_id": "abcdefg", + "signature": "some+signature", + }, + ) + ) + + # run the background task to remove duplicates + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + values={ + "update_name": "e2e_cross_signing_signatures_remove_duplicates", + "progress_json": "{}", + }, + ) + ) + + self.get_success( + updater.run_background_updates(False), + ) + + # re-add the unique index + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + values={ + "update_name": "e2e_cross_signing_signatures_add_key_id_to_index", + "progress_json": "{}", + }, + ) + ) + + self.get_success( + updater.run_background_updates(False), + ) + + # check that we only have one copy of each key + expected_values = [ + ( + "@alice:example.org", + "ed25519:abcdefg", + "@alice:example.org", + "hijklmnop", + "some+signature", + ), + ( + "@alice:example.org", + "ed25519:hijklmnop", + "@alice:example.org", + "abcdefg", + "some+signature", + ), + ] + + res = self.get_success( + self.store.db_pool.execute( + "", + "SELECT user_id, key_id, target_user_id, target_device_id, signature from e2e_cross_signing_signatures ORDER BY key_id", + ) + ) + self.assertEqual(len(res), len(expected_values)) + self.assertEqual(res, expected_values) From 03a3570a2100971a95ac0f8740bd83e213dc6798 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:47:05 +0100 Subject: [PATCH 135/158] Bump the minor-and-patches group across 1 directory with 5 updates (#20176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the minor-and-patches group with 5 updates in the / directory: | Package | From | To | | --- | --- | --- | | [actions/checkout](https://github.com/actions/checkout) | `7.0.0` | `7.0.1` | | [Swatinem/rust-cache](https://github.com/swatinem/rust-cache) | `2.9.1` | `2.9.2` | | [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) | `4.2.0` | `4.3.0` | | [docker/login-action](https://github.com/docker/login-action) | `4.5.2` | `4.6.0` | | [dorny/paths-filter](https://github.com/dorny/paths-filter) | `4.0.2` | `4.0.3` | Updates `actions/checkout` from 7.0.0 to 7.0.1
Release notes

Sourced from actions/checkout's releases.

v7.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v7...v7.0.1

Changelog

Sourced from actions/checkout's changelog.

Changelog

v7.0.1

v7.0.0

v6.0.3

v6.0.2

v6.0.1

v6.0.0

v5.0.1

v5.0.0

v4.3.1

v4.3.0

v4.2.2

v4.2.1

... (truncated)

Commits

Updates `Swatinem/rust-cache` from 2.9.1 to 2.9.2
Release notes

Sourced from Swatinem/rust-cache's releases.

v2.9.2

What's Changed

New Contributors

Full Changelog: https://github.com/Swatinem/rust-cache/compare/v2.9.1...v2.9.2

Changelog

Sourced from Swatinem/rust-cache's changelog.

Changelog

2.9.2

  • Fix credentials.toml cleanup
  • Improvements to cleanup, preserving more valid targets
  • Improvements to cargo install handling
  • Correctly sort/dedupe Rust versions

2.9.1

  • Fix regression in hash calculation

2.9.0

  • Update to node24
  • Support running from within a nix shell
  • Consider all installed toolchains for cache key
  • Use case-insensitive comparison to determine exact cache hit

2.8.2

  • Don't overwrite env for cargo-metadata call

2.8.1

  • Set empty CARGO_ENCODED_RUSTFLAGS when retrieving metadata
  • Various dependency updates

2.8.0

  • Add support for warpbuild cache provider
  • Add new cache-workspace-crates feature

2.7.8

  • Include CPU arch in the cache key

2.7.7

  • Also cache cargo install metadata

2.7.6

  • Allow opting out of caching $CARGO_HOME/bin
  • Add runner OS in cache key
  • Adds an option to do lookup-only of the cache

2.7.5

... (truncated)

Commits
  • 6323deb 2.9.2
  • b16e8d7 bump rollup and rebuild
  • 3bf42ac invert target/profile check in cleanup
  • 6e5b278 correctly sort and dedupe Rust versions
  • 5adc05f Bump the actions group across 1 directory with 3 updates (#368)
  • 66b1e95 fix: support Cargo V2 build dir layout (#371)
  • 72d126e Merge pull request #367 from Swatinem/dependabot/npm_and_yarn/dev-patch-2b495...
  • 48968d2 Bump the dev-patch group with 2 updates
  • 9f151ac update dependencies, rebuild
  • 0e24e5d Bump the actions group across 1 directory with 6 updates (#364)
  • Additional commits viewable in compare view

Updates `docker/setup-buildx-action` from 4.2.0 to 4.3.0
Release notes

Sourced from docker/setup-buildx-action's releases.

v4.3.0

Full Changelog: https://github.com/docker/setup-buildx-action/compare/v4.2.0...v4.3.0

Commits
  • 37fe631 Merge pull request #595 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • b5c4f91 [dependabot skip] chore: update generated content
  • 3e93b63 build(deps): bump @​docker/actions-toolkit from 0.92.0 to 0.95.0
  • e527031 Merge pull request #600 from docker/dependabot/npm_and_yarn/brace-expansion-1...
  • c68814b [dependabot skip] chore: update generated content
  • 3f891b0 build(deps): bump brace-expansion from 1.1.13 to 1.1.18
  • 787db26 Merge pull request #585 from docker/dependabot/npm_and_yarn/js-yaml-5.2.1
  • f779368 [dependabot skip] chore: update generated content
  • 7d5e604 build(deps): bump js-yaml from 5.2.0 to 5.3.0
  • 292c2fb Merge pull request #590 from docker/dependabot/github_actions/actions/setup-n...
  • Additional commits viewable in compare view

Updates `docker/login-action` from 4.5.2 to 4.6.0
Release notes

Sourced from docker/login-action's releases.

v4.6.0

Full Changelog: https://github.com/docker/login-action/compare/v4.5.2...v4.6.0

Commits
  • dbcb813 Merge pull request #1051 from docker/dependabot/npm_and_yarn/aws-sdk-dependen...
  • 5bcb015 [dependabot skip] chore: update generated content
  • b30b2f2 build(deps): bump the aws-sdk-dependencies group across 1 directory with 2 up...
  • 9087f1e Merge pull request #1057 from docker/dependabot/npm_and_yarn/js-yaml-5.2.2
  • 0009830 [dependabot skip] chore: update generated content
  • 2325523 build(deps): bump js-yaml from 5.2.1 to 5.2.2
  • 4ec1d4a Merge pull request #1056 from docker/dependabot/npm_and_yarn/postcss-8.5.22
  • 5fc99ba Merge pull request #1053 from docker/dependabot/github_actions/aws-actions/co...
  • e512bd5 Merge pull request #1052 from docker/dependabot/github_actions/codeql-actions...
  • a146c91 Merge pull request #1059 from crazy-max/harden-buildx-scope-paths
  • Additional commits viewable in compare view

Updates `dorny/paths-filter` from 4.0.2 to 4.0.3
Release notes

Sourced from dorny/paths-filter's releases.

v4.0.3

What's Changed

Security

New Contributors

Full Changelog: https://github.com/dorny/paths-filter/compare/v4...v4.0.3

Changelog

Sourced from dorny/paths-filter's changelog.

Changelog

v4.0.3

v4.0.2

v4.0.1

v4.0.0

v3.0.4

v3.0.3

v3.0.2

v3.0.1

v3.0.0

v2.11.1

v2.11.0

v2.10.2

... (truncated)

Commits
  • ceb8a2b Update CHANGELOG.md for v4.0.3 and v3.0.4 (#327)
  • ef09b88 Document safe handling of file list outputs in workflows (#326)
  • 44adc5b Merge commit from fork
  • 4711b7a feat: add 'some-with-excludes' predicate quantifier (#322)
  • 93c889f fix: escape multi-line filenames in list-files shell and csv output
  • b41dfa9 docs: add contents permission to PR example (#248)
  • 9af6e5a fix: scope base-ignored warning to API path (#319)
  • cae9006 docs: update outputs in readme to account for the 'every' predicate-quantifie...
  • See full diff in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/complement_tests.yml | 2 +- .github/workflows/docker.yml | 16 +++++++------- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 6 +++--- .github/workflows/push_complement_image.yml | 2 +- .github/workflows/release-artifacts.yml | 2 +- .github/workflows/schema_diff.yml | 2 +- .github/workflows/tests.yml | 24 ++++++++++----------- .github/workflows/twisted_trunk.yml | 6 +++--- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index a2510a247c..95d8c323ef 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -53,7 +53,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 # We use `poetry` in `complement.sh` - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 47ea46cd9f..90cfd9becf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -41,13 +41,13 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -79,7 +79,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -136,14 +136,14 @@ jobs: merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io @@ -176,14 +176,14 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} password: ${{ steps.import-secrets.outputs.OCI_PASSWORD }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Install Cosign uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 22ef26e78c..cc482a89b5 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -27,7 +27,7 @@ jobs: with: toolchain: ${{ env.RUST_VERSION }} components: clippy, rustfmt - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index c751968df4..6dd277be9a 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -47,7 +47,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. @@ -83,7 +83,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.postgres-version }} @@ -157,7 +157,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Ensure sytest runs `pip install` # Delete the lockfile so sytest will `pip install` rather than `poetry install` diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index cc20aad2aa..6fc7be4329 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -52,7 +52,7 @@ jobs: with: poetry-version: "2.4.1" - name: Login to registry - uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 291c0c75bd..75ceb0f533 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -67,7 +67,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Set up docker layer caching uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/schema_diff.yml b/.github/workflows/schema_diff.yml index 58aa720b9a..f0d148b071 100644 --- a/.github/workflows/schema_diff.yml +++ b/.github/workflows/schema_diff.yml @@ -56,7 +56,7 @@ jobs: --health-timeout 5s --health-retries 5 \ localhost/postgres-faketime - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fe33ead2f1..f17b03c0f3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }} golangci: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.golangci }} steps: - - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 + - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 id: filter # We only check on PRs if: startsWith(github.ref, 'refs/pull/') @@ -111,7 +111,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" @@ -177,7 +177,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -241,7 +241,7 @@ jobs: with: components: clippy toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo clippy -- -D warnings @@ -260,7 +260,7 @@ jobs: with: toolchain: ${{ env.RUST_NIGHTLY_VERSION }} components: clippy - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo clippy --all-features -- -D warnings @@ -277,7 +277,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -316,7 +316,7 @@ jobs: # `.rustfmt.toml`. toolchain: ${{ env.RUST_NIGHTLY_VERSION }} components: rustfmt - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo fmt --check @@ -432,7 +432,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -476,7 +476,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 # There aren't wheels for some of the older deps, so we need to install # their build dependencies @@ -591,7 +591,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Run SyTest run: /bootstrap.sh synapse @@ -732,7 +732,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo test @@ -753,7 +753,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_NIGHTLY_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo bench --no-run diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 88b279ffce..6cbb61be6b 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -48,7 +48,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -76,7 +76,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -121,7 +121,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Patch dependencies # Note: The poetry commands want to create a virtualenv in /src/.venv/, From 652558495f8b4cc5169965b22aa7b08dd6451e3a Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:45:39 +0100 Subject: [PATCH 136/158] Remove unstable `/auth_issuer` endpoint (#20163) --- changelog.d/20163.removal | 1 + synapse/rest/client/auth_metadata.py | 57 ------------------------- tests/rest/client/test_auth_metadata.py | 14 ------ 3 files changed, 1 insertion(+), 71 deletions(-) create mode 100644 changelog.d/20163.removal diff --git a/changelog.d/20163.removal b/changelog.d/20163.removal new file mode 100644 index 0000000000..44d8ccb144 --- /dev/null +++ b/changelog.d/20163.removal @@ -0,0 +1 @@ +Drop `GET /_matrix/client/unstable/org.matrix.msc2965/auth_issuer` endpoint which never ended up being used. \ No newline at end of file diff --git a/synapse/rest/client/auth_metadata.py b/synapse/rest/client/auth_metadata.py index 42decfdd6a..c5b874b533 100644 --- a/synapse/rest/client/auth_metadata.py +++ b/synapse/rest/client/auth_metadata.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging import typing from synapse.api.auth.mas import MasDelegatedAuth @@ -26,61 +25,6 @@ if typing.TYPE_CHECKING: from synapse.server import HomeServer -logger = logging.getLogger(__name__) - - -class AuthIssuerServlet(RestServlet): - """ - Advertises what OpenID Connect issuer clients should use to authorise users. - This endpoint was defined in a previous iteration of MSC2965, and is still - used by some clients. - """ - - PATTERNS = client_patterns( - "/org.matrix.msc2965/auth_issuer$", - unstable=True, - releases=(), - ) - - def __init__(self, hs: "HomeServer"): - super().__init__() - self._config = hs.config - self._auth = hs.get_auth() - - async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: - # This endpoint is unauthenticated and the response only depends on - # the metadata we get from Matrix Authentication Service. Internally, - # MasDelegatedAuth.issuer() is already caching the - # response in memory anyway. Ideally we would follow any Cache-Control directive - # given by MAS, but this is fine for now. - # - # - `public` means it can be cached both in the browser and in caching proxies - # - `max-age` controls how long we cache on the browser side. 10m is sane enough - # - `s-maxage` controls how long we cache on the proxy side. Since caching - # proxies usually have a way to purge caches, it is fine to cache there for - # longer (1h), and issue cache invalidations in case we need it - # - `stale-while-revalidate` allows caching proxies to serve stale content while - # revalidating in the background. This is useful for making this request always - # 'snappy' to end users whilst still keeping it fresh - request.setHeader( - b"Cache-Control", - b"public, max-age=600, s-maxage=3600, stale-while-revalidate=600", - ) - - if self._config.mas.enabled: - assert isinstance(self._auth, MasDelegatedAuth) - return 200, {"issuer": await self._auth.issuer()} - - else: - # Wouldn't expect this to be reached: the servelet shouldn't have been - # registered. Still, fail gracefully if we are registered for some reason. - raise SynapseError( - 404, - "OIDC discovery has not been configured on this homeserver", - Codes.NOT_FOUND, - ) - - class AuthMetadataServlet(RestServlet): """ Advertises the OAuth 2.0 server metadata for the homeserver. @@ -139,5 +83,4 @@ class AuthMetadataServlet(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: if hs.config.mas.enabled: - AuthIssuerServlet(hs).register(http_server) AuthMetadataServlet(hs).register(http_server) diff --git a/tests/rest/client/test_auth_metadata.py b/tests/rest/client/test_auth_metadata.py index e9f8597c5a..683fce46e3 100644 --- a/tests/rest/client/test_auth_metadata.py +++ b/tests/rest/client/test_auth_metadata.py @@ -27,20 +27,6 @@ from synapse.rest.client import auth_metadata from tests.unittest import HomeserverTestCase -class AuthIssuerTestCase(HomeserverTestCase): - servlets = [ - auth_metadata.register_servlets, - ] - - def test_returns_404_when_mas_disabled(self) -> None: - # Make an unauthenticated request for the discovery info. - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) - - @parameterized_class( ("endpoint",), [ From fbd36aab6f8405328f7ff6b9b1dfde97b08b8642 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:46:21 +0000 Subject: [PATCH 137/158] Bump the patches group across 1 directory with 3 updates (#20175) Signed-off-by: dependabot[bot] --- Cargo.lock | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3cb739d495..2215b23bab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,9 +25,9 @@ checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", @@ -218,9 +218,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -243,15 +243,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -260,38 +260,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", From 54bfa1a01bc70e44a77c4adb469cea6a912ba6d9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:12:49 +0100 Subject: [PATCH 138/158] MSC4242: State DAGs (storage) (#19718) Split out from https://github.com/element-hq/synapse/pull/19425 Add storage functions needed for federation support for MSC4242: - `get_missing_events_state_dag`: will be used to satisfy `/get_missing_events` requests. - `get_state_dag`: will be used to satisfy `/send_join` requests. Part of a series of 5x PRs to land the federation part of MSC4242 (storage, fedclient, serving, inbound-joins, inbound-pulls). ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Eric Eastwood --- changelog.d/19718.misc | 1 + synapse/storage/database.py | 8 + .../databases/main/event_federation.py | 195 +++++++ tests/storage/test_event_federation.py | 481 ++++++++++++++++++ 4 files changed, 685 insertions(+) create mode 100644 changelog.d/19718.misc diff --git a/changelog.d/19718.misc b/changelog.d/19718.misc new file mode 100644 index 0000000000..5914943f34 --- /dev/null +++ b/changelog.d/19718.misc @@ -0,0 +1 @@ +Add storage functions for future [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) work. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 1de0cc5fe0..469639fbf3 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -2637,6 +2637,9 @@ def make_in_list_sql_clause( using the `ANY` form on postgres means that it views queries with different length iterables as the same, helping the query stats. + An empty `iterable` yields a constant clause: nothing can be a member of an empty + list, so the clause is `FALSE` (or `TRUE` when `negative`). + Args: database_engine column: Name of the column @@ -2647,6 +2650,11 @@ def make_in_list_sql_clause( A tuple of SQL query and the args """ + if not iterable: + # Spell the empty case out rather than relying on each engine's handling of an + # empty list: sqlite permits `IN ()` but postgres does not. + return ("TRUE" if negative else "FALSE"), [] + if database_engine.supports_using_any_list: # This should hopefully be faster, but also makes postgres query # stats easier to understand. diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 4cb55f8c46..1908b26f68 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -38,6 +38,7 @@ from synapse.api.constants import MAX_DEPTH from synapse.api.errors import StoreError from synapse.api.room_versions import EventFormatVersions, RoomVersion from synapse.events import EventBase, make_event_from_dict +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.logging.opentracing import tag_args, trace from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import wrap_as_background_process @@ -1194,6 +1195,200 @@ class EventFederationWorkerStore( # Return all events where not all sets can reach them. return {eid for eid, n in event_to_missing_sets.items() if n} + # FIXME(2026-04-22): Remove comment when used. Unused currently, but will be used in + # future MSC4242 PRs. + async def get_state_dag( + self, room_id: str, forward_extrems: set[str] + ) -> dict[str, MSC4242Event]: + """Get the current state DAG for the given room. + + This function is called when calculating a /send_join response. + This does not check that the room is an state DAG room, so check this + before calling this function! + + This functions guarantees that the returned state DAG is connected. + + Args: + room_id: The room to get the state dag for + forward_extrems: latest event IDs in the room. The state DAG is all events reachable from these events. + Returns: + A map of event_id => event + """ + + def _get_state_events_txn(txn: LoggingTransaction, room_id: str) -> list[str]: + sql = """ + SELECT event_id FROM msc4242_state_dag_edges WHERE room_id = ? + """ + txn.execute(sql, (room_id,)) + event_ids = [ev_id for (ev_id,) in txn] + return event_ids + + # Pull out all state events for this room using the events_by_room_and_type index. + event_ids = await self.db_pool.runInteraction( + "_get_state_events_txn", + _get_state_events_txn, + room_id, + ) + event_map = await self.get_events(event_ids) + # Filter the returned state events to only include ones on the paths back from the forward + # extremities. + result: dict[str, MSC4242Event] = {} + next_ids = forward_extrems + seen: set[str] = set() + while len(next_ids) > 0: + # Pull the event and add the prev_state_events. + # We must have the event. + event_id = next_ids.pop() + if event_id in seen: + continue + seen.add(event_id) + ev = event_map[event_id] + # `prev_state_events` only exists on MSC4242 event formats, and this is only + # called for state DAG rooms. + assert supports_msc4242_state_dag(ev) + result[event_id] = ev + for prev_state_event_id in ev.prev_state_events: + next_ids.add(prev_state_event_id) + + assert len(result) > 0 # we always return the forward extremities + # Assert that the create event was returned. Pick the first event (any will do) to verify + # that this room version supports room IDs as hashes. + first_event: MSC4242Event = next(iter(result.values())) + if first_event.room_version.msc4291_room_ids_as_hashes: + create_event_id = f"${room_id[1:]}" + assert create_event_id in result + + return result + + # FIXME(2026-04-22): Remove comment when used. Unused currently, but will be used in + # future MSC4242 PRs. + async def get_missing_events_state_dag( + self, + *, + room_id: str, + earliest_event_ids: list[str], + latest_event_ids: list[str], + limit: int, + ) -> list[EventBase]: + """Get parts of the state DAG in response to a /get_missing_events query. + + Args: + room_id: The state DAG to look at + earliest_event_ids: Which events the caller has seen. These events will not be returned. + latest_event_ids: Which events to start walking back from via prev_state_events. + limit: The max number of events to return. + Returns: + A list of events, deterministically ordered according to MSC4242. + """ + ids = await self.db_pool.runInteraction( + "get_missing_events_state_dag", + self._get_missing_events_state_dag_txn, + room_id, + earliest_event_ids, + latest_event_ids, + limit, + ) + return await self.get_events_as_list(ids) + + def _get_missing_events_state_dag_txn( + self, + txn: LoggingTransaction, + room_id: str, + earliest_event_ids: list[str], + latest_event_ids: list[str], + limit: int, + ) -> list[str]: + """Walk the state DAG backward from `latest_event_ids`, stopping at + `limit` results or the beginning of the DAG, whichever comes first. + + Earliest events are treated as already-visited: they are not emitted, + and their predecessors are not traversed via them. + + Results are deterministic and ordered by breadth-first search (BFS), with lexicographic + tie-breaking among siblings at the same hops away. + + Executes as a single recursive CTE in both SQLite and Postgres. + """ + earliest_set = set(earliest_event_ids) + # Sort the seeds lexicographically by event ID: seeds are all 0 hops away from + # themselves, so the event ID is the only tie-breaker available for them and we need + # the walk to start from a deterministic order. + # See https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/placeholder-1/proposals/4242-state-dags.md#get_missing_events + seed_ids = sorted(set(latest_event_ids) - earliest_set) + if not seed_ids or limit <= 0: + return [] + + seed_clause, seed_args = make_in_list_sql_clause( + self.database_engine, "e.event_id", seed_ids + ) + + # With no earliest events this is a no-op clause that is TRUE for every row. + earliest_clause, earliest_args = make_in_list_sql_clause( + self.database_engine, + "e.prev_state_event_id", + earliest_event_ids, + negative=True, + ) + + query = f""" + WITH RECURSIVE walk(event_id, hops) AS ( + SELECT + e.prev_state_event_id, + 1 + FROM msc4242_state_dag_edges e + WHERE e.room_id = ? + AND {seed_clause} + -- The create event has no edges, so it is stored as a single row with a NULL + -- `prev_state_event_id`. Skipping NULLs therefore only skips that sentinel + -- row: the create event is still returned by the walk, because it appears as + -- the `prev_state_event_id` of the events that reference it. + AND e.prev_state_event_id IS NOT NULL + AND {earliest_clause} + + -- `UNION` rather than `UNION ALL`: the same (event_id, hops) pair can be + -- reached via multiple children, and de-duplicating here stops us expanding + -- the same row over and over. + UNION + + SELECT + e.prev_state_event_id, + w.hops + 1 + FROM walk w + JOIN msc4242_state_dag_edges e + ON e.room_id = ? + AND e.event_id = w.event_id + WHERE e.prev_state_event_id IS NOT NULL + AND {earliest_clause} + -- Bound the recursion by `limit`: every extra hop adds at least one event to + -- the result, so events more than `limit` hops away can never make it into a + -- `limit`-sized response. This is also what makes the query safe against a + -- cyclic edge set: a cycle cannot be created via the normal write path, but + -- if one did exist the walk would still stop after `limit` hops rather than + -- spinning forever. + AND w.hops < ? + ) + -- An event can be reached by several paths with different hop counts. MSC4242 + -- orders by distance from the seed events, which is the *shortest* such path, so + -- collapse each event to its minimum hop count before sorting. `hops` is only + -- selected because we need it as the primary sort key. + SELECT event_id, MIN(hops) AS hops + FROM walk + GROUP BY event_id + ORDER BY hops, event_id + LIMIT ? + """ + + params: list = [room_id] + params.extend(seed_args) + params.extend(earliest_args) + params.append(room_id) + params.extend(earliest_args) + params.append(limit) + params.append(limit) + + txn.execute(query, params) + return [row[0] for row in txn] + @trace @tag_args async def get_backfill_points_in_room( diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py index 59e914ca8b..55be7265bc 100644 --- a/tests/storage/test_event_federation.py +++ b/tests/storage/test_event_federation.py @@ -19,6 +19,7 @@ # import datetime +import random from typing import ( Collection, Iterable, @@ -27,6 +28,7 @@ from typing import ( TypeVar, cast, ) +from unittest import mock import attr from parameterized import parameterized @@ -40,18 +42,22 @@ from synapse.api.room_versions import ( RoomVersion, ) from synapse.events import EventBase +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag +from synapse.events.snapshot import EventContext from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.storage.database import LoggingTransaction from synapse.storage.types import Cursor from synapse.synapse_rust.events import EventInternalMetadata +from synapse.synapse_rust.room_versions import RoomVersions from synapse.types import JsonDict from synapse.util.clock import Clock from synapse.util.json import json_encoder import tests.unittest import tests.utils +from tests.test_utils.event_builders import make_test_event # The silly auth graph we use to test the auth difference algorithm, # where the top are the most recent events. @@ -1414,6 +1420,358 @@ class EventFederationWorkerStoreTestCase(tests.unittest.HomeserverTestCase): # elapsed past the backoff range so there is no events to backoff from. self.assertEqual(event_ids_with_backoff, {}) + def _create_msc4242_room(self) -> tuple[str, str]: + """Create an MSC4242 room with an additional join rules state event. + + Returns: + A tuple of the room ID and the event ID of the most recent state event. + """ + user_id = self.register_user("alice", "test") + tok = self.login("alice", "test") + room_id = self.helper.create_room_as( + room_creator=user_id, + tok=tok, + room_version=RoomVersions.MSC4242v12.identifier, + ) + resp = self.helper.send_state( + room_id, + "m.room.join_rules", + {"join_rule": "knock"}, + tok=tok, + ) + return room_id, resp["event_id"] + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_state_dag(self) -> None: + """ + Test that MSC4242 state dag rooms can return the complete state dag on request. + """ + room_id, latest = self._create_msc4242_room() + state_dag = self.get_success( + self.store.get_state_dag(room_id, {latest}), + ) + # create <- member <- pl <- join_rules <- his vis <- join_rules + self.assertEquals(len(state_dag), 6) + want_types = [ + EventTypes.Create, + EventTypes.Member, + EventTypes.PowerLevels, + EventTypes.JoinRules, + EventTypes.RoomHistoryVisibility, + EventTypes.JoinRules, + ] + got_types = [] + curr = {latest} + while len(curr) > 0: + event_id = curr.pop() + ev = state_dag[event_id] + got_types.append(ev.type) + curr.update(ev.prev_state_events) + got_types.reverse() # we walked up the graph but want_types is walking down + self.assertEqual(got_types, want_types) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_state_dag_at_create_event(self) -> None: + """ + Test that asking for the state DAG at the create event returns just the create event: + we always return the requested forward extremities, and there are no earlier events to + walk back to. + """ + room_id, _ = self._create_msc4242_room() + # Room IDs are the hash of the create event in this room version, so the create event + # ID is derivable from the room ID. + create_event_id = f"${room_id[1:]}" + state_dag = self.get_success( + self.store.get_state_dag(room_id, {create_event_id}), + ) + self.assertEqual(list(state_dag), [create_event_id]) + + +class EventFederationGetMissingEventsStateDAGTestCase( + tests.unittest.HomeserverTestCase +): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + persist_events = hs.get_datastores().persist_events + assert persist_events is not None + self.persist_events = persist_events + + # Primarily testing to make sure that we sort events + # correctly when there are multiple prev_state_events + # .- C -- D ---. + # A <- B E + # `- R -- W --` + # `-- T -` + graph: dict[str, list[str]] = { + "A": [], + "B": ["A"], + "C": ["B"], + "R": ["B"], + "D": ["C"], + "W": ["R"], + "T": ["R"], + "E": ["W", "D", "T"], + } + self.graph = graph + (self.room_id, self.graph_events) = self._persist_state_dag( + "@test_get_missing_events_state_dag:localhost", graph + ) + + def _persist_state_dag( + self, creator: str, graph: dict[str, list[str]] + ) -> tuple[str, dict[str, MSC4242Event]]: + """Build and persist a state DAG in its own room, as `build_state_dag` returns it.""" + (room_id, graph_events) = build_state_dag(creator, graph) + + def insert(txn: LoggingTransaction) -> None: + mock_context = mock.Mock(spec=EventContext) + mock_context.rejected = False + for ev in graph_events.values(): + # store the event first to satisfy fk constraints + self.persist_events._store_event_txn( + txn, + [(ev, mock_context)], + ) + self.persist_events._store_state_dag_edges( + txn, + ev, + ) + + # satisfy fk constraints + self.get_success( + self.store.store_room(room_id, creator, False, RoomVersions.MSC4242v12) + ) + self.get_success( + self.store.db_pool.runInteraction( + "_store_state_dag_edges", + insert, + ) + ) + return room_id, graph_events + + def _get_missing_events( + self, + latest: list[str], + earliest: list[str], + limit: int, + room_id: str | None = None, + graph_events: dict[str, MSC4242Event] | None = None, + ) -> list[str]: + """Run `get_missing_events_state_dag` in terms of fake event IDs, returning the fake + event IDs which came back. Queries the room built in `prepare` unless told otherwise. + """ + if room_id is None or graph_events is None: + room_id = self.room_id + graph_events = self.graph_events + fake_event_ids = {ev.event_id: fake for fake, ev in graph_events.items()} + got = self.get_success( + self.store.get_missing_events_state_dag( + room_id=room_id, + earliest_event_ids=[graph_events[fake].event_id for fake in earliest], + latest_event_ids=[graph_events[fake].event_id for fake in latest], + limit=limit, + ), + ) + return [fake_event_ids[ev.event_id] for ev in got] + + @parameterized.expand( + [ + (["E"], ["D", "T", "W"], 3), + (["E"], ["D", "T"], 2), + (["E"], ["D"], 1), + (["W", "T", "D"], ["C", "R"], 2), + # breadth first and new entries are added to the end, sorted lexicographically + (["E"], ["D", "T", "W", "C", "R", "B", "A"], 100), + # we should sort the latest values initially + (["E", "C"], ["B", "D", "T", "W"], 4), + (["C", "E"], ["B", "D", "T", "W"], 4), + # dupes are ignored + (["E", "E", "C", "C", "C"], ["B", "D", "T", "W"], 4), + # include latest events in response. W included because reachable from E. + # sort order is based on # hops not processing order of parents + # (which would produce D,T,W,R as E is processed first, then W). + (["W", "E"], ["D", "R", "T", "W"], 4), + ] + ) + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag( + self, latest: list[str], want: list[str], limit: int + ) -> None: + # .- C -- D ---. + # A <- B E + # `- R -- W --` + # `-- T -` + self.assertEquals( + self._get_missing_events(latest=latest, earliest=[], limit=limit), + want, + f"latest={latest} want={want} limit={limit}", + ) + # These expectations are written by hand, so use them to check `walk_state_dag`, which + # the randomly generated graphs below are compared against. + self.assertEquals( + walk_state_dag(self.graph, self.graph_events, latest, [], limit), + want, + f"latest={latest} want={want} limit={limit}", + ) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag_limit_truncates(self) -> None: + """Lowering `limit` must truncate the result rather than change it. + + `limit` also bounds how many hops the walk takes, so this checks the two uses agree: + as every extra hop contributes at least one event, a shallower walk cannot miss an + event that a `limit`-sized response should have contained. + """ + full = self._get_missing_events( + latest=["E"], earliest=[], limit=len(self.graph_events) + ) + self.assertEquals(full, ["D", "T", "W", "C", "R", "B", "A"]) + for limit in range(1, len(full) + 1): + self.assertEquals( + self._get_missing_events(latest=["E"], earliest=[], limit=limit), + full[:limit], + f"limit={limit}", + ) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag_returns_nothing(self) -> None: + """The cases where there is nothing to walk back to.""" + # Nothing was asked for. + self.assertEquals( + self._get_missing_events(latest=["E"], earliest=[], limit=0), [] + ) + # Every event in the room has already been seen. + self.assertEquals( + self._get_missing_events( + latest=["E"], earliest=list(self.graph), limit=100 + ), + [], + ) + # Every event walked back from has already been seen. + self.assertEquals( + self._get_missing_events(latest=["E", "C"], earliest=["C", "E"], limit=100), + [], + ) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag_random_graphs(self) -> None: + """Check the query against a direct implementation of the MSC4242 ordering. + + Uses randomly generated DAGs, as the hand-written cases above can only cover the + shapes we thought to write down. The seed is fixed so that a failure is reproducible + and cannot appear on an unrelated change. + """ + rand = random.Random(42) + # Single letters keep `build_state_dag`'s hash mining cheap. At least 4 events, as + # smaller graphs have too little to order for the walk to be interesting. + names = "ABCDEFGH" + for room_number in range(20): + # Each event picks its prev_state_events from the events before it, which keeps + # the graph acyclic and in causal order. "A" is the create event. + graph: dict[str, list[str]] = {names[0]: []} + for index in range(1, rand.randint(4, len(names))): + graph[names[index]] = sorted( + rand.sample(names[:index], rand.randint(1, index)) + ) + built = list(graph) + (room_id, graph_events) = self._persist_state_dag( + f"@test_random_state_dag_{room_number}:localhost", graph + ) + + for _ in range(3): + # `latest` is sampled with replacement, as callers can repeat an event ID. + latest = [ + rand.choice(built) for _ in range(rand.randint(1, len(built))) + ] + earliest = rand.sample(built, rand.randint(0, len(built) // 2)) + limit = rand.randint(0, len(built) + 1) + message = ( + f"graph={graph} latest={latest} earliest={earliest} limit={limit}" + ) + self.assertEquals( + self._get_missing_events( + latest=latest, + earliest=earliest, + limit=limit, + room_id=room_id, + graph_events=graph_events, + ), + walk_state_dag(graph, graph_events, latest, earliest, limit), + message, + ) + + # Lowering `limit` must truncate the result rather than change it. Take the + # limit from the events actually available rather than from the random one, + # which is often high enough that nothing is dropped. + full = self._get_missing_events( + latest=latest, + earliest=earliest, + limit=len(built), + room_id=room_id, + graph_events=graph_events, + ) + if len(full) > 1: + self.assertEquals( + self._get_missing_events( + latest=latest, + earliest=earliest, + limit=len(full) - 1, + room_id=room_id, + graph_events=graph_events, + ), + full[:-1], + message, + ) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag_with_cycle(self) -> None: + """The walk must terminate even if the edges table contains a cycle. + + A cycle cannot be produced by the normal write path, as an event's prev_state_events + are fixed at creation and its event ID is a hash of them. Insert one directly to check + that the query is bounded regardless. + """ + # Claim that A (the create event) has E as a prev_state_event, so walking back from E + # eventually reaches A and then E again. + self.get_success( + self.store.db_pool.simple_insert( + table="msc4242_state_dag_edges", + values={ + "room_id": self.room_id, + "event_id": self.graph_events["A"].event_id, + "prev_state_event_id": self.graph_events["E"].event_id, + }, + desc="insert_state_dag_cycle", + ) + ) + + # Same as the acyclic walk from E, with E itself now reachable via A at the end. Each + # event appears exactly once: the CTE groups by event ID and keeps the fewest hops. + self.assertEquals( + self._get_missing_events(latest=["E"], earliest=[], limit=100), + ["D", "T", "W", "C", "R", "B", "A", "E"], + ) + @attr.s(auto_attribs=True) class FakeEvent: @@ -1431,3 +1789,126 @@ class FakeEvent: def is_state(self) -> bool: return True + + +def walk_state_dag( + graph: dict[str, list[str]], + graph_events: dict[str, MSC4242Event], + latest: list[str], + earliest: list[str], + limit: int, +) -> list[str]: + """Work out what `get_missing_events_state_dag` should return, from the MSC4242 rules. + + Walks back from `latest` via prev_state_events, then orders the events found by how many + hops they are from `latest`, breaking ties on the real event ID. An `earliest` event is + never returned and is never walked through, though its predecessors are still returned if + some other path reaches them. + + Takes the graph and events as `build_state_dag` returns them, in terms of fake event IDs, + and returns the fake event IDs which should come back, in order. Ties break on the real + event ID rather than the fake one because the create event is the one event whose real ID + does not start with its fake ID. + """ + hops: dict[str, int] = {} + # An event `limit` hops away can only be reached by returning an event at every hop + # before it, so a walk deeper than `limit` cannot contribute to the response. + frontier = sorted(set(latest) - set(earliest)) + for hop in range(1, limit + 1): + next_frontier = set() + for fake_event_id in frontier: + for prev_fake_event_id in graph[fake_event_id]: + if prev_fake_event_id in earliest or prev_fake_event_id in hops: + continue + # The first time we see an event is by definition its fewest hops. + hops[prev_fake_event_id] = hop + next_frontier.add(prev_fake_event_id) + frontier = sorted(next_frontier) + return sorted( + hops, + key=lambda fake: (hops[fake], graph_events[fake].event_id), + )[:limit] + + +def build_state_dag( + creator: str, graph: dict[str, list[str]] +) -> tuple[str, dict[str, MSC4242Event]]: + """Build an MSC4242 state DAG. + + Args: + creator: The user ID creating the graph. Should be unique per-test to ensure room IDs change between tests. + graph: A map of fake event ID e.g. "B" to a list of prev_state_events e.g. ["A"]. Graphs must + be created in causal order (earliest events first). The first entry is assumed to be the + create event, so it must have no prev_state_events; every later entry must have at least one. + Returns: + A tuple of the room ID and a map from fake event ID e.g. "B" to real event which you can use .event_id + to extract the real event ID. Guarantees that the real event IDs start with the fake event ID e.g. + the real event for "B" is guarantees to start "$B...." which makes sorting tests much easier to reason about. + The create event is the exception: its ID is fixed by the room ID, so it does not start with its + fake event ID. + """ + graph_events: dict[str, MSC4242Event] = {} # graph ID => built event + create_event = make_test_event( + { + "type": EventTypes.Create, + "state_key": "", + "content": { + "room_version": RoomVersions.MSC4242v12.identifier, + }, + "sender": creator, + "origin_server_ts": 1, + }, + room_version=RoomVersions.MSC4242v12, + ) + # Narrow to an MSC4242 event, so that `prev_state_events` can be used. + assert supports_msc4242_state_dag(create_event) + room_id = create_event.room_id + entropy = 1 + for index, graph_event_id in enumerate(graph): + if index == 0: + # The first entry is the create event, which roots the state DAG. + assert len(graph[graph_event_id]) == 0, ( + f"the first graph event {graph_event_id} is the create event, so it cannot " + "have any prev_state_events" + ) + graph_events[graph_event_id] = create_event + continue + assert len(graph[graph_event_id]) > 0, ( + f"graph event {graph_event_id} has no prev_state_events, but only the create event " + "(the first entry) may be missing them" + ) + # Map previous event IDs to real event IDs. Requires us to build events in causal order. + prev_state_event_ids = [ + graph_events[prev_graph_event_id].event_id + for prev_graph_event_id in graph[graph_event_id] + ] + # Event IDs are hashes of the event, so we cannot choose them directly. Instead, keep + # rebuilding the event with a different `origin_server_ts` until the hash happens to + # start with the fake event ID (e.g. "B" produces "$B..."). Ordering in the state DAG + # breaks ties lexicographically on the real event ID, so having the real IDs sort in + # the same order as the fake names is what makes the expectations in these tests + # readable. + while graph_event_id not in graph_events: + graph_event = make_test_event( + { + "type": "foo", + # All the fake events share a `type`, so the `state_key` is what makes + # each one a distinct piece of state. + "state_key": graph_event_id, + "content": {}, + "sender": creator, + "origin_server_ts": 1 + entropy, + "prev_state_events": prev_state_event_ids, + # Nothing in these tests looks at the message DAG, so just mirror the + # state DAG here. + "prev_events": prev_state_event_ids, + "room_id": room_id, + }, + room_version=RoomVersions.MSC4242v12, + ) + assert supports_msc4242_state_dag(graph_event) + if not graph_event.event_id[1:].startswith(graph_event_id): + entropy += 1 + continue + graph_events[graph_event_id] = graph_event + return (room_id, graph_events) From b6798a13531d20eed415e5055197eb3acbac2264 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Wed, 2 Sep 2026 15:27:38 +0200 Subject: [PATCH 139/158] Fix `event_search` background reindex skipping `m.room.topic` events (#20119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Room topics are dropped from the search index whenever the `event_search` background reindex runs (e.g. after a search index rebuild, or when the background update is re-run on an upgraded homeserver), making topics unsearchable even though the live write path indexes them correctly. The cause is a trailing comma in `_background_reindex_search`, which turns the topic `value` into a 1-tuple instead of a string: https://github.com/element-hq/synapse/blob/14c96c0f5444cbe28b6ac0361cf94216b8d352db/synapse/storage/databases/main/search.py#L211-L213 The downstream `if not isinstance(value, str): continue` guard then silently skips *every* `m.room.topic` event, so no topic ever reaches `event_search` during a reindex. The regression was introduced in #18195, which added rich-text topic support (MSC3765) to the reindex path. ### Problem Example 1. A room has topic "project roadmap". 2. An admin rebuilds the search index (or the `event_search` background update re-runs). 3. `m.room.message` and `m.room.name` events are reindexed fine, but every `m.room.topic` event is skipped. 4. Searching for "project roadmap" with key `content.topic` returns 0 results — the topic is permanently unsearchable until the event is sent again. --- ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20119.bugfix | 1 + synapse/storage/databases/main/search.py | 4 +- tests/storage/test_room_search.py | 81 ++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20119.bugfix diff --git a/changelog.d/20119.bugfix b/changelog.d/20119.bugfix new file mode 100644 index 0000000000..a09602941a --- /dev/null +++ b/changelog.d/20119.bugfix @@ -0,0 +1 @@ +Fix a bug where the `event_search` background reindex skipped all `m.room.topic` events, making room topics unsearchable after a search index rebuild. diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index d6eace5efa..b350978d34 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -208,9 +208,7 @@ class SearchBackgroundUpdateStore(SearchWorkerStore): value = content["body"] elif etype == "m.room.topic": key = "content.topic" - value = ( - get_plain_text_topic_from_event_content(content) or "", - ) + value = get_plain_text_topic_from_event_content(content) or "" elif etype == "m.room.name": key = "content.name" value = content["name"] diff --git a/tests/storage/test_room_search.py b/tests/storage/test_room_search.py index 2c0ef19e9e..ccfe5b012d 100644 --- a/tests/storage/test_room_search.py +++ b/tests/storage/test_room_search.py @@ -19,6 +19,7 @@ # # +import json from unittest.case import SkipTest from twisted.internet.testing import MemoryReactor @@ -202,6 +203,86 @@ class EventSearchInsertionTest(HomeserverTestCase): ) self.assertCountEqual(values, ["hi", "2"]) + def _rebuild_search_index(self) -> None: + """Simulate a full search-index rebuild: wipe `event_search` so nothing + is indexed, then schedule and run the `event_search` background reindex, + which exercises `_background_reindex_search`. + """ + store = self.hs.get_datastores().main + + self.get_success( + store.db_pool.runInteraction( + "clear_event_search", + lambda txn: txn.execute("DELETE FROM event_search"), + ) + ) + + max_stream_id = store.get_room_max_stream_ordering() + store.db_pool.updates._all_done = False + self.get_success( + store.db_pool.simple_insert( + "background_updates", + { + "update_name": store.EVENT_SEARCH_UPDATE_NAME, + "progress_json": json.dumps( + { + "target_min_stream_id_inclusive": 0, + "max_stream_id_exclusive": max_stream_id + 1, + "rows_inserted": 0, + } + ), + }, + ) + ) + self.wait_for_background_updates() + + def test_reindex_search(self) -> None: + """The `event_search` background reindex must index all searchable event + types: `m.room.message`, `m.room.name` and `m.room.topic`. + """ + store = self.hs.get_datastores().main + + # Create a room and set a searchable topic, name and message through the + # normal client API so the events land in `events`/`event_json`. + self.register_user("alice", "password") + access_token = self.login("alice", "password") + room_id = self.helper.create_room_as("alice", tok=access_token) + + self.helper.send_state( + room_id, + "m.room.topic", + {"topic": "searchable topic keyword"}, + tok=access_token, + ) + self.helper.send_state( + room_id, + "m.room.name", + {"name": "searchable name keyword"}, + tok=access_token, + ) + self.helper.send(room_id, "searchable message keyword", tok=access_token) + + self._rebuild_search_index() + + message_results = self.get_success( + store.search_msgs([room_id], "searchable message keyword", ["content.body"]) + ) + self.assertEqual(message_results["count"], 1, "message was not reindexed") + + name_results = self.get_success( + store.search_msgs([room_id], "searchable name keyword", ["content.name"]) + ) + self.assertEqual(name_results["count"], 1, "name was not reindexed") + + topic_results = self.get_success( + store.search_msgs([room_id], "searchable topic keyword", ["content.topic"]) + ) + self.assertEqual( + topic_results["count"], + 1, + "room topic was not indexed by the search reindex", + ) + class MessageSearchTest(HomeserverTestCase): """ From f94abb6924178abbf59150608d74ac16c79ad33c Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Wed, 2 Sep 2026 15:29:08 +0200 Subject: [PATCH 140/158] Return 403 instead of 400 when profile changes are disabled (#20173) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With `enable_set_displayname: false` (or `enable_set_avatar_url: false`), refusing a profile change returned the right errcode with the wrong status: ``` PUT /_matrix/client/v3/profile/@alice:example.com/displayname (displayname already set) → 400 {"errcode": "M_FORBIDDEN", "error": "Changing display name is disabled on this server"} DELETE /_matrix/client/v3/profile/@alice:example.com/displayname → 400 {"errcode": "M_FORBIDDEN", "error": "Changing display name is disabled on this server"} ``` With this fix: ``` PUT /_matrix/client/v3/profile/@alice:example.com/displayname (displayname already set) → 403 {"errcode": "M_FORBIDDEN", "error": "Changing display name is disabled on this server"} DELETE /_matrix/client/v3/profile/@alice:example.com/displayname → 403 {"errcode": "M_FORBIDDEN", "error": "Changing display name is disabled on this server"} ``` The spec defines the [403 response of `PUT /_matrix/client/v3/profile/{userId}/{keyName}`](https://spec.matrix.org/v1.19/client-server-api/#put_matrixclientv3profileuseridkeyname) as "The server is unwilling to perform the operation, either due to insufficient permissions or **because profile modifications are disabled**", while 400 is reserved for malformed input (`M_BAD_JSON`, `M_MISSING_PARAM`, …). Clients seem to rely on `errcode` field more than the HTTP Status Code, that change seems safe. --- changelog.d/20173.bugfix | 1 + synapse/handlers/profile.py | 10 +++- tests/handlers/test_profile.py | 10 ++-- tests/rest/client/test_profile.py | 78 +++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20173.bugfix diff --git a/changelog.d/20173.bugfix b/changelog.d/20173.bugfix new file mode 100644 index 0000000000..3cb83fcb44 --- /dev/null +++ b/changelog.d/20173.bugfix @@ -0,0 +1 @@ +Fix `PUT`/`DELETE` on a profile field returning HTTP 400 instead of 403 (with errcode `M_FORBIDDEN`) when profile changes are disabled via `enable_set_displayname` or `enable_set_avatar_url`. diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 067a16262c..c4f17eb0fb 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -247,8 +247,11 @@ class ProfileHandler: if not by_admin and not self.hs.config.registration.enable_set_displayname: profile = await self.store.get_profileinfo(target_user) if profile.display_name: + # The spec reserves 400 for malformed input; disabled profile + # modifications are covered by the 403 response of + # https://spec.matrix.org/v1.19/client-server-api/#put_matrixclientv3profileuseridkeyname raise SynapseError( - 400, + 403, "Changing display name is disabled on this server", Codes.FORBIDDEN, ) @@ -369,8 +372,11 @@ class ProfileHandler: if not by_admin and not self.hs.config.registration.enable_set_avatar_url: profile = await self.store.get_profileinfo(target_user) if profile.avatar_url: + # The spec reserves 400 for malformed input; disabled profile + # modifications are covered by the 403 response of + # https://spec.matrix.org/v1.19/client-server-api/#put_matrixclientv3profileuseridkeyname raise SynapseError( - 400, "Changing avatar is disabled on this server", Codes.FORBIDDEN + 403, "Changing avatar is disabled on this server", Codes.FORBIDDEN ) if not isinstance(new_avatar_url, str): diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 31e7e1c0a7..7af0345c87 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -31,7 +31,7 @@ from synapse.api.constants import ( ProfileFields, ProfileUpdateAction, ) -from synapse.api.errors import AuthError, SynapseError +from synapse.api.errors import AuthError, Codes, SynapseError from synapse.rest import admin from synapse.rest.client import knock, login, room from synapse.server import HomeServer @@ -907,7 +907,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) # Setting displayname a second time is forbidden - self.get_failure( + f = self.get_failure( self.handler.set_field( target_user=self.frank, requester=synapse.types.create_requester(self.frank), @@ -916,6 +916,8 @@ class ProfileTestCase(unittest.HomeserverTestCase): ), SynapseError, ) + self.assertEqual(f.value.code, 403) + self.assertEqual(f.value.errcode, Codes.FORBIDDEN) def test_set_my_name_noauth(self) -> None: self.get_failure( @@ -1061,7 +1063,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) # Set avatar a second time is forbidden - self.get_failure( + f = self.get_failure( self.handler.set_field( target_user=self.frank, requester=synapse.types.create_requester(self.frank), @@ -1070,6 +1072,8 @@ class ProfileTestCase(unittest.HomeserverTestCase): ), SynapseError, ) + self.assertEqual(f.value.code, 403) + self.assertEqual(f.value.errcode, Codes.FORBIDDEN) def test_avatar_constraints_no_config(self) -> None: """Tests that the method to check an avatar against configured constraints skips diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 023a376ed1..95b11ff2b2 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -127,6 +127,37 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) self.assertEqual(channel.code, 400, channel.result) + @unittest.override_config({"enable_set_displayname": False}) + def test_set_displayname_disabled(self) -> None: + """Changing an existing displayname while `enable_set_displayname` is off + should get a 403 with M_FORBIDDEN.""" + channel = self.make_request( + "PUT", + "/profile/%s/displayname" % (self.owner,), + content={"displayname": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + res = self._get_displayname() + self.assertEqual(res, "owner") + + @unittest.override_config({"enable_set_displayname": False}) + def test_delete_displayname_disabled(self) -> None: + """Deleting an existing displayname while `enable_set_displayname` is off + should get a 403 with M_FORBIDDEN.""" + channel = self.make_request( + "DELETE", + "/profile/%s/displayname" % (self.owner,), + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + res = self._get_displayname() + self.assertEqual(res, "owner") + def test_get_avatar_url(self) -> None: res = self._get_avatar_url() self.assertIsNone(res) @@ -177,6 +208,53 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) self.assertEqual(channel.code, 400, channel.result) + @unittest.override_config({"enable_set_avatar_url": False}) + def test_set_avatar_url_disabled(self) -> None: + """Changing an existing avatar while `enable_set_avatar_url` is off should + get a 403 with M_FORBIDDEN. Setting it for the first time is allowed.""" + channel = self.make_request( + "PUT", + "/profile/%s/avatar_url" % (self.owner,), + content={"avatar_url": "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + "/profile/%s/avatar_url" % (self.owner,), + content={"avatar_url": "http://my.server/me.png"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + res = self._get_avatar_url() + self.assertEqual(res, "http://my.server/pic.gif") + + @unittest.override_config({"enable_set_avatar_url": False}) + def test_delete_avatar_url_disabled(self) -> None: + """Deleting an existing avatar while `enable_set_avatar_url` is off should + get a 403 with M_FORBIDDEN.""" + channel = self.make_request( + "PUT", + "/profile/%s/avatar_url" % (self.owner,), + content={"avatar_url": "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "DELETE", + "/profile/%s/avatar_url" % (self.owner,), + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + res = self._get_avatar_url() + self.assertEqual(res, "http://my.server/pic.gif") + def _get_displayname(self, name: str | None = None) -> str | None: channel = self.make_request( "GET", "/profile/%s/displayname" % (name or self.owner,) From c42794d9c595ef37b3c43e05fb86b63ed86de1b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:40:04 +0100 Subject: [PATCH 141/158] Bump thrift from 0.22.0 to 0.24.0 (#20177) --- poetry.lock | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index ccf78bc962..508379acd5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3142,20 +3142,55 @@ tornado = "*" [[package]] name = "thrift" -version = "0.22.0" +version = "0.24.0" description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] markers = "extra == \"all\" or extra == \"opentracing\"" files = [ - {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, + {file = "thrift-0.24.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efe85c4508adaf6c9e6f7fac2b1c3c9beb4b39b19c375519966335a70b28e64a"}, + {file = "thrift-0.24.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a2baaec0c5cd7ba3eace54b26d81fdf0f5a85010468687cf4a131871d65abfed"}, + {file = "thrift-0.24.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cad27a826e86739a79a327e6afae5a9bea36aeed2989c4318fc2943b6cfad095"}, + {file = "thrift-0.24.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e26ca0f346e5ec2b6be9778573fb2e9d8b3eb162dbce94e61906176158b448b"}, + {file = "thrift-0.24.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:498ae392090d7ae17ab59ebd8dfabda76528eeb22d3890a57c9f226003d7ed6e"}, + {file = "thrift-0.24.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:acbd02baacfa0d2017f85b189ed69cd6baa522785d1ad86476eb7ce8cd16d063"}, + {file = "thrift-0.24.0-cp310-cp310-win_amd64.whl", hash = "sha256:887a10d718d85275da70fe4e9d4740268ae2854f205fb5869909d24cc5576b79"}, + {file = "thrift-0.24.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:873c93d8496cf71589efd3128ddc5af0be350e6c1a0e615475d840eaa54f6124"}, + {file = "thrift-0.24.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:048d768e5822c436e8920b987d89a6a26e4d438ddf2de4b9d3f81444cd03cd04"}, + {file = "thrift-0.24.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2592bb0bf232d0808583626a7a8d71e265851e974c182967d67dde1f16902aee"}, + {file = "thrift-0.24.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ca1b1ba00151565dc4e6673c924debd557aaf25b2790f1570b3430cb35503671"}, + {file = "thrift-0.24.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:afb7977ccb8ecd7b1520a5141de64b58e94712528a26c10ac8f5b4ef220112a5"}, + {file = "thrift-0.24.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c8d37a4311a87bda4bd2b8e4137eec638c18e3a57d11a1011bd98a8867523860"}, + {file = "thrift-0.24.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbf461351940ddaa85bf8c2ee1754c9cfdd33bb78322e635e1ea5cd3947ae49a"}, + {file = "thrift-0.24.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:306e0fe3c96b300471c19cec60bd6816064fd93d04713488ef07120aff84653b"}, + {file = "thrift-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ac42199ea2a6fc6275b0aeec32ae09e7f9edb43890ff9a1af5e34191fb422cc"}, + {file = "thrift-0.24.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cdd5f927eb98d0e8f00ab148b0cb66ae2598a396e907bd2b6febcbdcc46d80c"}, + {file = "thrift-0.24.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:134cd1a0d80f928377808348d1f9f647284ada093573beaa47040bed5507e219"}, + {file = "thrift-0.24.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:518199062feabfc0982767a0f7bceccb4fd1b485654f75e02913837444c3c130"}, + {file = "thrift-0.24.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8c5de86af2a44641d423624c71c554a691d9f80c8b87709f15c7f98ccc6aeac"}, + {file = "thrift-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:2f82bb16c4f2009dbc4fc9374604f05e998fb33be6a7787e095cea9842ecfa1d"}, + {file = "thrift-0.24.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebb8b389142d67554d0de814dd6c1d62b962751f68721537efca429c57b09327"}, + {file = "thrift-0.24.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18e7693f1bcef45937ad31a02564add55dec754710d42afc8ee3fc26ecf13028"}, + {file = "thrift-0.24.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937b398c311799fd5eddaeaffbd275510c68ead597eb1897e5e8113542fb2b50"}, + {file = "thrift-0.24.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b89a83d4e9ae6029e14f6f7c73305044bf02739d729c6aa8025ef3b6faef0ec0"}, + {file = "thrift-0.24.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6381093f2c54e0f108554004a8cac3f1ef7ba99d6c5a9909c0eb64f450142685"}, + {file = "thrift-0.24.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:28e034658d724aaa66007babb258ef12f0294036812fc449d7ce6fd15b07100f"}, + {file = "thrift-0.24.0-cp313-cp313-win_amd64.whl", hash = "sha256:f4a44391ae1e32817553639b2991b0753d84487259fe87f8111c956c6bdebf43"}, + {file = "thrift-0.24.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7beb05268c76895f7c3130ce747a8a8cf35558fbc7f464f994e20a0c92181cbd"}, + {file = "thrift-0.24.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f40bdfbaf8d2795b1593bb75b4d9c77831bbe96a1ef59da4634ccc125a17dca5"}, + {file = "thrift-0.24.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:272624d36faa7bee01b94e5da5efc7305d9d3da57d005382c93fcbde8c3edf6a"}, + {file = "thrift-0.24.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bc91ae93005bc16c7362edcb58818a547850b9b52fe9b8075cc6b8922bcde2d"}, + {file = "thrift-0.24.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c05bb4eac921836c12cdf30c237283df8a42aff6540f7970cd72c5959b139970"}, + {file = "thrift-0.24.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e5da6b391828d46df9471f22181374760a91773b7e07ad7eb44f351c90580662"}, + {file = "thrift-0.24.0-cp314-cp314-win_amd64.whl", hash = "sha256:829db909a053d4064cb9fe574cc1f5994c24d727f61cdf6446ca774cd3ba5268"}, + {file = "thrift-0.24.0.tar.gz", hash = "sha256:9ef601c49e988475ff0e741d8e1b45feec23b48514e524341efc274191f1789c"}, ] [package.extras] -all = ["tornado (>=4.0)", "twisted"] -tornado = ["tornado (>=4.0)"] -twisted = ["twisted"] +all = ["tornado (>=6.3.0)", "twisted (>=24.3.0)", "zope.interface (>=6.1)"] +tornado = ["tornado (>=6.3.0)"] +twisted = ["twisted (>=24.3.0)", "zope.interface (>=6.1)"] [[package]] name = "tomli" From c40ab6e3b9d6dc1cf677e3bf3b0320aa0c6ae54e Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Wed, 2 Sep 2026 16:52:41 +0200 Subject: [PATCH 142/158] Fix sending duplicate `m.room.encryption` events on room creation (#20106) On room creation when the client already supplies one in the initial state and `encryption_enabled_by_default_for_room_type` is enabled. --- changelog.d/20106.bugfix | 1 + .../configuration/config_documentation.md | 2 + schema/synapse-config.schema.yaml | 9 ++ synapse/handlers/room.py | 16 ++- tests/handlers/test_room.py | 109 ++++++++++++++++++ 5 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20106.bugfix diff --git a/changelog.d/20106.bugfix b/changelog.d/20106.bugfix new file mode 100644 index 0000000000..bd88506868 --- /dev/null +++ b/changelog.d/20106.bugfix @@ -0,0 +1 @@ +Do not send a duplicate `m.room.encryption` event on room creation when the client already supplies one in the initial state and `encryption_enabled_by_default_for_room_type` is enabled. Contributed by @FrenchGithubUser @Famedly. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index f725ee1a67..b2cd674130 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4010,6 +4010,8 @@ Possible options are "all", "invite", and "off". They are defined as: Note that this option will only affect rooms created after it is set. It will also not affect rooms created by other servers. +A client may supply its own `m.room.encryption` event in the `initial_state` of its `/createRoom` request. If that event is valid (it specifies an `algorithm` as a string), it takes precedence and this option will not overwrite it, allowing the client to, for example, choose a different encryption algorithm. An empty or otherwise invalid `m.room.encryption` event does not disable forced encryption: the default will still be applied on top of it. + Defaults to `"off"`. Example configuration: diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 9169677cd5..c229c74346 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -5005,6 +5005,15 @@ properties: Note that this option will only affect rooms created after it is set. It will also not affect rooms created by other servers. + + + A client may supply its own `m.room.encryption` event in the + `initial_state` of its `/createRoom` request. If that event is valid (it + specifies an `algorithm` as a string), it takes precedence and this option + will not overwrite it, allowing the client to, for example, choose a + different encryption algorithm. An empty or otherwise invalid + `m.room.encryption` event does not disable forced encryption: the default + will still be applied on top of it. enum: - all - invite diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0837325e89..26ebd42574 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1786,7 +1786,21 @@ class RoomCreationHandler: ) events_to_send.append((event, context)) - if preset_config["encrypted"] and not ignore_forced_encryption: + # If the client supplied its own `m.room.encryption` event in the + # initial state, only let it take precedence over the forced default + # if it is valid (i.e. specifies an `algorithm` as a string, as + # required by the spec). This prevents a client from bypassing forced + # encryption entirely by supplying an empty or malformed event. + supplied_encryption = initial_state.get((EventTypes.RoomEncryption, "")) + supplied_encryption_is_valid = isinstance(supplied_encryption, dict) and ( + isinstance(supplied_encryption.get("algorithm"), str) + ) + + if ( + preset_config["encrypted"] + and not ignore_forced_encryption + and not supplied_encryption_is_valid + ): encryption_event, encryption_context = await create_event( EventTypes.RoomEncryption, {"algorithm": RoomEncryptionAlgorithms.DEFAULT}, diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index e1d21441d6..2816ae9424 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -110,6 +110,115 @@ class EncryptedByDefaultTestCase(unittest.HomeserverTestCase): expect_code=404, ) + @override_config({"encryption_enabled_by_default_for_room_type": "all"}) + def test_user_supplied_encryption_event_is_not_overwritten(self) -> None: + """Tests that an m.room.encryption event supplied by the user in the + initial state takes precedence over the one that would otherwise be forced + by encryption_enabled_by_default_for_room_type, rather than a duplicate + default event being sent. + """ + # Create a user + user = self.register_user("user", "pass") + user_token = self.login(user, "pass") + + # Create a room, supplying our own encryption event with a non-default + # algorithm in the initial state. + custom_content = {"algorithm": "some.custom.invalid.algorithm"} + room_id = self.helper.create_room_as( + user, + is_public=False, + tok=user_token, + extra_content={ + "initial_state": [ + { + "type": EventTypes.RoomEncryption, + "state_key": "", + "content": custom_content, + } + ] + }, + ) + + # Check that the room's encryption event is the one we supplied, not the + # default that the config option would otherwise force. + event_content = self.helper.get_state( + room_id=room_id, + event_type=EventTypes.RoomEncryption, + tok=user_token, + ) + self.assertEqual(event_content, custom_content) + + @override_config({"encryption_enabled_by_default_for_room_type": "all"}) + def test_empty_encryption_event_does_not_bypass_forced_encryption(self) -> None: + """Tests that a user cannot bypass encryption_enabled_by_default_for_room_type + by supplying an empty m.room.encryption event in the initial state. Since + such an event is not valid (it lacks the required `algorithm` key), the + forced default must still be applied. + """ + # Create a user + user = self.register_user("user", "pass") + user_token = self.login(user, "pass") + + # Create a room, supplying an empty encryption event in the initial state. + room_id = self.helper.create_room_as( + user, + is_public=False, + tok=user_token, + extra_content={ + "initial_state": [ + { + "type": EventTypes.RoomEncryption, + "state_key": "", + "content": {}, + } + ] + }, + ) + + # Check that the forced default encryption was still applied on top of the + # empty event, rather than the bypass succeeding. + event_content = self.helper.get_state( + room_id=room_id, + event_type=EventTypes.RoomEncryption, + tok=user_token, + ) + self.assertEqual(event_content, {"algorithm": RoomEncryptionAlgorithms.DEFAULT}) + + @override_config({"encryption_enabled_by_default_for_room_type": "all"}) + def test_non_string_algorithm_does_not_bypass_forced_encryption(self) -> None: + """Tests that a user cannot bypass encryption_enabled_by_default_for_room_type + by supplying an m.room.encryption event whose `algorithm` is not a string. + The forced default must still be applied. + """ + # Create a user + user = self.register_user("user", "pass") + user_token = self.login(user, "pass") + + # Create a room, supplying an encryption event with a malformed + # (non-string) algorithm in the initial state. + room_id = self.helper.create_room_as( + user, + is_public=False, + tok=user_token, + extra_content={ + "initial_state": [ + { + "type": EventTypes.RoomEncryption, + "state_key": "", + "content": {"algorithm": 42}, + } + ] + }, + ) + + # Check that the forced default encryption was still applied. + event_content = self.helper.get_state( + room_id=room_id, + event_type=EventTypes.RoomEncryption, + tok=user_token, + ) + self.assertEqual(event_content, {"algorithm": RoomEncryptionAlgorithms.DEFAULT}) + class RoomIDCollisionTestCase(unittest.HomeserverTestCase): servlets = [ From dea059a94dd7294c16b72cac929b8acc87971dc1 Mon Sep 17 00:00:00 2001 From: adis veletanlic <96535657+defaultdino@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:36:35 +0200 Subject: [PATCH 143/158] Restrict message events from being redacted by local users if past allowed period. (#20138) Closes: #20065 --------- Co-authored-by: defaultdino Co-authored-by: Olivier 'reivilibre --- changelog.d/20138.feature | 1 + .../configuration/config_documentation.md | 15 ++++ schema/synapse-config.schema.yaml | 22 ++++++ synapse/config/server.py | 9 +++ synapse/handlers/message.py | 45 +++++++++++ tests/rest/client/test_redactions.py | 75 +++++++++++++++++++ 6 files changed, 167 insertions(+) create mode 100644 changelog.d/20138.feature diff --git a/changelog.d/20138.feature b/changelog.d/20138.feature new file mode 100644 index 0000000000..654a1e67c3 --- /dev/null +++ b/changelog.d/20138.feature @@ -0,0 +1 @@ +Add a config option that limits the time period in which local users can redact their own messages. Contributed by @defaultdino. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index b2cd674130..4882569b7c 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1073,6 +1073,21 @@ Example configuration: redaction_retention_period: 28d ``` --- +### `redaction_allowed_period` + +How long after an `m.room.message` was sent a local user is still allowed to redact it. If a local user tries to redact a `m.room.message` older than this period Synapse responds with `403 M_FORBIDDEN` and does not redact the event. + +Only applies to `m.room.message` events redacted by local users. Redactions of other event types and redactions received over federation are unaffected. When the target of the redaction is an edit (`m.replace`), the age and type are taken from the original event and not the edit. + +Set to `null` (the default) to disable, allowing events to be redacted at any time. + +Defaults to `null`. + +Example configuration: +```yaml +redaction_allowed_period: 7d +``` +--- ### `forgotten_room_retention_period` How long to keep locally forgotten rooms before purging them from the DB. A value of `null` means it's disabled. Defaults to `null`. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index c229c74346..68ff696780 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1278,6 +1278,28 @@ properties: default: 7d examples: - 28d + redaction_allowed_period: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + How long after an `m.room.message` was sent a local user is still allowed + to redact it. If a local user tries to redact a `m.room.message` older + than this period Synapse responds with `403 M_FORBIDDEN` + and does not redact the event. + + + Only applies to `m.room.message` events redacted by local users. + Redactions of other event types and redactions received over federation + are unaffected. When the target of the redaction is an edit (`m.replace`), + the age and type are taken from the original event and not the edit. + + + Set to `null` (the default) to disable, allowing events to be redacted at + any time. + default: null + examples: + - 7d forgotten_room_retention_period: oneOf: - $ref: "#/$defs/duration" diff --git a/synapse/config/server.py b/synapse/config/server.py index 00b414ea30..9c5a42fc52 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -659,6 +659,15 @@ class ServerConfig(Config): else: self.redaction_retention_period = None + # How long to allow event redactions for on `m.room.message` + redaction_allowed_period = config.get("redaction_allowed_period", None) + if redaction_allowed_period is not None: + self.redaction_allowed_period: int | None = self.parse_duration( + redaction_allowed_period + ) + else: + self.redaction_allowed_period = None + # How long to keep locally forgotten rooms before purging them from the DB. forgotten_room_retention_period = config.get( "forgotten_room_retention_period", None diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b34ee9d50f..c61fccb5f2 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -704,6 +704,9 @@ class EventCreationHandler: Codes.USER_ACCOUNT_SUSPENDED, ) + if event_dict["type"] == EventTypes.Redaction: + await self._check_redaction_allowed_period(event_dict) + is_create_event = ( event_dict["type"] == EventTypes.Create and event_dict["state_key"] == "" ) @@ -2236,6 +2239,48 @@ class EventCreationHandler: return bool(original_event and sender != original_event.sender) + async def _check_redaction_allowed_period(self, event_dict: dict) -> None: + """Reject a redaction of an `m.room.message` older than the configured period. + + Only applies to `m.room.message` targets. Enforced for local users only + (federated redactions bypass `create_event`). When the target is an edit + (`m.replace`), the age and type of the original event are used, not the + edit. + """ + period = self.config.server.redaction_allowed_period + if period is None: + return + + redacts = event_dict["content"].get("redacts") or event_dict.get("redacts") + room_id = event_dict["room_id"] + + if redacts is None: + return + + target = await self.store.get_event( + redacts, check_room_id=room_id, allow_none=True + ) + if target is None: + return + + relation = relation_from_event(target) + if relation is not None and relation.rel_type == RelationTypes.REPLACE: + original = await self.store.get_event( + relation.parent_id, check_room_id=room_id, allow_none=True + ) + if original is not None: + target = original + + if target.type != EventTypes.Message: + return + + if target.origin_server_ts < self.clock.time_msec() - period: + raise SynapseError( + 403, + f"Events older than {period}ms cannot be redacted.", + Codes.FORBIDDEN, + ) + async def _maybe_kick_guest_users( self, event: EventBase, context: EventContext ) -> None: diff --git a/tests/rest/client/test_redactions.py b/tests/rest/client/test_redactions.py index c8ab9577a3..a2f73d43f4 100644 --- a/tests/rest/client/test_redactions.py +++ b/tests/rest/client/test_redactions.py @@ -24,6 +24,7 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, RelationTypes +from synapse.api.errors import Codes from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.rest import admin from synapse.rest.client import login, room, sync @@ -170,6 +171,80 @@ class RedactionsTestCase(HomeserverTestCase): self.assertEqual(timeline[-3]["unsigned"]["redacted_by"], redaction_id) self.assertEqual(timeline[-3]["content"], {}) + @override_config({"redaction_allowed_period": "1h"}) + def test_can_redact_recent_event(self) -> None: + b = self.helper.send(room_id=self.room_id, tok=self.other_access_token) + self._redact_event( + self.other_access_token, self.room_id, b["event_id"], expect_code=200 + ) + + @override_config({"redaction_allowed_period": "1h"}) + def test_cannot_redact_old_event(self) -> None: + b = self.helper.send(room_id=self.room_id, tok=self.other_access_token) + msg_id = b["event_id"] + + # go past allowed period + self.reactor.advance(60 * 60 + 1) + + body = self._redact_event( + self.other_access_token, self.room_id, msg_id, expect_code=403 + ) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + @override_config({"redaction_allowed_period": "1h"}) + def test_edit_redaction_uses_original_timestamp(self) -> None: + b = self.helper.send(room_id=self.room_id, tok=self.other_access_token) + original_id = b["event_id"] + + self.reactor.advance(60 * 60 + 1) + + edit = self.helper.send_event( + self.room_id, + EventTypes.Message, + content={ + "msgtype": "m.text", + "body": "* edited", + "m.new_content": {"msgtype": "m.text", "body": "edited"}, + "m.relates_to": { + "rel_type": RelationTypes.REPLACE, + "event_id": original_id, + }, + }, + tok=self.other_access_token, + ) + + body = self._redact_event( + self.other_access_token, self.room_id, edit["event_id"], expect_code=403 + ) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + @override_config({"redaction_allowed_period": "1h"}) + def test_can_redact_old_non_message_event(self) -> None: + # the restriction only applies to m.room.message. other event types + # (a reaction in this test) can still be redacted after the period. + msg = self.helper.send(room_id=self.room_id, tok=self.other_access_token) + reaction = self.helper.send_event( + self.room_id, + "m.reaction", + content={ + "m.relates_to": { + "rel_type": RelationTypes.ANNOTATION, + "event_id": msg["event_id"], + "key": "👍", + } + }, + tok=self.other_access_token, + ) + + self.reactor.advance(60 * 60 + 1) + + self._redact_event( + self.other_access_token, + self.room_id, + reaction["event_id"], + expect_code=200, + ) + def test_redact_nonexistent_event(self) -> None: # control case: an existing event b = self.helper.send(room_id=self.room_id, tok=self.other_access_token) From 8f27ac006f29f1a9fac8d61a4553c11a8a0a928d Mon Sep 17 00:00:00 2001 From: guillermo <60621778+guillemo12@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:49:44 +0200 Subject: [PATCH 144/158] Add unit tests for parse_stripped_state_event (#20136) This PR adds a suite of tests for Synapse state events. It covers key scenarios around creation, updates and state consistency to prevent regressions in event processing and serialization. The primary goal is to increase test coverage. --- changelog.d/20136.misc | 1 + tests/events/test_utils.py | 117 ++++++++++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20136.misc diff --git a/changelog.d/20136.misc b/changelog.d/20136.misc new file mode 100644 index 0000000000..3a335cdd6e --- /dev/null +++ b/changelog.d/20136.misc @@ -0,0 +1 @@ +Add missing tests for parse_stripped_state_event. Contributed by @guillemo12. diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 8435d6308a..9fd85fc542 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, Mapping from synapse.api.constants import EventContentFields from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase +from synapse.events import EventBase, StrippedStateEvent from synapse.events.utils import ( FilteredEvent, PowerLevelsContent, @@ -35,6 +35,7 @@ from synapse.events.utils import ( format_event_for_client_v2_without_room_id, format_event_raw, maybe_upsert_event_field, + parse_stripped_state_event, prune_event, ) from synapse.types import JsonDict @@ -1003,6 +1004,120 @@ class CopyPowerLevelsContentTestCase(stdlib_unittest.TestCase): copy_and_fixup_power_levels_contents({"a": {"b": {"c": 1}}}) # type: ignore[dict-item] +class TestParseStrippedStateEvent(stdlib_unittest.TestCase): + def test_valid_dict(self) -> None: + """A valid dict should be parsed into a StrippedStateEvent.""" + raw = { + "type": "m.room.member", + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + result = parse_stripped_state_event(raw) + self.assertEqual( + result, + StrippedStateEvent( + type="m.room.member", + state_key="@alice:example.com", + sender="@alice:example.com", + content={"membership": "join"}, + ), + ) + + def test_invalid_type(self) -> None: + """Non-dict inputs should return None.""" + self.assertIsNone(parse_stripped_state_event("string")) + self.assertIsNone(parse_stripped_state_event(123)) + self.assertIsNone(parse_stripped_state_event([])) + self.assertIsNone(parse_stripped_state_event(None)) + + def test_missing_fields(self) -> None: + """Dicts with missing required fields should return None.""" + self.assertIsNone( + parse_stripped_state_event( + { + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + } + ) + ) + + def test_invalid_field_types(self) -> None: + """Dicts with invalid field types should return None.""" + # Type must be string + self.assertIsNone( + parse_stripped_state_event( + { + "type": 123, + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + # State key must be string + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": 123, + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + # Sender must be string + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": "@alice:example.com", + "sender": 123, + "content": {"membership": "join"}, + } + ) + ) + # Content must be dict + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + "content": "membership_join", + } + ) + ) + + class FormatEventForClientTestCase(stdlib_unittest.TestCase): """Tests for the standalone `format_event_*` transforms. From 92fb8a06dc351c184eb96360c67b124f26c2d54b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 2 Sep 2026 16:23:38 -0500 Subject: [PATCH 145/158] 1.160.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d6d9b22f2f..6fbfa1adfd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.160.0 (2026-09-02) + +No significant changes since 1.160.0rc2. + + + + # Synapse 1.160.0rc2 (2026-08-31) ## Bugfixes diff --git a/debian/changelog b/debian/changelog index 63c2d5fcad..529bfb865c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.160.0) stable; urgency=medium + + * New synapse release 1.160.0. + + -- Synapse Packaging team Wed, 02 Sep 2026 21:22:51 +0000 + matrix-synapse-py3 (1.160.0~rc2) stable; urgency=medium * New synapse release 1.160.0rc2. diff --git a/pyproject.toml b/pyproject.toml index 69a493ee09..600660bd93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.160.0rc2" +version = "1.160.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From d4de2ae1c5338f1d3c2f44aa6ef7bc6b1562f1da Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Thu, 3 Sep 2026 12:58:56 +0200 Subject: [PATCH 146/158] Fix `state_after` for left rooms including post-leave lazy-loaded memberships (#20169) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## The bug With the experimental [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) implementation enabled (`use_state_after`) and lazy-loading of room members, an incremental `/sync` could disclose state from **after** the user's leave in a left room's `state_after`. 1. Alice syncs with `lazy_load_members: true` and `use_state_after=true`. 2. Bob sends a message in a room they share. 3. Alice leaves the room. 4. Bob updates his per-room displayname 5. Alice does an incremental sync covering steps 2–3. Alice's `state_after` contains Bob's post-leave membership event from step 4 Alice should not see the new per-room display name of Bob. ## The fix Copy what has been done for `_compute_state_delta_for_full_sync`: pass `joined` down and, for rooms the user is no longer joined to, fetch the memberships as of `end_token` via state groups (`get_state_ids_at`) instead of current state. --- changelog.d/20169.bugfix | 1 + synapse/handlers/sync.py | 30 +++++++-- tests/handlers/test_sync.py | 2 + tests/rest/client/test_sync.py | 115 +++++++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 changelog.d/20169.bugfix diff --git a/changelog.d/20169.bugfix b/changelog.d/20169.bugfix new file mode 100644 index 0000000000..1845478a22 --- /dev/null +++ b/changelog.d/20169.bugfix @@ -0,0 +1 @@ +Fix `/sync` returning membership events from after the user's leave in `state_after` for left rooms when lazy-loading room members (experimental [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) implementation). diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f8a766feee..049726a97e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1249,6 +1249,7 @@ class SyncHandler: end_token, members_to_fetch, timeline_state, + joined, ) # If we only have partial state for the room, `state_ids` may be missing the @@ -1471,6 +1472,7 @@ class SyncHandler: end_token: StreamToken, members_to_fetch: set[str] | None, timeline_state: StateMap[str], + joined: bool, ) -> StateMap[str]: """Calculate the state events to be included in an incremental sync response. @@ -1495,6 +1497,7 @@ class SyncHandler: events in the timeline. Otherwise, `None`. timeline_state: The contribution to the room state from state events in `batch`. Only contains the last event for any given state key. + joined: whether the user is currently joined to the room Returns: A map from (type, state_key) to event_id, for each event that we believe @@ -1520,13 +1523,28 @@ class SyncHandler: # events to understand the events in this timeline. So we always # fish out all the member events corresponding to the timeline # here. The caller will then dedupe any redundant ones. - member_ids = await self._state_storage_controller.get_current_state_ids( - room_id=room_id, - state_filter=StateFilter.from_types( - (EventTypes.Member, member) for member in members_to_fetch - ), - await_full_state=await_full_state, + member_filter = StateFilter.from_types( + (EventTypes.Member, member) for member in members_to_fetch ) + if joined: + member_ids = ( + await self._state_storage_controller.get_current_state_ids( + room_id=room_id, + state_filter=member_filter, + await_full_state=await_full_state, + ) + ) + else: + # The user is no longer in the room, so `end_token` points + # at the user's leave/etc event, and the current state may + # include state from after that point. Use state groups to + # get the memberships as of `end_token` instead. + member_ids = await self._state_storage_controller.get_state_ids_at( + room_id, + stream_position=end_token, + state_filter=member_filter, + await_full_state=await_full_state, + ) delta_state_ids.update(member_ids) # We don't do LL filtering for incremental syncs - see diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index b732c501d9..e100943413 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -2888,6 +2888,7 @@ class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): end_token=end_stream_token, members_to_fetch=None, timeline_state={}, + joined=True, ) ) self.assertEqual(state[("m.test_event", "")], second_state["event_id"]) @@ -2919,6 +2920,7 @@ class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): end_token=end_stream_token, members_to_fetch=set(), timeline_state={}, + joined=True, ) ) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 74a8678ae9..039aea4d78 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -33,6 +33,7 @@ from synapse.api.constants import ( ReceiptTypes, RelationTypes, ) +from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.rest.client import devices, knock, login, read_marker, receipts, room, sync from synapse.server import HomeServer from synapse.types import JsonDict @@ -1272,3 +1273,117 @@ class SyncCancellationTestCase(unittest.HomeserverTestCase): ) self.assertEqual(200, channel.code, msg=channel.result["body"]) + + +class SyncStateAfterArchivedRoomTestCase(unittest.HomeserverTestCase): + """Tests MSC4222 `state_after` behaviour for rooms the syncing user has + left (i.e. rooms in the `leave` section of the sync response).""" + + servlets = [ + synapse.rest.admin.register_servlets, + room.register_servlets, + login.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + + def test_archived_room_state_after_not_newer_than_leave(self) -> None: + """`state_after` for a left room must be the state at the end of that + room's timeline, i.e. at the user's leave point — never state from + after the leave. + + Scenario: with lazy-loading of members and `use_state_after` enabled, + Alice does an incremental sync covering the window in which Bob sent a + message and Alice then left. Bob changed his per-room displayname + *after* Alice's leave; that post-leave membership event must NOT + appear in Alice's `state_after` for the left room. + """ + alice = self.register_user("alice", "password") + alice_tok = self.login("alice", "password") + bob = self.register_user("bob", "password") + bob_tok = self.login("bob", "password") + + # Opt Alice in to MSC4222. + self.get_success( + self.store.set_features_for_user(alice, {ExperimentalFeature.MSC4222: True}) + ) + + # Name the room to avoid heroes: those come from the *current* + # summary — a separate leak path from the one under test. + room_id = self.helper.create_room_as( + alice, tok=alice_tok, extra_content={"name": "Some room name"} + ) + self.helper.join(room_id, bob, tok=bob_tok) + + # Bob's membership as it will stand at Alice's leave point. + channel = self.make_request( + "GET", + f"/_matrix/client/v3/rooms/{room_id}/state/m.room.member/{bob}?format=event", + access_token=alice_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + bob_member_event_id_at_leave = channel.json_body["event_id"] + + # Lazy-load members; `include_redundant_members` bypasses the members + # cache so Bob's membership appears in the incremental sync below. + sync_filter = json.dumps( + { + "room": { + "state": { + "lazy_load_members": True, + "include_redundant_members": True, + }, + } + } + ) + sync_url = f"/sync?filter={sync_filter}&org.matrix.msc4222.use_state_after=true" + + # Initial sync. + channel = self.make_request("GET", sync_url, access_token=alice_tok) + self.assertEqual(channel.code, 200, channel.result) + since = channel.json_body["next_batch"] + + # Bob becomes a timeline sender in the next sync window. + self.helper.send(room_id, body="hello", tok=bob_tok) + + # Alice leaves the room. + self.helper.leave(room_id, alice, tok=alice_tok) + + # Bob's membership changes AFTER Alice's leave. + post_leave_member_event = self.helper.send_state( + room_id, + EventTypes.Member, + {"membership": "join", "displayname": "bob-post-leave"}, + tok=bob_tok, + state_key=bob, + ) + post_leave_member_event_id = post_leave_member_event["event_id"] + + # Incremental sync: the room is in the `leave` section. + channel = self.make_request( + "GET", f"{sync_url}&since={since}", access_token=alice_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + left_room = channel.json_body["rooms"]["leave"][room_id] + state_after_events = left_room["org.matrix.msc4222.state_after"]["events"] + + # Post-leave state must not appear in `state_after`. + self.assertNotIn( + post_leave_member_event_id, + [e["event_id"] for e in state_after_events], + f"state_after contains state from after the user's leave: " + f"{state_after_events}", + ) + + # Bob's membership must be the one at the leave point. + self.assertEqual( + [ + e["event_id"] + for e in state_after_events + if e["type"] == EventTypes.Member and e["state_key"] == bob + ], + [bob_member_event_id_at_leave], + ) From 1727ceee7ba461152836399e6ba6c12cd7baec80 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Thu, 3 Sep 2026 13:01:21 +0200 Subject: [PATCH 147/158] Ratelimit the room reporting endpoint (MSC4151) (#20036) Apply the `rc_reports` rate limit to the room reporting endpoint, [`POST /_matrix/client/v3/rooms/{roomId}/report`](https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3roomsroomidreport) (added in Matrix v1.13). The spec marks this endpoint as **Rate-limited: Yes** (clients must expect a `429 M_LIMIT_EXCEEDED`), and homeservers [SHOULD implement rate limiting](https://spec.matrix.org/v1.19/client-server-api/#rate-limiting) in general, but Synapse currently applies no limit here. The sibling user reporting endpoint already uses `rc_reports`, so this reuses the same limit instead of introducing a new config option. Changes: - Move the room report logic from `ReportRoomRestServlet` into a new `ReportsHandler.report_room`, mirroring the existing `report_user`. The rate limit is checked before the room existence lookup, so it bounds the DB work a caller can trigger and cannot be used to tell existing rooms from non-existing ones. - The servlet keeps the existing behaviour of returning `200` regardless of room existence when `msc4277_enabled` is set (the spec allows this since v1.18). - Add a regression test covering the `429` response and the per-user rate limit override. --- changelog.d/20036.bugfix | 1 + synapse/handlers/reports.py | 34 ++++++++++++++++++++++++++++- synapse/rest/client/reporting.py | 27 +++++++---------------- tests/rest/client/test_reporting.py | 18 +++++++++++++++ 4 files changed, 60 insertions(+), 20 deletions(-) create mode 100644 changelog.d/20036.bugfix diff --git a/changelog.d/20036.bugfix b/changelog.d/20036.bugfix new file mode 100644 index 0000000000..b2f7c3e79f --- /dev/null +++ b/changelog.d/20036.bugfix @@ -0,0 +1 @@ +Apply the `rc_reports` rate limit to the [room reporting endpoint](https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3roomsroomidreport), which the spec declares as rate-limited. diff --git a/synapse/handlers/reports.py b/synapse/handlers/reports.py index a7b8a4bed7..6cf688e8cc 100644 --- a/synapse/handlers/reports.py +++ b/synapse/handlers/reports.py @@ -17,7 +17,7 @@ import logging from http import HTTPStatus from typing import TYPE_CHECKING -from synapse.api.errors import Codes, SynapseError +from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.types import ( Requester, @@ -91,6 +91,38 @@ class ReportsHandler: received_ts=self._clock.time_msec(), ) + async def report_room( + self, requester: Requester, room_id: str, reason: str + ) -> None: + """Files a report against a room from a user. + + A rate limit is applied to the report. + + If the report is otherwise valid (for a room which exists on our + server), we append it to the database for later processing. + + Args: + requester - The user filing the report. + room_id - The room being reported. + reason - The user-supplied reason the room is being reported. + + Raises: + NotFoundError if the room does not exist. + """ + + await self._check_limits(requester) + + room = await self._store.get_room(room_id) + if room is None: + raise NotFoundError("Room does not exist") + + await self._store.add_room_report( + room_id=room_id, + user_id=requester.user.to_string(), + reason=reason, + received_ts=self._clock.time_msec(), + ) + async def _check_limits(self, requester: Requester) -> None: await self._reports_ratelimiter.ratelimit( requester, diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index 0c594b9f3f..c99d982acc 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -131,8 +131,7 @@ class ReportRoomRestServlet(RestServlet): super().__init__() self.hs = hs self.auth = hs.get_auth() - self.clock = hs.get_clock() - self.store = hs.get_datastores().main + self.reports_handler = hs.get_reports_handler() class PostBody(RequestBodyModel): reason: StrictStr @@ -141,25 +140,15 @@ class ReportRoomRestServlet(RestServlet): self, request: SynapseRequest, room_id: str ) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - user_id = requester.user.to_string() - body = parse_and_validate_json_object_from_request(request, self.PostBody) - room = await self.store.get_room(room_id) - if room is None: - if self.hs.config.experimental.msc4277_enabled: - # Respond with 200 and no content regardless of whether the room - # exists to prevent enumeration attacks. - return 200, {} - else: - raise NotFoundError("Room does not exist") - - await self.store.add_room_report( - room_id=room_id, - user_id=user_id, - reason=body.reason, - received_ts=self.clock.time_msec(), - ) + try: + await self.reports_handler.report_room(requester, room_id, body.reason) + except NotFoundError: + if not self.hs.config.experimental.msc4277_enabled: + raise + # Respond with 200 and no content regardless of whether the room + # exists to prevent enumeration attacks. return 200, {} diff --git a/tests/rest/client/test_reporting.py b/tests/rest/client/test_reporting.py index 96697b96d5..ed33ff1b79 100644 --- a/tests/rest/client/test_reporting.py +++ b/tests/rest/client/test_reporting.py @@ -211,6 +211,24 @@ class ReportRoomTestCase(unittest.HomeserverTestCase): msg=channel.result["body"], ) + @override_config({"rc_reports": {"per_second": 0.5, "burst_count": 1}}) + def test_ratelimit(self) -> None: + """ + Tests that the room report endpoint is rate limited. + """ + data = {"reason": "this makes me sad"} + + self._assert_status(200, data) + self._assert_status(429, data) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.other_user, 0, 0) + ) + + # Test that the request isn't ratelimited anymore. + self._assert_status(200, data) + @override_config({"experimental_features": {"msc4277_enabled": True}}) def test_room_existence_hidden(self) -> None: """ From 366a020f79eaf7bca640347ec26fad29601c7092 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Thu, 3 Sep 2026 13:05:08 +0200 Subject: [PATCH 148/158] Fix MSC4178 error codes on `/account/3pid/{email,msisdn}/requestToken` (#20101) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of https://github.com/element-hq/synapse/issues/18118 ## What the spec says Since Matrix v1.13 (introduced by [MSC4178](https://github.com/matrix-org/matrix-spec-proposals/pull/4178)), the `400` response of [`POST /_matrix/client/v3/account/3pid/email/requestToken`](https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3account3pidemailrequesttoken) lists, among the "Error codes that can be returned": > - `M_THREEPID_MEDIUM_NOT_SUPPORTED`: The homeserver does not support adding email addresses. > - `M_INVALID_PARAM`: The email address given was not valid. and [`POST /_matrix/client/v3/account/3pid/msisdn/requestToken`](https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3account3pidmsisdnrequesttoken) likewise: > - `M_THREEPID_MEDIUM_NOT_SUPPORTED`: The homeserver does not support adding phone numbers. > - `M_INVALID_PARAM`: The phone number given was not valid. ## What was missing Synapse implemented the headline case (unsupported medium), but around it: - A malformed email address or country code was reported with the generic `M_BAD_JSON` instead of `M_INVALID_PARAM`. The email validator deliberately kept `M_BAD_JSON` "to ensure backward compatibility of HTTP error codes" (matrix-org/synapse#13687, 2022) — that predates Matrix v1.13, which now lists `M_INVALID_PARAM` for this case. - On the msisdn variant, the unsupported-medium check ran after the denied/in-use checks, so a request wrong in two ways reported the other fault; the email variant checks it first. ## What this PR changes - Malformed email addresses and country codes on `/account/3pid/{email,msisdn}/requestToken` are reported with `M_INVALID_PARAM`. Both flow through the existing errcode translation as `value_error`: the email validator raises a plain `ValueError`, and the country-code constraint (`ISO3166_1_Alpha_2`) declares its own error via pydantic-core's `custom_error_schema`. - On the msisdn variant the unsupported-medium check now runs before the denied/in-use checks, as on the email variant. - The country-code type is renamed from `ISO3116_1_Alpha_2` to `ISO3166_1_Alpha_2` (typo in the standard's number). [`/account/password/email/requestToken`](https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3accountpasswordemailrequesttoken) (not covered by the v1.13 change) shares the email request body model, so a malformed email there is now also reported with `M_INVALID_PARAM` instead of `M_BAD_JSON`. Its `400` response is described as "the request was invalid" and only names `M_SERVER_NOT_TRUSTED` explicitly ("can be returned if…") rather than restricting the server to a fixed list, and `M_INVALID_PARAM` is the spec's generic code for "A parameter that was specified has the wrong value" ([other error codes](https://spec.matrix.org/v1.19/client-server-api/#other-error-codes)). --- changelog.d/20101.bugfix | 1 + synapse/rest/client/account.py | 22 +++---- synapse/types/rest/client/__init__.py | 33 ++++++---- tests/rest/client/test_account.py | 91 +++++++++++++++++++++++++-- 4 files changed, 120 insertions(+), 27 deletions(-) create mode 100644 changelog.d/20101.bugfix diff --git a/changelog.d/20101.bugfix b/changelog.d/20101.bugfix new file mode 100644 index 0000000000..60e1d3ada2 --- /dev/null +++ b/changelog.d/20101.bugfix @@ -0,0 +1 @@ +Return the `M_INVALID_PARAM` error code specified by Matrix v1.13 ([MSC4178](https://github.com/matrix-org/matrix-spec-proposals/pull/4178)) when a malformed email address or country code is submitted to `/account/3pid/{email,msisdn}/requestToken`, and report an unsupported medium ahead of the denied/in-use checks on the msisdn variant. diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 3b01e40121..13bc703628 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -423,6 +423,17 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): self.identity_handler = hs.get_identity_handler() async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: + if not self.hs.config.registration.account_threepid_delegate_msisdn: + logger.warning( + "No upstream msisdn account_threepid_delegate configured on the server to " + "handle this request" + ) + raise SynapseError( + 400, + "Adding phone numbers to user account is not supported by this homeserver", + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, + ) + body = parse_and_validate_json_object_from_request( request, MsisdnRequestTokenBody ) @@ -462,17 +473,6 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): logger.info("MSISDN %s is already in use by %s", msisdn, existing_user_id) raise SynapseError(400, "MSISDN is already in use", Codes.THREEPID_IN_USE) - if not self.hs.config.registration.account_threepid_delegate_msisdn: - logger.warning( - "No upstream msisdn account_threepid_delegate configured on the server to " - "handle this request" - ) - raise SynapseError( - 400, - "Adding phone numbers to user account is not supported by this homeserver", - Codes.THREEPID_MEDIUM_NOT_SUPPORTED, - ) - ret = await self.identity_handler.requestMsisdnToken( self.hs.config.registration.account_threepid_delegate_msisdn, body.country, diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index 353367c0d7..17647e621d 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -26,6 +26,7 @@ from pydantic import ( ConfigDict, Field, GetCoreSchemaHandler, + GetPydanticSchema, StrictBool, StrictInt, StrictStr, @@ -33,7 +34,7 @@ from pydantic import ( field_validator, model_validator, ) -from pydantic_core import CoreSchema, PydanticCustomError +from pydantic_core import CoreSchema from typing_extensions import Annotated, Self from synapse.types import Absent, AbsentType, NonNegativeStrictInt @@ -91,25 +92,33 @@ class EmailRequestTokenBody(ThreepidRequestTokenBody): # know the exact spelling (eg. upper and lower case) of address in the database. # Without this, an email stored in the database as "foo@bar.com" would cause # user requests for "FOO@bar.com" to raise a Not Found error. + # + # A ValueError produces a Pydantic error of type "value_error", which + # validate_json_object translates to M_INVALID_PARAM, the errcode the spec + # lists for an invalid address on /account/3pid/email/requestToken: + # https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3account3pidemailrequesttoken @field_validator("email") @classmethod def _email_validator(cls, email: StrictStr) -> StrictStr: - try: - return validate_email(email) - except ValueError as e: - # To ensure backward compatibility of HTTP error codes, we return a - # Pydantic error with the custom, unrecognized error type - # "email_custom_err_type" instead of the default error type - # "value_error". This results in the more generic BAD_JSON HTTP - # error instead of the more specific INVALID_PARAM one. - raise PydanticCustomError("email_custom_err_type", str(e), None) from e + return validate_email(email) -ISO3116_1_Alpha_2 = Annotated[str, StringConstraints(pattern="[A-Z]{2}", strict=True)] +ISO3166_1_Alpha_2 = Annotated[ + str, + GetPydanticSchema( + lambda source, handler: pydantic_core.core_schema.custom_error_schema( + pydantic_core.core_schema.str_schema(pattern="[A-Z]{2}", strict=True), + custom_error_type="value_error", + custom_error_context={ + "error": "Not a valid ISO 3166-1 alpha-2 country code" + }, + ) + ), +] class MsisdnRequestTokenBody(ThreepidRequestTokenBody): - country: ISO3116_1_Alpha_2 + country: ISO3166_1_Alpha_2 phone_number: StrictStr diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 8ffac644b6..074186b1e6 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -42,7 +42,7 @@ from synapse.types import JsonDict, UserID, create_requester from synapse.util.clock import Clock from tests import unittest -from tests.server import FakeSite, make_request +from tests.server import FakeChannel, FakeSite, make_request from tests.unittest import override_config @@ -359,6 +359,26 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): self._validate_token(link, next_link) + def test_password_reset_invalid_email(self) -> None: + """A malformed email address is reported with M_INVALID_PARAM, as on + /account/3pid/email/requestToken (the two endpoints share the request + body model). + """ + channel = self.make_request( + "POST", + b"account/password/email/requestToken", + { + "client_secret": "foobar", + "email": "address-without-at.bar", + "send_attempt": 1, + }, + ) + self.assertEqual( + HTTPStatus.BAD_REQUEST, channel.code, msg=channel.result["body"] + ) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertIn("Unable to parse email address", channel.json_body["error"]) + def _request_token( self, email: str, @@ -1010,21 +1030,21 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): def test_add_email_no_at(self) -> None: self._request_token_invalid_email( "address-without-at.bar", - expected_errcode=Codes.BAD_JSON, + expected_errcode=Codes.INVALID_PARAM, expected_error="Unable to parse email address", ) def test_add_email_two_at(self) -> None: self._request_token_invalid_email( "foo@foo@test.bar", - expected_errcode=Codes.BAD_JSON, + expected_errcode=Codes.INVALID_PARAM, expected_error="Unable to parse email address", ) def test_add_email_bad_format(self) -> None: self._request_token_invalid_email( "user@bad.example.net@good.example.com", - expected_errcode=Codes.BAD_JSON, + expected_errcode=Codes.INVALID_PARAM, expected_error="Unable to parse email address", ) @@ -1435,6 +1455,69 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): self.assertIn(expected_email, threepids) +class ThreepidMsisdnRestTestCase(unittest.HomeserverTestCase): + """Tests the error codes of /account/3pid/msisdn/requestToken. + + See https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3account3pidmsisdnrequesttoken + (error codes added in Matrix v1.13 by MSC4178). + """ + + servlets = [ + account.register_servlets, + login.register_servlets, + synapse.rest.admin.register_servlets_for_client_rest_resource, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.user_id = self.register_user("kermit", "test") + + def _request_token(self, country: str, phone_number: str) -> FakeChannel: + return self.make_request( + "POST", + b"account/3pid/msisdn/requestToken", + { + "client_secret": "foobar", + "country": country, + "phone_number": phone_number, + "send_attempt": 1, + }, + ) + + @override_config({"account_threepid_delegates": {"msisdn": "https://id_server"}}) + def test_invalid_country_code(self) -> None: + """A malformed country code is reported with M_INVALID_PARAM.""" + channel = self._request_token("gb", "07700900001") + self.assertEqual( + HTTPStatus.BAD_REQUEST, channel.code, msg=channel.result["body"] + ) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + @override_config({"account_threepid_delegates": {"msisdn": "https://id_server"}}) + def test_invalid_phone_number(self) -> None: + """An unparseable phone number is reported with M_INVALID_PARAM.""" + channel = self._request_token("GB", "not a phone number") + self.assertEqual( + HTTPStatus.BAD_REQUEST, channel.code, msg=channel.result["body"] + ) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + @override_config({"allowed_local_3pids": [{"medium": "email", "pattern": ".*"}]}) + def test_medium_not_supported_checked_before_denied(self) -> None: + """When the server cannot send validation SMSes, it reports + M_THREEPID_MEDIUM_NOT_SUPPORTED even if the phone number would be + denied: the unsupported-medium check comes first, as on the email + variant. + """ + channel = self._request_token("GB", "07700900001") + self.assertEqual( + HTTPStatus.BAD_REQUEST, channel.code, msg=channel.result["body"] + ) + self.assertEqual( + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, channel.json_body["errcode"] + ) + + class AccountStatusTestCase(unittest.HomeserverTestCase): servlets = [ account.register_servlets, From 207f6560ce13d3d7d729fd26914f0be377e26ee6 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Thu, 3 Sep 2026 16:07:51 +0200 Subject: [PATCH 149/158] Stabilize the `M_APPSERVICE_LOGIN_UNSUPPORTED` error code (#20180) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of: https://github.com/element-hq/synapse/issues/19415 Return `M_APPSERVICE_LOGIN_UNSUPPORTED` error code instead of the unstable `IO.ELEMENT.MSC4190.M_APPSERVICE_LOGIN_UNSUPPORTED` identifier. > Servers MUST still allow application services to use the `/register` endpoint with a login type of `m.login.application_service` even if they don't support the Legacy Authentication API. In that case application services MUST set the `"inhibit_login": true` parameter as they cannot use it to log in as users. If the `inhibit_login` parameter is not set to `true`, the server MUST return a 400 HTTP status code with an `M_APPSERVICE_LOGIN_UNSUPPORTED` error code. > > [...] > > Application services MUST NOT use the `/login` endpoint if the server doesn't support the Legacy authentication API. If `/login` is called with the `m.login.application_service` login type the server MUST return a 400 HTTP status code with an `M_APPSERVICE_LOGIN_UNSUPPORTED` error code. > > — [Matrix v1.19, Application Service API](https://spec.matrix.org/v1.19/application-service-api/#registration) Synapse returns the correct 400 on both endpoints, but with the unstable identifier. Before: ``` POST /_matrix/client/v3/login {"type": "m.login.application_service", ...} # appservice with MSC4190 device management POST /_matrix/client/v3/register {"type": "m.login.application_service", ...} # without "inhibit_login": true 400 {"errcode": "IO.ELEMENT.MSC4190.M_APPSERVICE_LOGIN_UNSUPPORTED"} ``` After: ``` POST /_matrix/client/v3/login {"type": "m.login.application_service", ...} # appservice with MSC4190 device management POST /_matrix/client/v3/register {"type": "m.login.application_service", ...} # without "inhibit_login": true 400 {"errcode": "M_APPSERVICE_LOGIN_UNSUPPORTED"} ``` A Sister PR exists in MAS: https://github.com/element-hq/matrix-authentication-service/pull/5961 ; when delegation is enabled, `/login` reaches MAS instead of Synapse, and MAS currently answers `m.login.application_service` with `M_UNKNOWN`. --- changelog.d/20180.bugfix | 1 + synapse/api/errors.py | 2 +- tests/handlers/test_oauth_delegation.py | 28 +++++++++++++++++++++++++ tests/rest/client/test_login.py | 4 ++-- tests/rest/client/test_register.py | 2 +- 5 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 changelog.d/20180.bugfix diff --git a/changelog.d/20180.bugfix b/changelog.d/20180.bugfix new file mode 100644 index 0000000000..72f4fc51d0 --- /dev/null +++ b/changelog.d/20180.bugfix @@ -0,0 +1 @@ +Return the stable `M_APPSERVICE_LOGIN_UNSUPPORTED` error code, added in Matrix 1.17, instead of its unstable MSC4190-prefixed identifier. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 78fd8c3a60..00b5fa9a76 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -143,7 +143,7 @@ class Codes(str, Enum): INVITE_BLOCKED = "M_INVITE_BLOCKED" # Part of MSC4190 - APPSERVICE_LOGIN_UNSUPPORTED = "IO.ELEMENT.MSC4190.M_APPSERVICE_LOGIN_UNSUPPORTED" + APPSERVICE_LOGIN_UNSUPPORTED = "M_APPSERVICE_LOGIN_UNSUPPORTED" # Part of MSC4306: Thread Subscriptions MSC4306_CONFLICTING_UNSUBSCRIPTION = ( diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 995a1134b2..71f83607ff 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -587,6 +587,34 @@ class DisabledEndpointsTestCase(HomeserverTestCase): "POST", "/_matrix/client/v3/register/msisdn/requestToken" ) + def test_appservice_registration_without_inhibit_login(self) -> None: + """Test that appservice registration without `inhibit_login` is rejected + with the `M_APPSERVICE_LOGIN_UNSUPPORTED` errcode.""" + appservice = ApplicationService( + token="i_am_an_app_service", + id="1234", + namespaces={"users": [{"regex": r"@alice:.+", "exclusive": True}]}, + sender=UserID.from_string("@as_main:test"), + ) + + self.hs.get_datastores().main.services_cache = [appservice] + channel = self.make_request( + "POST", + "/_matrix/client/v3/register", + { + "username": "alice", + "type": "m.login.application_service", + }, + shorthand=False, + access_token="i_am_an_app_service", + ) + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual( + channel.json_body.get("errcode"), + "M_APPSERVICE_LOGIN_UNSUPPORTED", + channel.json_body, + ) + def test_session_management_endpoints_removed(self) -> None: """Test that session management endpoints that were removed in MSC2964 are no longer available.""" self.expect_unrecognized("GET", "/_matrix/client/v3/login") diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index d83604a696..f1639bfeee 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -1521,7 +1521,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 200, msg=channel.result) def test_login_appservice_msc4190_fail(self) -> None: - """Test that an appservice user can use /login""" + """Test that an appservice with MSC4190 enabled can't use /login""" self.register_appservice_user( "as3_user_alice", self.msc4190_service.token, inhibit_login=True ) @@ -1537,7 +1537,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 400, msg=channel.result) self.assertEqual( channel.json_body.get("errcode"), - Codes.APPSERVICE_LOGIN_UNSUPPORTED, + "M_APPSERVICE_LOGIN_UNSUPPORTED", channel.json_body, ) diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index 1b04ea9d5c..67f78ee24f 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -183,7 +183,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 400, channel.json_body) self.assertEqual( channel.json_body.get("errcode"), - Codes.APPSERVICE_LOGIN_UNSUPPORTED, + "M_APPSERVICE_LOGIN_UNSUPPORTED", channel.json_body, ) From 0088e43da78c61f63305faad08ed85f264270fce Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:08:38 +0100 Subject: [PATCH 150/158] MSC4242: State DAGs (fed client) (#20127) Split out from https://github.com/element-hq/synapse/pull/19425 - Add federation support for MSC4242 fields in `/send_join` and `/get_missing_events` - Move the gate preventing joining MSC4242 rooms higher in the call stack Part of a series of 5x PRs to land the federation part of MSC4242 (storage, fedclient, serving, inbound-joins, inbound-pulls). ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20127.feature | 1 + synapse/federation/federation_client.py | 190 +++++++++++++++--------- synapse/federation/transport/client.py | 32 +++- synapse/handlers/federation.py | 7 + tests/handlers/test_device.py | 1 + tests/handlers/test_federation.py | 1 + tests/handlers/test_room_member.py | 1 + tests/storage/test_stream.py | 1 + 8 files changed, 157 insertions(+), 77 deletions(-) create mode 100644 changelog.d/20127.feature diff --git a/changelog.d/20127.feature b/changelog.d/20127.feature new file mode 100644 index 0000000000..5e6738e035 --- /dev/null +++ b/changelog.d/20127.feature @@ -0,0 +1 @@ +Add experimental federation client support for [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242): State DAGs. diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 5b33454325..e46392944b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -125,6 +125,9 @@ class SendJoinResult: # Always contains the server we joined off. servers_in_room: AbstractSet[str] + # Only valid for state DAG rooms (MSC4242) + state_dag: list[EventBase] | None + class FederationClient(FederationBase): def __init__(self, hs: "HomeServer"): @@ -1107,11 +1110,12 @@ class FederationClient(FederationBase): SynapseError: if the chosen remote server returns a 300/400 code, or no servers successfully handle the request. """ - # See related restriction in /createRoom requests in handlers/room.py - if room_version.msc4242_state_dags: - raise UnsupportedRoomVersionError( - "Homeserver does not support this room version over federation" - ) + + def find_create_event(events: list[EventBase]) -> EventBase | None: + for e in events: + if (e.type, e.state_key) == (EventTypes.Create, ""): + return e + return None async def send_request(destination: str) -> SendJoinResult: response = await self._do_send_join( @@ -1141,13 +1145,16 @@ class FederationClient(FederationBase): state = response.state auth_chain = response.auth_events + state_dag: list[EventBase] = [] + if room_version.msc4242_state_dags: + if not response.state_dag: + raise InvalidResponseError("No state_dag returned") + state_dag = response.state_dag - create_event = None - for e in state: - if (e.type, e.state_key) == (EventTypes.Create, ""): - create_event = e - break - + # Validate the create event and room version are what we expect to see. + create_event = find_create_event( + state_dag if room_version.msc4242_state_dags else state + ) if create_event is None: # If the state doesn't have a create event then the room is # invalid, and it would fail auth checks anyway. @@ -1165,57 +1172,7 @@ class FederationClient(FederationBase): % (create_room_version,) ) - logger.info( - "Processing from send_join %d events", len(state) + len(auth_chain) - ) - - # We now go and check the signatures and hashes for the event. Note - # that we limit how many events we process at a time to keep the - # memory overhead from exploding. - valid_pdus_map: dict[str, EventBase] = {} - - async def _execute(pdu: EventBase) -> None: - valid_pdu = await self._check_sigs_and_hash_and_fetch_one( - pdu=pdu, - origin=destination, - room_version=room_version, - ) - - if valid_pdu: - valid_pdus_map[valid_pdu.event_id] = valid_pdu - - await concurrently_execute( - _execute, itertools.chain(state, auth_chain), 10000 - ) - - # NB: We *need* to copy to ensure that we don't have multiple - # references being passed on, as that causes... issues. - signed_state = [ - valid_pdus_map[p.event_id].deep_copy() - for p in state - if p.event_id in valid_pdus_map - ] - - signed_auth = [ - valid_pdus_map[p.event_id] - for p in auth_chain - if p.event_id in valid_pdus_map - ] - - # double-check that the auth chain doesn't include a different create event - auth_chain_create_events = [ - e.event_id - for e in signed_auth - if (e.type, e.state_key) == (EventTypes.Create, "") - ] - if auth_chain_create_events and auth_chain_create_events != [ - create_event.event_id - ]: - raise InvalidResponseError( - "Unexpected create event(s) in auth chain: %s" - % (auth_chain_create_events,) - ) - + # Validate and set faster room joins fields servers_in_room = None if response.servers_in_room is not None: servers_in_room = set(response.servers_in_room) @@ -1235,15 +1192,107 @@ class FederationClient(FederationBase): # Fix things up in case the remote homeserver is badly behaved. servers_in_room.add(destination) - return SendJoinResult( - event=event, - state=signed_state, - auth_chain=signed_auth, - origin=destination, - partial_state=response.members_omitted, - servers_in_room=servers_in_room or frozenset(), + logger.info( + "Processing from send_join %d events", + len(state_dag) + if room_version.msc4242_state_dags + else (len(state) + len(auth_chain)), ) + # We now go and check the signatures and hashes for the event. Note + # that we limit how many events we process at a time to keep the + # memory overhead from exploding. + valid_pdus_map: dict[str, EventBase] = {} + + async def _execute(pdu: EventBase) -> None: + valid_pdu = await self._check_sigs_and_hash_and_fetch_one( + pdu=pdu, + origin=destination, + room_version=room_version, + ) + + if valid_pdu: + valid_pdus_map[valid_pdu.event_id] = valid_pdu + + # Verify signatures/hashes on events, and make sure they all refer to the same room. + if room_version.msc4242_state_dags: + if state or auth_chain or servers_in_room: + raise InvalidResponseError( + "State DAG rooms must not set servers_in_room, state or auth_chain fields" + ) + await concurrently_execute(_execute, itertools.chain(state_dag), 10000) + # NB: We *need* to copy to ensure that we don't have multiple + # references being passed on, as that causes... issues. + signed_state_dag = [ + valid_pdus_map[p.event_id].deep_copy() + for p in state_dag + if p.event_id in valid_pdus_map + ] + + # Verify each event is for this room (and thus has the same create event as it is v12+) + for state_event in signed_state_dag: + if state_event.room_id != pdu.room_id: + raise InvalidResponseError( + "%s in state_dag belongs to room %s, not %s which we are joining" + % (state_event.event_id, state_event.room_id, pdu.room_id) + ) + return SendJoinResult( + event=event, + state=[], + auth_chain=[], + state_dag=signed_state_dag, + origin=destination, + # The current Synapse implementation of MSC4242 does not support + # faster remote room joins, so always set partial_state=False. + partial_state=False, + servers_in_room=frozenset(), + ) + else: + if state_dag: + raise InvalidResponseError( + "Room does not support state DAGs but set state_dag field" + ) + await concurrently_execute( + _execute, itertools.chain(state, auth_chain), 10000 + ) + + # NB: We *need* to copy to ensure that we don't have multiple + # references being passed on, as that causes... issues. + signed_state = [ + valid_pdus_map[p.event_id].deep_copy() + for p in state + if p.event_id in valid_pdus_map + ] + + signed_auth = [ + valid_pdus_map[p.event_id] + for p in auth_chain + if p.event_id in valid_pdus_map + ] + + # double-check that the auth chain doesn't include a different create event + auth_chain_create_events = [ + e.event_id + for e in signed_auth + if (e.type, e.state_key) == (EventTypes.Create, "") + ] + if auth_chain_create_events and auth_chain_create_events != [ + create_event.event_id + ]: + raise InvalidResponseError( + "Unexpected create event(s) in auth chain: %s" + % (auth_chain_create_events,) + ) + return SendJoinResult( + event=event, + state=signed_state, + auth_chain=signed_auth, + origin=destination, + partial_state=response.members_omitted, + servers_in_room=servers_in_room or frozenset(), + state_dag=None, + ) + # MSC3083 defines additional error codes for room joins. failover_errcodes = None if room_version.restricted_join_rule: @@ -1542,6 +1591,7 @@ class FederationClient(FederationBase): limit: int, min_depth: int, timeout: int, + state_dag: bool = False, ) -> list[EventBase]: """Tries to fetch events we are missing. This is called when we receive an event without having received all of its ancestors. @@ -1557,6 +1607,7 @@ class FederationClient(FederationBase): limit: Maximum number of events to return. min_depth: Minimum depth of events to return. timeout: Max time to wait in ms + state_dag: True to walk the state DAG (MSC4242 rooms) """ try: content = await self.transport_layer.get_missing_events( @@ -1567,6 +1618,7 @@ class FederationClient(FederationBase): limit=limit, min_depth=min_depth, timeout=timeout, + state_dag=state_dag, ) received_time = self._clock.time_msec() diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 5d5212ef96..e4747016c4 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -776,18 +776,21 @@ class TransportLayerClient: limit: int, min_depth: int, timeout: int, + state_dag: bool, ) -> JsonDict: path = _create_v1_path("/get_missing_events/%s", room_id) - + request_body = { + "limit": int(limit), + "min_depth": int(min_depth), + "earliest_events": earliest_events, + "latest_events": latest_events, + } + if state_dag: + request_body["org.matrix.msc4242.state_dag"] = True return await self.client.post_json( destination=destination, path=path, - data={ - "limit": int(limit), - "min_depth": int(min_depth), - "earliest_events": earliest_events, - "latest_events": latest_events, - }, + data=request_body, timeout=timeout, ) @@ -986,6 +989,10 @@ class SendJoinResponse: # "event" is not included in the response. event: EventBase | None = None + # MSC4242: State DAGs. Always included for state dag rooms, else None. + # Replaces auth_events. + state_dag: list[EventBase] | None = None + # The room state is incomplete members_omitted: bool = False @@ -1068,7 +1075,7 @@ class SendJoinParser(ByteParser[SendJoinResponse]): MAX_RESPONSE_SIZE = 500 * 1024 * 1024 def __init__(self, room_version: RoomVersion, v1_api: bool): - self._response = SendJoinResponse([], [], event_dict={}) + self._response = SendJoinResponse([], [], event_dict={}, state_dag=[]) self._room_version = room_version self._coros: list[Generator[None, bytes, None]] = [] @@ -1112,6 +1119,15 @@ class SendJoinParser(ByteParser[SendJoinResponse]): ) ) + if room_version.msc4242_state_dags: + self._coros.append( + ijson.items_coro( + _event_list_parser(room_version, self._response.state_dag), + prefix + "state_dag.item", + use_float=True, + ) + ) + def write(self, data: bytes) -> int: for c in self._coros: c.send(data) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4816c79c7b..7c5534f403 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -53,6 +53,7 @@ from synapse.api.errors import ( PartialStateConflictError, RequestSendFailed, SynapseError, + UnsupportedRoomVersionError, ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.crypto.event_signing import compute_event_signature @@ -669,6 +670,12 @@ class FederationHandler: room_id ) + # See related restriction in /createRoom requests in handlers/room.py + if room_version_obj.msc4242_state_dags: + raise UnsupportedRoomVersionError( + "Homeserver does not support this room version over federation" + ) + ret = await self.federation_client.send_join( host_list, event, diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index cb047d118a..ce1bd06b6d 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -844,6 +844,7 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): partial_state=True, # Only REMOTE1_SERVER_NAME is known at join time. servers_in_room={self.REMOTE1_SERVER_NAME}, + state_dag=None, ) ) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 1f42490d96..71f29be2ea 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -574,6 +574,7 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase): ], partial_state=True, servers_in_room={"example.com"}, + state_dag=None, ) ) diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 0a7475856a..a4a57b64d9 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -169,6 +169,7 @@ class TestJoinsLimitedByPerRoomRateLimiter(FederatingHomeserverTestCase): auth_chain=[create_event], partial_state=False, servers_in_room=frozenset(), + state_dag=None, ) ) diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index de127e3971..4ece16e9a0 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -1455,6 +1455,7 @@ class GetCurrentStateDeltaMembershipChangesForUserFederationTestCase( auth_chain=[create_event, creator_join_event], partial_state=False, servers_in_room=frozenset(), + state_dag=None, ) ) From a0b5a45c918a765e90a87873ac25e83b857509a1 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Fri, 4 Sep 2026 03:35:28 -0500 Subject: [PATCH 151/158] fix MSC3912-relation-based-redaction for room versions > 10 (#19782) --- changelog.d/19782.misc | 1 + synapse/handlers/relations.py | 30 +++++++++++++++++++++------- synapse/rest/client/room.py | 1 + tests/rest/client/test_redactions.py | 4 ---- 4 files changed, 25 insertions(+), 11 deletions(-) create mode 100644 changelog.d/19782.misc diff --git a/changelog.d/19782.misc b/changelog.d/19782.misc new file mode 100644 index 0000000000..35f6c32385 --- /dev/null +++ b/changelog.d/19782.misc @@ -0,0 +1 @@ +Put the `redacts` key under `content` when generating [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3912) (relation based redactions) for room versions greater than 10. Contributed by @famedly. diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index a8db082feb..dee4746dcb 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -44,6 +44,7 @@ from synapse.synapse_rust.events import ( # noqa: F401 BundledAggregations, ThreadAggregation, ) +from synapse.synapse_rust.room_versions import RoomVersion from synapse.types import JsonDict, Requester, UserID from synapse.util.async_helpers import gather_results from synapse.visibility import filter_and_transform_events_for_client @@ -193,6 +194,7 @@ class RelationsHandler: event_id: str, initial_redaction_event: EventBase, relation_types: list[str], + room_version: RoomVersion, ) -> None: """Redacts all events related to the given event ID with one of the given relation types. @@ -210,6 +212,8 @@ class RelationsHandler: event_id. relation_types: The types of relations to look for. If "*" is in the list, all related events will be redacted regardless of the type. + room_version: The RoomVersion of the room. Used for deciding where the + 'redacts' key should go in the event dict. Raises: ShadowBanError if the requester is shadow-banned @@ -226,16 +230,28 @@ class RelationsHandler: ) for related_event_id in related_event_ids: + new_redaction_content = dict(initial_redaction_event.content) + event_dict: JsonDict = { + "type": EventTypes.Redaction, + "content": new_redaction_content, + "room_id": initial_redaction_event.room_id, + "sender": requester.user.to_string(), + } + # Depending on the room version involved, the "redacts" key can go in one of + # two places. + # + # The Matrix Spec page for changes in Room Version 11 asks that we maintain + # a backward and forwards compatibility for clients over that API. That + # compatibility fixup will be in the client event serialization code. Here + # we form and persist the event strictly by the version of the room. + if room_version.updated_redaction_rules: + event_dict["content"].update({"redacts": related_event_id}) + else: + event_dict["redacts"] = related_event_id try: await self._event_creation_handler.create_and_send_nonmember_event( requester, - { - "type": EventTypes.Redaction, - "content": initial_redaction_event.content, - "room_id": initial_redaction_event.room_id, - "sender": requester.user.to_string(), - "redacts": related_event_id, - }, + event_dict, ratelimit=False, ) except SynapseError as e: diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index c3bb82e15c..1b8774fde3 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -1412,6 +1412,7 @@ class RoomRedactEventRestServlet(TransactionRestServlet): event_id=event_id, initial_redaction_event=event, relation_types=with_relations, + room_version=room_version, ) event_id = event.event_id diff --git a/tests/rest/client/test_redactions.py b/tests/rest/client/test_redactions.py index a2f73d43f4..4cab3fc8c3 100644 --- a/tests/rest/client/test_redactions.py +++ b/tests/rest/client/test_redactions.py @@ -66,10 +66,6 @@ class RedactionsTestCase(HomeserverTestCase): self.room_id = self.helper.create_room_as( self.mod_user_id, tok=self.mod_access_token, - # FIXME: Using room version 10 here as using later version requires - # https://github.com/element-hq/synapse/pull/19782 - # Remove the override when the above PR is merged. - room_version="10", ) # Invite the other user From bd21823818279ff92b3dc1a4c97cb8c543dc9c0c Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Mon, 7 Sep 2026 10:34:30 +0200 Subject: [PATCH 152/158] Fix 500 error when setting a custom profile field for a user with no profile row (#20172) Follow-up to #20149, which fixed the 500 when *getting* a profile field for a user with no `profiles` row. The same crash was still reachable when *setting* one via `PUT /_matrix/client/v3/profile/{userId}/{field}` (as server admin). This PR splits that case in two: * **The user exists but has no `profiles` row** (e.g. profile erased upon deactivation): * Before: `500 M_UNKNOWN` (`TypeError: cannot unpack non-sequence NoneType` in the profile size check). * After: `200`, the profile row is recreated with the field set. * **The user does not exist at all**: * Before: `500 M_UNKNOWN` (same crash). * After: `404 M_NOT_FOUND`, without conjuring up an orphan profile row. Fixing the crash also surfaced a latent SQLite-only bug where a field set on a freshly created profile row was stored under the wrong key, making it 404 on `GET` right after a successful `PUT`. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20172.bugfix | 1 + synapse/handlers/profile.py | 5 +++++ synapse/storage/databases/main/profile.py | 7 +++++-- tests/rest/client/test_profile.py | 15 ++++++++++++++ tests/storage/test_profile.py | 25 +++++++++++++++++++++++ 5 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20172.bugfix diff --git a/changelog.d/20172.bugfix b/changelog.d/20172.bugfix new file mode 100644 index 0000000000..a25b5e1250 --- /dev/null +++ b/changelog.d/20172.bugfix @@ -0,0 +1 @@ +Fix a bug where a server admin setting a custom profile field for a user with no profile received a 500 error; this now succeeds for existing (e.g. deactivated) users and returns a 404 error if the user does not exist. diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index c4f17eb0fb..112ac4c8c5 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -720,6 +720,11 @@ class ProfileHandler: if not by_admin and target_user != requester.user: raise AuthError(403, "Cannot set another user's profile") + # Don't recreate a profile row for a user that does not exist at all; + # deactivated (e.g. erased) users do exist, so are allowed through. + if await self.store.get_user_by_id(target_user.to_string()) is None: + raise SynapseError(404, "User not found", Codes.NOT_FOUND) + stream_id = await self.store.set_profile_field( target_user, field_name, diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index fe48636ffb..1a2dea7460 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -728,7 +728,10 @@ class ProfileWorkerStore(SQLBaseStore): # possible due to the grammar. (f'$."{new_field_name}"', user_id.localpart), ) - row = cast(tuple[int | None, int | None, int | None], txn.fetchone()) + row = cast("tuple[int | None, int | None, int | None] | None", txn.fetchone()) + # The user may have no profile row at all; treat as an empty profile. + if row is None: + row = (None, None, None) # The values return null if the column is null. total_bytes = ( @@ -830,7 +833,7 @@ class ProfileWorkerStore(SQLBaseStore): ( user_id.localpart, user_id.to_string(), - json_field_name, + field_name, canonical_value, json_field_name, canonical_value, diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 95b11ff2b2..524045c751 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -845,6 +845,21 @@ class ProfileTestCase(unittest.HomeserverTestCase): avatar_url = self._get_avatar_url() self.assertEqual(avatar_url, "mxc://test/good") + def test_set_custom_field_never_existed_user(self) -> None: + """Setting a profile field for a user that does not exist should not + conjure up a profile row for them, even for a server admin.""" + self.register_user("admin", "pass", admin=True) + admin_tok = self.login("admin", "pass") + + channel = self.make_request( + "PUT", + "/_matrix/client/v3/profile/@never-existed:test/custom_field", + content={"custom_field": "test"}, + access_token=admin_tok, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) + def test_set_custom_field_other(self) -> None: """Setting someone else's profile field should fail""" channel = self.make_request( diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index 0fc40794c7..0d9abaff47 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -112,6 +112,31 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): ) self.assertEqual(f.value.code, HTTPStatus.NOT_FOUND) + def test_set_profile_field_without_profile(self) -> None: + """ + Setting a custom profile field for a user that has no row in the + `profiles` table at all should create the row and store the field. + + Regression test (we previously would trigger an unhandled exception in + the profile size check, and then store the field under a wrong key on + SQLite). Can happen for users whose profile was erased upon + deactivation. + """ + self.get_success( + self.store.set_profile_field( + user_id=self.u_frank, + field_name="org.example.field", + new_value="test", + ) + ) + + self.assertEqual( + "test", + self.get_success( + self.store.get_profile_field(self.u_frank, "org.example.field") + ), + ) + def test_profiles_bg_migration(self) -> None: """ Test background job that copies entries from column user_id to full_user_id, adding From 470b9418956d347be399b431e271bd092a639fb1 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Mon, 7 Sep 2026 10:35:03 +0200 Subject: [PATCH 153/158] Fix private read receipts of other users being leaked to application services (#20114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Application services that opt in to receiving ephemeral events (`receive_ephemeral: true` in the [registration](https://spec.matrix.org/v1.19/application-service-api/#registration), added in Matrix v1.13 from [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409)) are sent the private read receipts (`m.read.private`) of **every** user in the rooms they are interested in — not just their own users. The [Application Service API](https://spec.matrix.org/v1.19/application-service-api/#pushing-ephemeral-data) is explicit about this (*Pushing ephemeral data*, `m.receipt`): > Private read receipts MUST only be sent for users matching one of the application service's namespaces. Normal read receipts and threaded read receipts are always sent. This matches the [Client-Server API](https://spec.matrix.org/v1.19/client-server-api/#private-read-receipts): "Servers MUST NOT send the `m.read.private` receipt to any other user than the one which originally sent it." The restriction to namespaced users is safe because the appservice could learn those receipts anyway by syncing as the user. For everyone else, `m.read.private` exists precisely so that nobody — including bridges and bots in the room — can observe it. For example, take an appservice registered with: ```yaml namespaces: users: - regex: "@_bridge_.*:example\\.org" exclusive: true ``` When `@alice:example.org` (a regular user, not one of the appservice's) and `@_bridge_bob:example.org` (a namespaced user) each send read receipts in a bridged room, the appservice receives: ```json { "type": "m.receipt", "room_id": "!room:example.org", "content": { "$event": { "m.read": { "@alice:example.org": { "ts": 1436451550453 } }, "m.read.private": { "@_bridge_bob:example.org": { "ts": 1436451550453 }, "@alice:example.org": { "ts": 1436451550453 } } } } } ``` - `m.read` from `@alice` — correct, public read receipts are always sent. - `m.read.private` from `@_bridge_bob` — correct, the user is within the appservice's namespaces. - `m.read.private` from `@alice` — **the leak**: their private read receipt must not be sent to the appservice. ## History - matrix-org/synapse#8437 (Oct 2020, Synapse 1.22.0) implemented MSC2409 ephemeral event delivery to appservices. No leak at that point: private read receipts did not exist yet. - matrix-org/synapse#10413 (Jul 2021, Synapse 1.40.0) added the initial, experimental-flag-gated implementation of MSC2285 ("hidden" read receipts) and filtered them out of the `/sync` path (`filter_out_hidden`) — but not out of the appservice path in the same file. This is where the leak originates, for servers with `msc2285_enabled`. - matrix-org/synapse#12168 (May 2022, Synapse 1.59.0) reworked this into the `m.read.private` receipt type and `filter_out_private_receipts`; the appservice path was again left unfiltered. - matrix-org/synapse#13273 (Aug 2022, Synapse 1.65.0) moved to the stable `m.read.private` identifier; the appservice path has leaked it ever since. --- ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20114.bugfix | 1 + synapse/handlers/receipts.py | 62 +++++++++++++----- tests/handlers/test_appservice.py | 101 +++++++++++++++++++++++++++++- 3 files changed, 149 insertions(+), 15 deletions(-) create mode 100644 changelog.d/20114.bugfix diff --git a/changelog.d/20114.bugfix b/changelog.d/20114.bugfix new file mode 100644 index 0000000000..967a3717c0 --- /dev/null +++ b/changelog.d/20114.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where private read receipts of users outside an application service's namespaces were sent to application services that opted in to receiving ephemeral events ([MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409)). diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index f6383baf0b..c208b4e91a 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -19,7 +19,7 @@ # # import logging -from typing import TYPE_CHECKING, Iterable, Sequence +from typing import TYPE_CHECKING, Callable, Iterable, Sequence from synapse.api.constants import EduTypes, ReceiptTypes from synapse.appservice import ApplicationService @@ -228,6 +228,31 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): Args: rooms: A list of mappings, each mapping has a `content` field, which is a map of event ID -> receipt type -> user ID -> receipt information. + user_id: The user whose private read receipts should be kept. + + Returns: + The same as rooms, but filtered. + """ + return ReceiptEventSource._filter_private_receipts( + rooms, lambda receipt_user_id: receipt_user_id == user_id + ) + + @staticmethod + def _filter_private_receipts( + rooms: Sequence[JsonMapping], is_visible: Callable[[str], bool] + ) -> list[JsonMapping]: + """ + Filters a list of serialized receipts and removes private read receipts + of users for which `is_visible` returns False. + + This may operate on the return value of cached functions. Care must be + taken to ensure that the input values are not modified. + + Args: + rooms: A list of mappings, each mapping has a `content` field, which + is a map of event ID -> receipt type -> user ID -> receipt information. + is_visible: Called with each private read receipt's user ID; returns + whether that user's private read receipts may be included. Returns: The same as rooms, but filtered. @@ -237,7 +262,7 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): # Iterate through each room's receipt content. for room in rooms: - # The receipt content with other user's private read receipts removed. + # The receipt content with hidden users' private read receipts removed. content = {} # Iterate over each event ID / receipts for that event. @@ -246,30 +271,34 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): # If there are private read receipts, additional logic is necessary. if ReceiptTypes.READ_PRIVATE in event_content: # Make a copy without private read receipts to avoid leaking - # other user's private read receipts.. + # hidden users' private read receipts.. event_content = { receipt_type: receipt_value for receipt_type, receipt_value in event_content.items() if receipt_type != ReceiptTypes.READ_PRIVATE } - # Copy the current user's private read receipt from the - # original content, if it exists. - user_private_read_receipt = orig_event_content[ - ReceiptTypes.READ_PRIVATE - ].get(user_id, None) - if user_private_read_receipt: - event_content[ReceiptTypes.READ_PRIVATE] = { - user_id: user_private_read_receipt - } + # Copy the visible users' private read receipts from the + # original content, if there are any. + visible_private_read_receipts = { + receipt_user_id: receipt_value + for receipt_user_id, receipt_value in orig_event_content[ + ReceiptTypes.READ_PRIVATE + ].items() + if is_visible(receipt_user_id) + } + if visible_private_read_receipts: + event_content[ReceiptTypes.READ_PRIVATE] = ( + visible_private_read_receipts + ) # Include the event if there is at least one non-private read - # receipt or the current user has a private read receipt. + # receipt or a visible user has a private read receipt. if event_content: content[event_id] = event_content # Include the event if there is at least one non-private read receipt - # or the current user has a private read receipt. + # or a visible user has a private read receipt. if content: # Build a new event to avoid mutating the cache. new_room = {k: v for k, v in room.items() if k != "content"} @@ -345,6 +374,11 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): events.append(event) + # Private read receipts must only be sent for users matching one of the + # appservice's namespaces (or its sender user). See + # https://spec.matrix.org/v1.19/application-service-api/#pushing-ephemeral-data + events = self._filter_private_receipts(events, service.is_interested_in_user) + return events, to_key def get_current_key(self) -> MultiWriterStreamToken: diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 08a4ab624a..011d0552a5 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -36,7 +36,7 @@ from twisted.internet.testing import MemoryReactor import synapse.rest.admin import synapse.storage -from synapse.api.constants import EduTypes, EventTypes +from synapse.api.constants import EduTypes, EventTypes, ReceiptTypes from synapse.appservice import ( ApplicationService, TransactionOneTimeKeysCount, @@ -704,6 +704,105 @@ class ApplicationServicesHandlerSendEventsTestCase(unittest.HomeserverTestCase): latest_read_receipt["content"][event_id]["m.read"], {self.local_user: {}} ) + def test_application_services_receive_private_read_receipts_of_namespaced_users_only( + self, + ) -> None: + """Tests that private read receipts are only sent to an application + service for users within the appservice's namespaces, while public read + receipts are sent regardless of the sending user. + + See https://spec.matrix.org/v1.19/application-service-api/#pushing-ephemeral-data + """ + # Register an application service that's interested in a certain user + # and room prefix + interested_appservice = self._register_application_service( + namespaces={ + ApplicationService.NS_USERS: [ + { + "regex": "@exclusive_as_user:.+", + "exclusive": True, + } + ], + ApplicationService.NS_ROOMS: [ + { + "regex": "!fakeroom_.*", + "exclusive": True, + } + ], + }, + ) + + room_id = "!fakeroom_private:test" + event_id = "$eventid" + + # A public read receipt from a user outside the appservice's namespaces. + self.get_success( + self.hs.get_datastores().main.insert_receipt( + room_id=room_id, + receipt_type=ReceiptTypes.READ, + user_id=self.local_user, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + # A private read receipt from a user within the appservice's namespaces. + self.get_success( + self.hs.get_datastores().main.insert_receipt( + room_id=room_id, + receipt_type=ReceiptTypes.READ_PRIVATE, + user_id=self.exclusive_as_user, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + # A private read receipt on the same event from a user outside the + # appservice's namespaces. + self.get_success( + self.hs.get_datastores().main.insert_receipt( + room_id=room_id, + receipt_type=ReceiptTypes.READ_PRIVATE, + user_id=self.local_user, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + + # Notify the appservice handler about the receipts in one go. + # note: stream tokens start at 2, so the three receipts above have + # stream IDs 2, 3 and 4. + self.get_success( + self.hs.get_application_service_handler()._notify_interested_services_ephemeral( + services=[interested_appservice], + stream_key=StreamKeyType.RECEIPT, + new_token=MultiWriterStreamToken(stream=4), + users=[self.local_user, self.exclusive_as_user], + ) + ) + + self.send_mock.assert_called_once() + ephemeral_events = self.send_mock.call_args[0][2] + + # All receipts for the room are batched into a single m.receipt event. + self.assertEqual(len(ephemeral_events), 1) + receipt_event = ephemeral_events[0] + self.assertEqual(receipt_event["type"], EduTypes.RECEIPT) + self.assertEqual(receipt_event["room_id"], room_id) + + # The public read receipt and the namespaced user's private read receipt + # should have been sent, but not the other user's private read receipt. + self.assertEqual( + receipt_event["content"], + { + event_id: { + ReceiptTypes.READ: {self.local_user: {}}, + ReceiptTypes.READ_PRIVATE: {self.exclusive_as_user: {}}, + }, + }, + ) + @unittest.override_config( {"experimental_features": {"msc2409_to_device_messages_enabled": True}} ) From e35c8f0eff7d62a1dd830d54d8676d8e961cff3a Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Mon, 7 Sep 2026 10:35:33 +0200 Subject: [PATCH 154/158] Fix `client_secret` param validator (#20104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The validation of `client_secret` params accepts invalid values: any value that includes a char in `[0-9a-zA-Z.=_-]` is accepted, for example `"café"` is accepted. Instead synapse should only accept if all chars are within `[0-9a-zA-Z.=_-]`, not just one. This is the current validator: ```python ClientSecretStr = Annotated[ str, StringConstraints( pattern="[0-9a-zA-Z.=_-]", min_length=1, max_length=255, strict=True, ), ] ``` Unfortunately, Pydantic only defines the `pattern` argument as: > ### pattern > A regex pattern that the string must match. Which is extremely imprecise. Here is a little script to verify the behavior: ```python from typing import Annotated from pydantic import BaseModel, StringConstraints, ValidationError def check(pattern: str) -> None: ClientSecretStr = Annotated[ str, StringConstraints(pattern=pattern, min_length=1, max_length=255, strict=True), ] class Body(BaseModel): client_secret: ClientSecretStr try: Body.model_validate({"client_secret": "café"}) print(f"pattern = {pattern!r}: 'café' ACCEPTED <-- should have been rejected") except ValidationError: print(f"pattern = {pattern!r}: 'café' rejected") check("[0-9a-zA-Z.=_-]") # before the fix (unanchored) check("^[0-9a-zA-Z.=_-]+$") # after the fix ``` which would output ``` pattern = '[0-9a-zA-Z.=_-]': 'café' ACCEPTED <-- should have been rejected pattern = '^[0-9a-zA-Z.=_-]+$': 'café' rejected ``` ## History This is a regression of a previously reported and fixed bug: - matrix-org/synapse#6766 (2020) reported that Synapse did not enforce the spec's `client_secret` regex at all — with real-world fallout: FluffyChat had started sending secrets containing `:` because nothing rejected them. Fixed by introducing `assert_valid_client_secret` (matrix-org/synapse#6767). - matrix-org/synapse#13188 (Synapse 1.66.0) ported the account endpoints to Pydantic and transcribed the regex without anchors/quantifier; Pydantic v1's `re.match` semantics meant only the *first* character was validated. - #19071 (Synapse 1.142.0) migrated to Pydantic v2, whose *search* semantics weakened it further to "any one character anywhere". --- ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/20104.bugfix | 1 + synapse/types/rest/client/__init__.py | 2 +- tests/rest/client/test_models.py | 40 ++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20104.bugfix diff --git a/changelog.d/20104.bugfix b/changelog.d/20104.bugfix new file mode 100644 index 0000000000..f04945660c --- /dev/null +++ b/changelog.d/20104.bugfix @@ -0,0 +1 @@ +Fix a regression where `client_secret` request parameters were not validated against the character set required by the spec on Pydantic-validated endpoints, regressed in Synapse 1.66.0 (originally fixed for https://github.com/matrix-org/synapse/issues/6766). diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index 17647e621d..dbd4455b82 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -62,7 +62,7 @@ class AuthenticationData(RequestBodyModel): ClientSecretStr = Annotated[ str, StringConstraints( - pattern="[0-9a-zA-Z.=_-]", + pattern="^[0-9a-zA-Z.=_-]+$", min_length=1, max_length=255, strict=True, diff --git a/tests/rest/client/test_models.py b/tests/rest/client/test_models.py index f297856830..e5dc922023 100644 --- a/tests/rest/client/test_models.py +++ b/tests/rest/client/test_models.py @@ -23,7 +23,7 @@ from typing import Literal from pydantic import BaseModel, ValidationError -from synapse.types.rest.client import EmailRequestTokenBody +from synapse.types.rest.client import ClientSecretStr, EmailRequestTokenBody class ThreepidMediumEnumTestCase(stdlib_unittest.TestCase): @@ -48,6 +48,44 @@ class ThreepidMediumEnumTestCase(stdlib_unittest.TestCase): self.Model.model_validate({"medium": 123}) +class ClientSecretStrTestCase(stdlib_unittest.TestCase): + class Model(BaseModel): + client_secret: ClientSecretStr + + def test_accepts_valid_client_secrets(self) -> None: + """Secrets consisting entirely of `[0-9a-zA-Z.=_-]` are accepted.""" + for client_secret in ( + "this.is-a_valid=secret", + "foobar", + "a", + "0123456789", + "a" * 255, + ): + with self.subTest(client_secret=client_secret): + model = self.Model.model_validate({"client_secret": client_secret}) + self.assertEqual(model.client_secret, client_secret) + + def test_rejects_client_secrets_with_invalid_characters(self) -> None: + for client_secret in ( + "foo bar", + "secret!", + "café", + # Little bobby tables + "Robert'; DROP TABLE students;--", + ): + with self.subTest(client_secret=client_secret): + with self.assertRaises(ValidationError): + self.Model.model_validate({"client_secret": client_secret}) + + def test_rejects_empty_client_secret(self) -> None: + with self.assertRaises(ValidationError): + self.Model.model_validate({"client_secret": ""}) + + def test_rejects_overlong_client_secret(self) -> None: + with self.assertRaises(ValidationError): + self.Model.model_validate({"client_secret": "a" * 256}) + + class EmailRequestTokenBodyTestCase(stdlib_unittest.TestCase): base_request = { "client_secret": "hunter2", From 118c631dbdc9d709cdb59557700764ffa6b466f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:32:10 +0200 Subject: [PATCH 155/158] Bump dtolnay/rust-toolchain from e97e2d8cc328f1b50210efc529dca0028893a2d9 to 6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 (#20123) Bumps [dtolnay/rust-toolchain](https://github.com/dtolnay/rust-toolchain) from e97e2d8cc328f1b50210efc529dca0028893a2d9 to 6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772.
Commits

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | dtolnay/rust-toolchain | [> 1.66.0] |
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/complement_tests.yml | 2 +- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 6 +++--- .github/workflows/tests.yml | 22 +++++++++++----------- .github/workflows/twisted_trunk.yml | 6 +++--- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index 95d8c323ef..220ea57e1a 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -50,7 +50,7 @@ jobs: shell: bash - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index cc482a89b5..31ac11bfcd 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -23,7 +23,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} components: clippy, rustfmt diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 6dd277be9a..fe324e3dbb 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -80,7 +80,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -154,7 +154,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f17b03c0f3..90b7f9021a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -108,7 +108,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -174,7 +174,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -237,7 +237,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: components: clippy toolchain: ${{ env.RUST_VERSION }} @@ -256,7 +256,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_NIGHTLY_VERSION }} components: clippy @@ -274,7 +274,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -310,7 +310,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: # We use nightly so that we can use some unstable options that we use in # `.rustfmt.toml`. @@ -429,7 +429,7 @@ jobs: postgres:${{ matrix.job.postgres-version }} - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -473,7 +473,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -588,7 +588,7 @@ jobs: run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -729,7 +729,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -750,7 +750,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_NIGHTLY_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 6cbb61be6b..9836f743c7 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -45,7 +45,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -73,7 +73,7 @@ jobs: - run: sudo apt-get -qq install xmlsec1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -118,7 +118,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 From 20d203a415df8c0def4a93b7ef27a4d07d698960 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 7 Sep 2026 13:28:00 +0300 Subject: [PATCH 156/158] Fix missing validation for `membership` field after `make_*` requests (#20189) It was specced as a part of https://github.com/matrix-org/matrix-spec/pull/2284 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20189.bugfix | 1 + synapse/handlers/federation.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/20189.bugfix diff --git a/changelog.d/20189.bugfix b/changelog.d/20189.bugfix new file mode 100644 index 0000000000..af3cba350e --- /dev/null +++ b/changelog.d/20189.bugfix @@ -0,0 +1 @@ +Fix missing validation of `membership` when making `make_*` requests over federation. Contributed by @tulir @ Beeper. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 7c5534f403..c67974c575 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1268,6 +1268,7 @@ class FederationHandler: assert event.sender == user_id assert event.state_key == user_id assert event.room_id == room_id + assert event.content.get("membership") == membership return origin, event, room_version async def on_make_leave_request( From 6dbaf300afd90c5500e1c28bca900ca7c83d3025 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:39:08 +0000 Subject: [PATCH 157/158] Bump http from 1.4.2 to 1.5.0 (#20121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [http](https://github.com/hyperium/http) from 1.4.2 to 1.5.0.
Release notes

Sourced from http's releases.

v1.5.0

What's Changed

New Contributors

Full Changelog: https://github.com/hyperium/http/compare/v1.4.2...v1.5.0

Changelog

Sourced from http's changelog.

1.5.0 (July 29, 2026)

  • Add Method::QUERY constant for the new QUERY method defined in RFC 10008.
  • Fix uri::Builder::path_and_query() to allow empty strings to mean no path.
  • Fix uri::PathAndQuery parsing to enforce URI max length.
Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2215b23bab..6062122c15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -416,9 +416,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", From 07bfe4c2a1c17c842b36d7f0cff77259ae448ccd Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Mon, 7 Sep 2026 16:44:28 +0200 Subject: [PATCH 158/158] Remove the unstable `org.matrix.msc3202.device_id` masquerading alias (#20192) Part of: https://github.com/element-hq/synapse/issues/19415 Follow-up to #19033 TLDR: date to drop support for prefixed `org.matrix.msc3202.device_id` parameter was 2026-01-01. This PR removes the prefixed param. Synapse has accepted the stable `device_id` parameter since v1.141.0 (#19033). Before: ``` GET /_matrix/client/v3/account/whoami?user_id=@alice:test&org.matrix.msc3202.device_id=DEVICEID # appservice token 200 {"user_id": "@alice:test", "is_guest": false, "device_id": "DEVICEID"} ``` After: ``` GET /_matrix/client/v3/account/whoami?user_id=@alice:test&org.matrix.msc3202.device_id=DEVICEID # appservice token 200 {"user_id": "@alice:test", "is_guest": false} ``` ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20192.removal | 1 + synapse/api/auth/base.py | 7 +------ 2 files changed, 2 insertions(+), 6 deletions(-) create mode 100644 changelog.d/20192.removal diff --git a/changelog.d/20192.removal b/changelog.d/20192.removal new file mode 100644 index 0000000000..b1de7efa95 --- /dev/null +++ b/changelog.d/20192.removal @@ -0,0 +1 @@ +Remove support for the unstable `org.matrix.msc3202.device_id` query parameter for application service device masquerading. diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index 0b10640b50..7b8214fabb 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -317,9 +317,6 @@ class BaseAuth: - The returned device ID, if present, has been checked to be a valid device ID for the returned user ID. """ - # TODO: We can drop unstable support after 2026-01-01 (couple months after stable support) - UNSTABLE_DEVICE_ID_ARG_NAME = b"org.matrix.msc3202.device_id" - app_service = self.store.get_app_service_by_token(access_token) if app_service is None: return None @@ -340,9 +337,7 @@ class BaseAuth: else: effective_user_id = app_service.sender - effective_device_id_args = request.args.get( - b"device_id", request.args.get(UNSTABLE_DEVICE_ID_ARG_NAME) - ) + effective_device_id_args = request.args.get(b"device_id") if effective_device_id_args: effective_device_id = effective_device_id_args[0].decode("utf8") # We only just set this so it can't be None!