From 6100f6e4f7fb0c72f1ae2802683ebc811c0e3a77 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 1 May 2026 11:42:00 -0500 Subject: [PATCH 01/42] Backfill from nearby points past pagination token (#19611) The juicy details and explanation are in the diff itself. Split out from https://github.com/element-hq/synapse/pull/18873 in order to fix paginating from [MSC3871](https://github.com/matrix-org/matrix-spec-proposals/pull/3871) gap tokens actually backfilling history. To be clear, this is a good change to make outside of the [MSC3871](https://github.com/matrix-org/matrix-spec-proposals/pull/3871) use case. For example (as the new Complement test shows), fixes a problem where if you try to paginate `/messages` from tokens returned by `/context`, we could fail to backfill anything new and hide away history. Also fixes https://github.com/matrix-org/complement/pull/853 --- changelog.d/19611.bugfix | 1 + complement/tests/room_messages_test.go | 394 ++++++++++++++++++ synapse/handlers/federation.py | 20 +- .../databases/main/event_federation.py | 87 +++- tests/handlers/test_federation.py | 117 ------ 5 files changed, 475 insertions(+), 144 deletions(-) create mode 100644 changelog.d/19611.bugfix create mode 100644 complement/tests/room_messages_test.go diff --git a/changelog.d/19611.bugfix b/changelog.d/19611.bugfix new file mode 100644 index 0000000000..4952fd00db --- /dev/null +++ b/changelog.d/19611.bugfix @@ -0,0 +1 @@ +Fix Synapse not backfilling new history when attempting to use a pagination token near a backward extremity. diff --git a/complement/tests/room_messages_test.go b/complement/tests/room_messages_test.go new file mode 100644 index 0000000000..d491d1db85 --- /dev/null +++ b/complement/tests/room_messages_test.go @@ -0,0 +1,394 @@ +// This file is licensed under the Affero General Public License (AGPL) version 3. +// +// Copyright (C) 2026 Element Creations Ltd +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// See the GNU Affero General Public License for more details: +// . + +package synapse_tests + +import ( + "encoding/json" + "fmt" + "net/url" + "slices" + "strings" + "testing" + + "github.com/matrix-org/complement" + "github.com/matrix-org/complement/b" + "github.com/matrix-org/complement/client" + "github.com/matrix-org/complement/helpers" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/tidwall/gjson" +) + +func TestMessagesOverFederation(t *testing.T) { + deployment := complement.Deploy(t, 2) + defer deployment.Destroy(t) + + alice := deployment.Register(t, "hs1", helpers.RegistrationOpts{ + LocalpartSuffix: "alice", + }) + bob := deployment.Register(t, "hs2", helpers.RegistrationOpts{ + LocalpartSuffix: "bob", + }) + + // The typical convention to find backfill points is from the backward extremities in + // the DAG. Backward extremities are the oldest events we know of in the room but we + // only know of them because some other event referenced them by prev_event and aren't + // known to the homeserver yet (meaning we don't know their depth specifically). So we + // can only do approximate depth comparisons (use the depth of the known events + // they're connected to). And we don't know if those backward extremities point to a + // long chain/fork of history that could stretch back far enough to be visible. + // + // This means a naive homeserver implementation that looks for backward extremities <= + // depth of the `/messages?dir=b&from=xxx` token may overlook a backfill point that could + // reveal more history in the window the user is currently paginating in. + // + // This could be a near miss as this test is specifically stressing or a more deep miss + // as the backward extremity could reveal an entire fork of history that stretches + // back far enough to be visible. + // + // In Synapse, we consider "nearby" as anything within range of the `limit` specified + // in `/messages?dir=b&from=xxx&limit=xxx`. + // + // This test lives in our in-repo Complement tests for Synapse because the Matrix spec + // doesn't have any rules for how a homeserver should backfill. Practically speaking, + // homeservers that don't do anything for this problem will just hide messages from + // clients. This underscores the fact why it's necessary for homeservers to indicate that + // there is a gap (using MSC3871) at the very least. + // + // -------------------------------------------------- + // + // Even with MSC3871 gaps, the tested behavior here is necessary as the gap prev/next + // tokens point before/after the event (remember: tokens are positions between + // events), so if you use `/messages?dir=b&from=`, we can't + // rely on naive depth comparison. MSC3871 Complement tests will also exercise this. + // Example: + // + // t0 t1 t2 t3 t4 + // [A] <--- [B] <--- [C] <--- [bob join 4] + // + // When Bob calls `/messages?dir=b&backfill=false`, he sees a gap (`{ event_id: "bob + // join 4", prev_pagination_token: "t3", next_pagination_token: "t4" }`) and tries to + // fill it in with `/messages?dir=b&from=t3&limit=10&backfill=true`. To find backfill + // points, Synapse will compare `t3` with the backward extremity at an approximate + // depth of 4. Which is why we take `t3`, add the `limit=10` and then do the + // comparison (find any backfill points with an approximate depth <= 13). + t.Run("Backfill from nearby backward extremities past token", func(t *testing.T) { + // Alice creates the room + roomID := alice.MustCreateRoom(t, map[string]interface{}{ + // The `public_chat` preset includes `history_visibility: "shared"` ("Previous + // events are always accessible to newly joined members. All events in the + // room are accessible, even those sent when the member was not a part of the + // room."), which is what we want to test. + "preset": "public_chat", + }) + + // Keep track of the order + eventIDs := make([]string, 0) + // Map from event_id to event info + eventMap := make(map[string]EventInfo) + + // Send some message history into the room + numberOfMessagesToSend := 3 + messageDrafts := make([]MessageDraft, 0, numberOfMessagesToSend) + for i := 0; i < numberOfMessagesToSend; i++ { + messageDrafts = append( + messageDrafts, + MessageDraft{alice, fmt.Sprintf("message history %d", i+1)}, + ) + } + sendAndTrackMessages(t, roomID, messageDrafts, &eventIDs, &eventMap) + + // Bob joins the room + bob.MustJoinRoom(t, roomID, []spec.ServerName{ + deployment.GetFullyQualifiedHomeserverName(t, "hs1"), + }) + bobJoinEventID := getStateID(t, bob, roomID, "m.room.member", bob.UserID) + + // Make it easy to cross-reference the events being talked about in the logs + for eventIndex, eventID := range eventIDs { + t.Logf("Message %d -> event_id=%s", eventIndex, eventID) + } + + // Use a `/context` request to get a pagination token just before Bob's join event + // (remember: tokens are positions between events) + // + // Usually a client would just use `/messages?dir=b` to start getting history + // after joining but this is valid as well. To illustrate a more real example of + // this, someone can use `/timestamp_to_event` to jump back in history and + // `/context` to start paginating history. + contextRes := bob.MustDo( + t, + "GET", + []string{"_matrix", "client", "v3", "rooms", roomID, "context", bobJoinEventID}, + client.WithContentType("application/json"), + client.WithQueries(url.Values{ + "limit": []string{"0"}, + }), + ) + contextResResBody := client.ParseJSON(t, contextRes) + // > `start`: A token that can be used to paginate backwards with. + // > - https://spec.matrix.org/v1.17/client-server-api/#get_matrixclientv3roomsroomidcontexteventid + paginationToken := client.GetJSONFieldStr(t, contextResResBody, "start") + + // Paginate backwards from the join event + messagesRes := bob.MustDo( + t, + "GET", + []string{"_matrix", "client", "v3", "rooms", roomID, "messages"}, + client.WithContentType("application/json"), + client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + "from": []string{paginationToken}, + }), + ) + messagesResBody := client.ParseJSON(t, messagesRes) + + // Since `dir=b`, these will be in reverse chronological order + actualEventIDsFromRequest := extractEventIDsFromMessagesResponse(t, messagesResBody) + + // Put them in chronological order to match the expected list + chronologicalActualEventIds := slices.Clone(actualEventIDsFromRequest) + slices.Reverse(chronologicalActualEventIds) + + // Assert timeline order + assertEventsInOrder(t, chronologicalActualEventIds, eventIDs) + }) + + // TODO: Backfill test to make sure we backfill from forks when viewing history (see + // docstring above). + // + // 1. Alice (hs1, engineered homeserver) creates a room with events A, B + // 1. Bob (hs2) joins the room + // 1. Bob leaves the room + // 1. Alice creates a fork from A with some history (1, 2, 3) and connects it back with a new event C + // 1. Bob joins back + // 1. Bob paginates `/messages?dir=b&from=` + // 1. Ensure Bob sees events: B, 2, 1, A + // + // 1 <--- 2 <----- 3 + // / \ + // A <------- B ▲ <--- C <-- D + // | + // Paginate backwards from this point + // t.Run("Backfill from nearby backward extremities past token (fork)", func(t *testing.T) { +} + +// These utilities match what we're using in the Complement repo (see +// `matrix-org/complement` -> `tests/csapi/room_messages_test.go`) + +type MessageDraft struct { + Sender *client.CSAPI + Message string +} + +type EventInfo struct { + MessageDraft MessageDraft + EventID string +} + +func sendMessageDrafts( + t *testing.T, + roomID string, + messageDrafts []MessageDraft, +) []string { + t.Helper() + + eventIDs := make([]string, len(messageDrafts)) + for messageDraftIndex, messageDraft := range messageDrafts { + eventID := messageDraft.Sender.SendEventSynced(t, roomID, b.Event{ + Type: "m.room.message", + Content: map[string]interface{}{ + "msgtype": "m.text", + "body": messageDraft.Message, + }, + }) + eventIDs[messageDraftIndex] = eventID + } + + return eventIDs +} + +// sendAndTrackMessages sends the given message drafts to the room, keeping track of the +// new events in the list of `eventIDs` and `eventMap`. Returns the list of new event +// IDs that were sent. +func sendAndTrackMessages( + t *testing.T, + roomID string, + messageDrafts []MessageDraft, + eventIDs *[]string, + eventMap *map[string]EventInfo, +) []string { + t.Helper() + + newEventIDs := sendMessageDrafts(t, roomID, messageDrafts) + + *eventIDs = append(*eventIDs, newEventIDs...) + for i, eventID := range newEventIDs { + (*eventMap)[eventID] = EventInfo{ + MessageDraft: messageDrafts[i], + EventID: eventID, + } + } + + return newEventIDs +} + +// extractEventIDsFromMessagesResponse extracts the event IDs from the given +// `/messages` response body. +func extractEventIDsFromMessagesResponse( + t *testing.T, + messagesResBody json.RawMessage, +) []string { + t.Helper() + + wantKey := "chunk" + keyRes := gjson.GetBytes(messagesResBody, wantKey) + if !keyRes.Exists() { + t.Fatalf("extractEventIDsFromMessagesResponse: missing key '%s'", wantKey) + } + if !keyRes.IsArray() { + t.Fatalf( + "extractEventIDsFromMessagesResponse: key '%s' is not an array (was %s)", + wantKey, + keyRes.Type, + ) + } + + var eventIDs []string + actualEvents := keyRes.Array() + for _, event := range actualEvents { + eventIDs = append(eventIDs, event.Get("event_id").Str) + } + + return eventIDs +} + +func filterEventIDs(t *testing.T, actualEventIDs []string, expectedEventIDs []string) []string { + t.Helper() + + relevantActualEventIDs := make([]string, 0, len(expectedEventIDs)) + for _, eventID := range actualEventIDs { + if slices.Contains(expectedEventIDs, eventID) { + relevantActualEventIDs = append(relevantActualEventIDs, eventID) + } + } + + return relevantActualEventIDs +} + +// assertEventsInOrder asserts all `actualEventIDs` are present and in order according +// to `expectedEventIDs`. Other unrelated events can be in between. +func assertEventsInOrder(t *testing.T, actualEventIDs []string, expectedEventIDs []string) { + t.Helper() + + relevantActualEventIDs := filterEventIDs(t, actualEventIDs, expectedEventIDs) + + if len(relevantActualEventIDs) != len(expectedEventIDs) { + t.Fatalf( + "expected %d events in timeline (got %d relevant events filtered down from %d events)\n%s", + len(expectedEventIDs), + len(relevantActualEventIDs), + len(actualEventIDs), + generateEventOrderDiffString(relevantActualEventIDs, expectedEventIDs), + ) + } + + for i, eventID := range relevantActualEventIDs { + if eventID != expectedEventIDs[i] { + t.Fatalf( + "expected event ID %s (got %s) at index %d\n%s", + expectedEventIDs[i], + eventID, + i, + generateEventOrderDiffString(relevantActualEventIDs, expectedEventIDs), + ) + } + } +} + +func generateEventOrderDiffString(actualEventIDs []string, expectedEventIDs []string) string { + expectedLines := make([]string, len(expectedEventIDs)) + for i, expectedEventID := range expectedEventIDs { + isExpectedInActual := slices.Contains(actualEventIDs, expectedEventID) + isMissingIndicatorString := " " + if !isExpectedInActual { + isMissingIndicatorString = "?" + } + + expectedLines[i] = fmt.Sprintf("%2d: %s %s", i, isMissingIndicatorString, expectedEventID) + } + expectedDiffString := strings.Join(expectedLines, "\n") + + actualLines := make([]string, len(actualEventIDs)) + for actualEventIndex, actualEventID := range actualEventIDs { + isActualInExpected := slices.Contains(expectedEventIDs, actualEventID) + isActualInExpectedIndicatorString := " " + if isActualInExpected { + isActualInExpectedIndicatorString = "+" + } + + expectedIndex := slices.Index(expectedEventIDs, actualEventID) + expectedIndexString := "" + if actualEventIndex != expectedIndex { + expectedDirectionString := "⬆️" + if expectedIndex > actualEventIndex { + expectedDirectionString = "⬇️" + } + + expectedIndexString = fmt.Sprintf( + " (expected index %d %s)", + expectedIndex, + expectedDirectionString, + ) + } + + actualLines[actualEventIndex] = fmt.Sprintf("%2d: %s %s%s", + actualEventIndex, isActualInExpectedIndicatorString, actualEventID, expectedIndexString, + ) + } + actualDiffString := strings.Join(actualLines, "\n") + + return fmt.Sprintf( + "Actual events ('+' = found expected items):\n%s\nExpected events ('?' = missing expected items):\n%s", + actualDiffString, + expectedDiffString, + ) +} + +func getStateID( + t *testing.T, + c *client.CSAPI, + roomID string, + stateType string, + stateKey string, +) string { + t.Helper() + + stateRes := c.MustDo(t, "GET", []string{"_matrix", "client", "v3", "rooms", roomID, "state"}) + stateResBody := client.ParseJSON(t, stateRes) + eventJSON := gjson.ParseBytes(stateResBody) + if !eventJSON.IsArray() { + t.Fatalf("expected array of state events but found %s", eventJSON.Type) + } + + events := eventJSON.Array() + + for _, event := range events { + if event.Get("type").Str == stateType && event.Get("state_key").Str == stateKey { + return event.Get("event_id").Str + } + } + + t.Fatalf("Unable to find state event for (%s, %s). Room state: %s", stateType, stateKey, events) + return "" +} diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b3444dd2ef..166a02d7c7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -105,6 +105,12 @@ backfill_processing_before_timer = Histogram( ) +NUMBER_OF_EVENTS_TO_BACKFILL = 100 +""" +The number of events we try to backfill from other servers in a single request. +""" + + # TODO: We can refactor this away now that there is only one backfill point again class _BackfillPointType(Enum): # a regular backwards extremity (ie, an event which we don't yet have, but which @@ -255,7 +261,9 @@ class FederationHandler: _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY) for event_id, depth in await self.store.get_backfill_points_in_room( room_id=room_id, - current_depth=current_depth, + # Per the docstring, it's best to pad the `current_depth` by the + # number of messages you plan to backfill from these points. + nearby_depth=current_depth + NUMBER_OF_EVENTS_TO_BACKFILL, # We only need to end up with 5 extremities combined with the # insertion event extremities to make the `/backfill` request # but fetch an order of magnitude more to make sure there is @@ -299,12 +307,13 @@ class FederationHandler: # likely not to return anything relevant so we backfill in the background. The # only way, this could return something relevant is if we discover a new branch # of history that extends all the way back to where we are currently paginating - # and it's within the 100 events that are returned from `/backfill`. + # and it's within the `NUMBER_OF_EVENTS_TO_BACKFILL` events that are returned + # from `/backfill`. if not sorted_backfill_points and current_depth != MAX_DEPTH: # Check that we actually have later backfill points, if not just return. have_later_backfill_points = await self.store.get_backfill_points_in_room( room_id=room_id, - current_depth=MAX_DEPTH, + nearby_depth=MAX_DEPTH, limit=1, ) if not have_later_backfill_points: @@ -464,7 +473,10 @@ class FederationHandler: try: await self._federation_event_handler.backfill( - dom, room_id, limit=100, extremities=extremities_to_request + dom, + room_id, + limit=NUMBER_OF_EVENTS_TO_BACKFILL, + extremities=extremities_to_request, ) # If this succeeded then we probably already have the # appropriate stuff. diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 415926eb0a..d84c58dcf8 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1199,34 +1199,72 @@ class EventFederationWorkerStore( async def get_backfill_points_in_room( self, room_id: str, - current_depth: int, + nearby_depth: int, limit: int, ) -> list[tuple[str, int]]: """ Get the backward extremities to backfill from in the room along with the approximate depth. - Only returns events that are at a depth lower than or - equal to the `current_depth`. Sorted by depth, highest to lowest (descending) - so the closest events to the `current_depth` are first in the list. + Only returns events that are at a depth lower than or equal to the `nearby_depth`. + Sorted by depth, highest to lowest (descending) so the closest events to the + `nearby_depth` are first in the list. - We ignore extremities that are newer than the user's current scroll position - (ie, those with depth greater than `current_depth`) as: - 1. we don't really care about getting events that have happened - after our current position; and - 2. by the nature of paginating and scrolling back, we have likely - previously tried and failed to backfill from that extremity, so - to avoid getting "stuck" requesting the same backfill repeatedly - we drop those extremities. + ### Why `nearby_depth`? + + We find backfill points from the backward extremities in the DAG. Backward + extremities are the oldest events we know of in the room but we only know of + them because some other event referenced them by prev_event and aren't persisted + in our database yet (meaning we don't know their depth specifically). So we can + only do approximate depth comparisons (use the depth of the known events they're + connected to). And we don't know if those backward extremities point to a long + chain/fork of history that could stretch back far enough to be visible. + + This means a naive homeserver implementation that looks for backward extremities <= + depth of the `/messages?dir=b&from=xxx` token may overlook a backfill point that could + reveal more history in the window the user is currently paginating in. + + We consider "nearby" as anything within range of the number of events you plan + to backfill from the given backfill point. This is a good heuristic as since we + plan to backfill N events, the chain of events from a backfill point could + extend back into the visible window. + + Example: + + - Your pagination token represents a scroll position at a depth of `100`. + - We have a backfill point at an approximate depth of `125` + - You plan to backfill `50` events from that backfill point. + + When we pad the token `depth` with the number of messages we plan to backfill, + `100` + `50` = `150`, we find the backfill point at `125` (because <= `150`, our + `nearby_depth`), backfill `50` events to a depth of `75` in the timeline + (exposing new events that we can return `100` -> `75`). + + When we don't pad our token `depth`, `100` is lower than any of the backfill + points so we don't pick any and miss out on backfilling any events. Without + something like MSC3871 to indicate gaps in the timeline, clients will most + likely never know they are missing any events and never try to paginate again. + + Generally though, we ignore extremities that are newer than the user's current + scroll position (ie, those with depth greater than `nearby_depth`) as: + 1. we don't really care about getting events that have happened after our + current position; and + 2. by the nature of paginating and scrolling back, we have likely previously + tried and failed to backfill from that extremity, so to avoid getting + "stuck" requesting the same backfill repeatedly we drop those + extremities. Although we also have `event_failed_pull_attempts` nowadays + to backoff as well. Args: room_id: Room where we want to find the oldest events - current_depth: The depth at the user's current scrollback position + nearby_depth: Typically, this is depth at the user's current scrollback + position + the number of events you plan to backfill from these backfill + points. limit: The max number of backfill points to return Returns: List of (event_id, depth) tuples. Sorted by depth, highest to lowest - (descending) so the closest events to the `current_depth` are first + (descending) so the closest events to the `nearby_depth` are first in the list. """ @@ -1234,12 +1272,12 @@ class EventFederationWorkerStore( txn: LoggingTransaction, room_id: str ) -> list[tuple[str, int]]: # Assemble a tuple lookup of event_id -> depth for the oldest events - # we know of in the room. Backwards extremeties are the oldest + # we know of in the room. Backwards extremities are the oldest # events we know of in the room but we only know of them because # some other event referenced them by prev_event and aren't # persisted in our database yet (meaning we don't know their depth # specifically). So we need to look for the approximate depth from - # the events connected to the current backwards extremeties. + # the events connected to the current backwards extremities. if isinstance(self.database_engine, PostgresEngine): least_function = "LEAST" @@ -1259,7 +1297,7 @@ class EventFederationWorkerStore( ON edge.event_id = event.event_id /** * We find the "oldest" events in the room by looking for - * events connected to backwards extremeties (oldest events + * events connected to backwards extremities (oldest events * in the room that we know of so far). */ INNER JOIN event_backward_extremities AS backward_extrem @@ -1285,16 +1323,19 @@ class EventFederationWorkerStore( AND edge.is_state is FALSE /** * We only want backwards extremities that are older than or at - * the same position of the given `current_depth` (where older + * the same position of the given `nearby_depth` (where older * means less than the given depth) because we're looking backwards - * from the `current_depth` when backfilling. + * from the `nearby_depth` when backfilling. * - * current_depth (ignore events that come after this, ignore 2-4) + * Keep in mind that `event.depth` is an approximate depth of the + * backward extremity itself. + * + * nearby_depth (ignore events that come after this, ignore 2-4) * | * ▼ * [0]<--[1]<--[2]<--[3]<--[4] */ - AND event.depth <= ? /* current_depth */ + AND event.depth <= ? /* nearby_depth */ /** * Exponential back-off (up to the upper bound) so we don't retry the * same backfill point over and over. ex. 2hr, 4hr, 8hr, 16hr, etc. @@ -1312,7 +1353,7 @@ class EventFederationWorkerStore( ) ) /** - * Sort from highest (closest to the `current_depth`) to the lowest depth + * Sort from highest (closest to the `nearby_depth`) to the lowest depth * because the closest are most relevant to backfill from first. * Then tie-break on alphabetical order of the event_ids so we get a * consistent ordering which is nice when asserting things in tests. @@ -1325,7 +1366,7 @@ class EventFederationWorkerStore( sql, ( room_id, - current_depth, + nearby_depth, self.clock.time_msec(), BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS, BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index e4a41cf1ae..dde1785854 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -43,10 +43,8 @@ from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.storage.databases.main.events_worker import EventCacheEntry from synapse.util.clock import Clock -from synapse.util.events import generate_fake_event_id from tests import unittest -from tests.test_utils import event_injection logger = logging.getLogger(__name__) @@ -213,121 +211,6 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): self.assertEqual(sg, sg2) - def test_backfill_with_many_backward_extremities(self) -> None: - """ - Check that we can backfill with many backward extremities. - The goal is to make sure that when we only use a portion - of backwards extremities(the magic number is more than 5), - no errors are thrown. - - Regression test, see https://github.com/matrix-org/synapse/pull/11027 - """ - # create the room - user_id = self.register_user("kermit", "test") - tok = self.login("kermit", "test") - - room_id = self.helper.create_room_as(room_creator=user_id, tok=tok) - room_version = self.get_success(self.store.get_room_version(room_id)) - - # we need a user on the remote server to be a member, so that we can send - # extremity-causing events. - remote_server_user_id = f"@user:{self.OTHER_SERVER_NAME}" - self.get_success( - event_injection.inject_member_event( - self.hs, room_id, remote_server_user_id, "join" - ) - ) - - send_result = self.helper.send(room_id, "first message", tok=tok) - ev1 = self.get_success( - self.store.get_event(send_result["event_id"], allow_none=False) - ) - current_state = self.get_success( - self.store.get_events_as_list( - ( - self.get_success(self.store.get_partial_current_state_ids(room_id)) - ).values() - ) - ) - - # Create "many" backward extremities. The magic number we're trying to - # create more than is 5 which corresponds to the number of backward - # extremities we slice off in `_maybe_backfill_inner` - federation_event_handler = self.hs.get_federation_event_handler() - auth_events = [ - ev - for ev in current_state - if (ev.type, ev.state_key) - in {("m.room.create", ""), ("m.room.member", remote_server_user_id)} - ] - for _ in range(8): - event = make_event_from_dict( - self.add_hashes_and_signatures_from_other_server( - { - "origin_server_ts": 1, - "type": "m.room.message", - "content": { - "msgtype": "m.text", - "body": "message connected to fake event", - }, - "room_id": room_id, - "sender": remote_server_user_id, - "prev_events": [ - ev1.event_id, - # We're creating an backward extremity each time thanks - # to this fake event - generate_fake_event_id(), - ], - "auth_events": [ev.event_id for ev in auth_events], - "depth": ev1.depth + 1, - }, - room_version, - ), - room_version, - ) - - # we poke this directly into _process_received_pdu, to avoid the - # federation handler wanting to backfill the fake event. - state_handler = self.hs.get_state_handler() - context = self.get_success( - state_handler.compute_event_context( - event, - state_ids_before_event={ - (e.type, e.state_key): e.event_id for e in current_state - }, - partial_state=False, - ) - ) - self.get_success( - federation_event_handler._process_received_pdu( - self.OTHER_SERVER_NAME, - event, - context, - ) - ) - - # we should now have 8 backwards extremities. - backwards_extremities = self.get_success( - self.store.db_pool.simple_select_list( - "event_backward_extremities", - keyvalues={"room_id": room_id}, - retcols=["event_id"], - ) - ) - self.assertEqual(len(backwards_extremities), 8) - - current_depth = 1 - limit = 100 - - # Make sure backfill still works - self.get_success( - self.hs.get_federation_handler().maybe_backfill( - room_id, - current_depth, - limit, - ) - ) - def test_backfill_ignores_known_events(self) -> None: """ Tests that events that we already know about are ignored when backfilling. From 3f58bc50dfba5768ee43ce48c5e74c25ba0b078a Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 5 May 2026 08:40:17 -0500 Subject: [PATCH 02/42] fix: Cap `WorkerLock` timeout intervals to 60 seconds (#19394) Fixes the symptoms of https://github.com/element-hq/synapse/issues/19315 / https://github.com/element-hq/synapse/issues/19588 but not the underlying reason causing the number to grow so large in the first place. ``` ValueError: Exceeds the limit (4300 digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit ``` Copied from the original pull request on [Famedly's Synapse repo](https://github.com/famedly/synapse/pull/221) (with some edits): Basing the time interval around a 5 seconds leaves a big window of waiting especially as this window is doubled each retry, when another worker could be making progress but can not. Right now, the retry interval in seconds looks like `[0.2, 5, 10, 20, 40, 80, 160, 320, (continues to double)]` after which logging should start about excessive times and (relatively quickly) end up with an extremely large retry interval with an unrealistic expectation past the heat death of the universe. 1 year in seconds = 31,536,000. With this change, retry intervals in seconds should look more like: ``` [ 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 60, < never goes higher than this ] ``` Logging about excessive wait times will start at 10 minutes.
Previous breakdown when we were using 15 minutes ``` [ 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 102.4, # 1.7 minutes 204.8, # 3.41 minutes 409.6, # 6.83 minutes 819.2, # 13.65 minutes < logging about excessive times will start here, 13th iteration 900, # 15 minutes < never goes higher than this ] ```
Further suggested work in this area could be to define the cap, the retry interval starting point and the multiplier depending on how frequently this lock should be checked. See data below for reasons why. Increasing the jitter range may also be a good idea --------- Co-authored-by: Eric Eastwood --- changelog.d/19394.bugfix | 1 + synapse/handlers/worker_lock.py | 117 +++++++++++++++++++------- tests/handlers/test_worker_lock.py | 130 +++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 32 deletions(-) create mode 100644 changelog.d/19394.bugfix diff --git a/changelog.d/19394.bugfix b/changelog.d/19394.bugfix new file mode 100644 index 0000000000..4ca92cfb32 --- /dev/null +++ b/changelog.d/19394.bugfix @@ -0,0 +1 @@ +Capped the `WorkerLock` time out interval to a maximum of 60 seconds to prevent dealing with excessively long numbers. Contributed by Famedly. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 1537a18cc0..51be3b5084 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -54,6 +54,9 @@ logger = logging.getLogger(__name__) # will not disappear under our feet as long as we don't delete the room. NEW_EVENT_DURING_PURGE_LOCK_NAME = "new_event_during_purge_lock" +WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=60) +WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) + class WorkerLocksHandler: """A class for waiting on taking out locks, rather than using the storage @@ -206,9 +209,10 @@ class WaitingLock: lock_name: str lock_key: str write: bool | None + start_ts_ms: int = 0 deferred: "defer.Deferred[None]" = attr.Factory(defer.Deferred) _inner_lock: Lock | None = None - _retry_interval: float = 0.1 + _timeout_interval: float = 0.1 _lock_span: "opentracing.Scope" = attr.Factory( lambda: start_active_span("WaitingLock.lock") ) @@ -220,6 +224,7 @@ class WaitingLock: self.deferred.callback(None) async def __aenter__(self) -> None: + self.start_ts_ms = self.clock.time_msec() self._lock_span.__enter__() with start_active_span("WaitingLock.waiting_for_lock"): @@ -240,19 +245,44 @@ class WaitingLock: break try: - # Wait until the we get notified the lock might have been + # Wait until the notification that the lock might have been # released (by the deferred being resolved). We also - # periodically wake up in case the lock was released but we + # periodically wake up in case the lock was released, but we # weren't notified. with PreserveLoggingContext(): - timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=timeout, + timeout=self._timeout_interval, clock=self.clock, ) - except Exception: - pass + except defer.TimeoutError: + # Only increment the timeout value if this was an actual timeout + # (defer.TimeoutError) + self._increment_timeout_interval() + + now_ms = self.clock.time_msec() + time_spent_trying_to_lock = Duration( + milliseconds=now_ms - self.start_ts_ms + ) + if ( + time_spent_trying_to_lock.as_millis() + > WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION.as_millis() + ): + logger.warning( + "(WaitingLock (%s, %s)) Time spent waiting to acquire lock " + "is getting excessive: %ss. There may be a deadlock.", + self.lock_name, + self.lock_key, + time_spent_trying_to_lock.as_secs(), + ) + + except Exception as e: + logger.warning( + "Caught an exception while waiting on WaitingLock(lock_name=%s, lock_key=%s): %r", + self.lock_name, + self.lock_key, + e, + ) return await self._inner_lock.__aenter__() @@ -273,15 +303,14 @@ class WaitingLock: return r - def _get_next_retry_interval(self) -> float: - next = self._retry_interval - self._retry_interval = max(5, next * 2) - if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations - logger.warning( - "Lock timeout is getting excessive: %ss. There may be a deadlock.", - self._retry_interval, - ) - return next * random.uniform(0.9, 1.1) + def _increment_timeout_interval(self) -> float: + next_interval = self._timeout_interval + next_interval = min(WORKER_LOCK_MAX_RETRY_INTERVAL.as_secs(), next_interval * 2) + + # The jitter value is maintained for the timeout, to help avoid a "thundering + # herd" situation when all locks may time out at the same time. + self._timeout_interval = next_interval * random.uniform(0.9, 1.1) + return self._timeout_interval @attr.s(auto_attribs=True, eq=False) @@ -294,10 +323,11 @@ class WaitingMultiLock: store: LockStore handler: WorkerLocksHandler + start_ts_ms: int = 0 deferred: "defer.Deferred[None]" = attr.Factory(defer.Deferred) _inner_lock_cm: AsyncContextManager | None = None - _retry_interval: float = 0.1 + _timeout_interval: float = 0.1 _lock_span: "opentracing.Scope" = attr.Factory( lambda: start_active_span("WaitingLock.lock") ) @@ -309,6 +339,7 @@ class WaitingMultiLock: self.deferred.callback(None) async def __aenter__(self) -> None: + self.start_ts_ms = self.clock.time_msec() self._lock_span.__enter__() with start_active_span("WaitingLock.waiting_for_lock"): @@ -324,19 +355,42 @@ class WaitingMultiLock: break try: - # Wait until the we get notified the lock might have been + # Wait until the notification that the lock might have been # released (by the deferred being resolved). We also - # periodically wake up in case the lock was released but we + # periodically wake up in case the lock was released, but we # weren't notified. with PreserveLoggingContext(): - timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=timeout, + timeout=self._timeout_interval, clock=self.clock, ) - except Exception: - pass + except defer.TimeoutError: + # Only increment the timeout value if this was an actual timeout + # (defer.TimeoutError) + self._increment_timeout_interval() + + now_ms = self.clock.time_msec() + time_spent_trying_to_lock = Duration( + milliseconds=now_ms - self.start_ts_ms + ) + if ( + time_spent_trying_to_lock.as_millis() + > WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION.as_millis() + ): + logger.warning( + "(WaitingMultiLock (%r)) Time spent waiting to acquire lock " + "is getting excessive: %ss. There may be a deadlock.", + self.lock_names, + time_spent_trying_to_lock.as_secs(), + ) + + except Exception as e: + logger.warning( + "Caught an exception while waiting on WaitingMultiLock(lock_names=%r): %r", + self.lock_names, + e, + ) assert self._inner_lock_cm await self._inner_lock_cm.__aenter__() @@ -360,12 +414,11 @@ class WaitingMultiLock: return r - def _get_next_retry_interval(self) -> float: - next = self._retry_interval - self._retry_interval = max(5, next * 2) - if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations - logger.warning( - "Lock timeout is getting excessive: %ss. There may be a deadlock.", - self._retry_interval, - ) - return next * random.uniform(0.9, 1.1) + def _increment_timeout_interval(self) -> float: + next_interval = self._timeout_interval + next_interval = min(WORKER_LOCK_MAX_RETRY_INTERVAL.as_secs(), next_interval * 2) + + # The jitter value is maintained for the timeout, to help avoid a "thundering + # herd" situation when all locks may time out at the same time. + self._timeout_interval = next_interval * random.uniform(0.9, 1.1) + return self._timeout_interval diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index 61ff51ff92..74201f4151 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -26,7 +26,9 @@ from twisted.internet import defer from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer +from synapse.storage.databases.main.lock import _RENEWAL_INTERVAL from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase @@ -40,6 +42,7 @@ class WorkerLockTestCase(unittest.HomeserverTestCase): self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer ) -> None: self.worker_lock_handler = self.hs.get_worker_locks_handler() + self.store = self.hs.get_datastores().main def test_wait_for_lock_locally(self) -> None: """Test waiting for a lock on a single worker""" @@ -56,6 +59,66 @@ class WorkerLockTestCase(unittest.HomeserverTestCase): self.get_success(d2) self.get_success(lock2.__aexit__(None, None, None)) + def test_timeouts_for_lock_locally(self) -> None: + """ + Test that we regularly retry to reacquire locks. + + This is a regression test to make sure the lock retry time doesn't balloon to a value + so large it can't even be printed reliably anymore. + """ + + # Create and acquire the first lock + lock1 = self.worker_lock_handler.acquire_lock("name", "key") + self.get_success(lock1.__aenter__()) + + # Create and try to acquire the second lock + lock2 = self.worker_lock_handler.acquire_lock("name", "key") + d2 = defer.ensureDeferred(lock2.__aenter__()) + # Make sure we haven't acquired the lock yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Advance time by an hour (some duration that would previously cause our timeout + # to balloon if it weren't constrained). Max back-off (saturate) + # + # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an + # internal background looping call that runs every 30 seconds + # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value + # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # drop if this renewal is not allowed to run, which sours the test. + # self.pump(amount=Duration(hours=1)) + self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) + + # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Release the first lock (`lock1`). The second lock(`lock2`) should be + # automatically acquired by the `pump()` inside `get_success()` + self.get_success(lock1.__aexit__(None, None, None)) + + # We should now have the lock + self.successResultOf(d2) + + def _pump_by( + self, + *, + amount: Duration = Duration(seconds=0), + by: Duration = Duration(seconds=0.1), + ) -> None: + """ + Like `self.pump()` but you can specify the time increment to advance with until + you reach the time amount. + + Unlike `self.pump()`, this doesn't multiply the time at all. + + Args: + amount: The amount of time to advance + by: The time increment in seconds to advance time by until we reach the `amount` + """ + end_time_s = self.reactor.seconds() + amount.as_secs() + + while self.reactor.seconds() < end_time_s: + self.reactor.advance(by.as_secs()) + def test_lock_contention(self) -> None: """Test lock contention when a lot of locks wait on a single worker""" nb_locks_to_test = 500 @@ -124,3 +187,70 @@ class WorkerLockWorkersTestCase(BaseMultiWorkerStreamTestCase): self.get_success(d2) self.get_success(lock2.__aexit__(None, None, None)) + + def test_timeouts_for_lock_worker(self) -> None: + """ + Test that we regularly retry to reacquire locks. + + This is a regression test to make sure the lock retry time doesn't balloon to a value + so large it can't even be printed reliably anymore. + """ + worker = self.make_worker_hs( + "synapse.app.generic_worker", + extra_config={ + "redis": {"enabled": True}, + }, + ) + worker_lock_handler = worker.get_worker_locks_handler() + + # Create and acquire the first lock on the main process + lock1 = self.main_worker_lock_handler.acquire_lock("name", "key") + self.get_success(lock1.__aenter__()) + + # Create and try to acquire the second lock on the worker + lock2 = worker_lock_handler.acquire_lock("name", "key") + d2 = defer.ensureDeferred(lock2.__aenter__()) + # Make sure we haven't acquired the lock yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Advance time by an hour (some duration that would previously cause our timeout + # to balloon if it weren't constrained). Max back-off (saturate) + # + # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an + # internal background looping call that runs every 30 seconds + # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value + # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # drop if this renewal is not allowed to run, which sours the test. + # self.pump(amount=Duration(hours=1)) + self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) + + # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Release the first lock (`lock1`). The second lock(`lock2`) should be + # automatically acquired by the `pump()` inside `get_success()` + self.get_success(lock1.__aexit__(None, None, None)) + + # We should now have the lock + self.successResultOf(d2) + + def _pump_by( + self, + *, + amount: Duration = Duration(seconds=0), + by: Duration = Duration(seconds=0.1), + ) -> None: + """ + Like `self.pump()` but you can specify the time increment to advance with until + you reach the time amount. + + Unlike `self.pump()`, this doesn't multiply the time at all. + + Args: + amount: The amount of time to advance + by: The time increment in seconds to advance time by until we reach the `amount` + """ + end_time_s = self.reactor.seconds() + amount.as_secs() + + while self.reactor.seconds() < end_time_s: + self.reactor.advance(by.as_secs()) From 3e6bf10640d25d127399debb4beca2e9129524f4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2026 11:38:15 +0100 Subject: [PATCH 03/42] Port `Event.signatures` field to Rust (#19706) This is another stepping stone in porting the event class fully to Rust. The new `Signatures` class is relatively simple, as we actually don't interact with it that much in the code. It does *not* implement `Mapping` or `MutableMapping` as that takes quite a lot of effort that we don't need, even though it would be more ergonomic. --- changelog.d/19706.misc | 1 + rust/Cargo.toml | 6 +- rust/src/events/mod.rs | 2 + rust/src/events/signatures.rs | 348 ++++++++++++++++++ synapse/crypto/event_signing.py | 4 +- synapse/crypto/keyring.py | 12 +- synapse/event_auth.py | 6 +- synapse/events/__init__.py | 14 +- synapse/handlers/message.py | 2 +- synapse/handlers/room_policy.py | 9 +- .../databases/main/events_bg_updates.py | 4 +- synapse/synapse_rust/events.pyi | 31 +- tests/handlers/test_federation.py | 2 +- 13 files changed, 415 insertions(+), 26 deletions(-) create mode 100644 changelog.d/19706.misc create mode 100644 rust/src/events/signatures.rs diff --git a/changelog.d/19706.misc b/changelog.d/19706.misc new file mode 100644 index 0000000000..205abd09d4 --- /dev/null +++ b/changelog.d/19706.misc @@ -0,0 +1 @@ +Port `Event.signatures` field to Rust. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e6b378a092..5bdd194707 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -43,7 +43,7 @@ pyo3-log = "0.13.1" pythonize = "0.27.0" regex = "1.6.0" sha2 = "0.10.8" -serde = { version = "1.0.144", features = ["derive"] } +serde = { version = "1.0.144", features = ["derive", "rc"] } serde_json = { version = "1.0.85", features = ["raw_value"] } ulid = "1.1.2" icu_segmenter = "2.0.0" @@ -58,10 +58,6 @@ tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } once_cell = "1.18.0" itertools = "0.14.0" -[features] -extension-module = ["pyo3/extension-module"] -default = ["extension-module"] - [build-dependencies] blake2 = "0.10.4" hex = "0.4.3" diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 209efb917b..e42eb97739 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -27,11 +27,13 @@ use pyo3::{ pub mod filter; mod internal_metadata; +pub mod signatures; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; m.add_submodule(&child_module)?; diff --git a/rust/src/events/signatures.rs b/rust/src/events/signatures.rs new file mode 100644 index 0000000000..0f2acd5c9b --- /dev/null +++ b/rust/src/events/signatures.rs @@ -0,0 +1,348 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Class for representing event signatures + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError}, + pyclass, pymethods, + types::{PyAnyMethods, PyDict, PyMapping, PyMappingMethods}, + Bound, IntoPyObject, PyAny, PyResult, Python, +}; +use serde::{Deserialize, Serialize}; + +/// A class representing the signatures on an event. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Signatures { + inner: Arc>>>, +} + +#[pymethods] +impl Signatures { + #[new] + #[pyo3(signature = (signatures = None))] + fn py_new(signatures: Option>>) -> Self { + let mut signatures = signatures.unwrap_or_default(); + + // Prune any entries that have no signatures. + signatures.retain(|_, server_sigs| !server_sigs.is_empty()); + + Self { + inner: Arc::new(RwLock::new(signatures)), + } + } + + /// Check if the signatures contain a signature for the given server name. + fn __contains__(&self, key: Bound<'_, PyAny>) -> PyResult { + let Ok(key) = key.extract::<&str>() else { + return Ok(false); + }; + + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + Ok(signatures.contains_key(key)) + } + + /// Get the number of servers that have signatures. + fn __len__(&self) -> PyResult { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + Ok(signatures.len()) + } + + /// Get the signature for the given server name and key ID, if it exists. + fn get_signature(&self, server_name: &str, key_id: &str) -> PyResult> { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + Ok(signatures + .get(server_name) + .and_then(|server_sigs| server_sigs.get(key_id).cloned())) + } + + /// Get the signatures for the given server name. + fn __getitem__(&self, key: Bound<'_, PyAny>) -> PyResult> { + let Some(server_name) = key.extract::<&str>().ok() else { + return Err(PyKeyError::new_err(key.to_string())); + }; + + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + if let Some(server_sigs) = signatures.get(server_name) { + Ok(server_sigs.clone()) + } else { + Err(PyKeyError::new_err(server_name.to_string())) + } + } + + /// Add a signature for the given server name and key ID. + fn add_signature( + &self, + server_name: String, + key_id: String, + signature: String, + ) -> PyResult<()> { + let mut signatures = self + .inner + .write() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + signatures + .entry(server_name) + .or_default() + .insert(key_id, signature); + + Ok(()) + } + + /// Update the signatures with the given signatures. + /// + /// Will overwrite all existing signatures for the server names provided. + fn update(&self, other: &Bound<'_, PyMapping>) -> PyResult<()> { + let mut signatures = self + .inner + .write() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + for list_entry in other.items()? { + let (server_name, server_sigs) = list_entry.extract::<(String, Bound)>()?; + + let mut entry = HashMap::new(); + for list_entry in server_sigs.items()? { + let (key, value) = list_entry.extract::<(String, String)>()?; + entry.insert(key, value); + } + + // Only insert the entry if it has at least one signature. + if !entry.is_empty() { + signatures.insert(server_name, entry); + } else { + signatures.remove(&server_name); + } + } + + Ok(()) + } + + /// Return a copy of the signatures as a dictionary. + fn as_dict<'py>(&self, py: Python<'py>) -> PyResult> { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + (&*signatures).into_pyobject(py) + } + + fn __repr__(&self) -> PyResult { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + Ok(format!("Signatures({signatures:?})")) + } +} + +#[cfg(test)] +mod tests { + use pythonize::pythonize; + + use super::*; + + /// Helper that reads the inner map directly. + fn read_inner(sigs: &Signatures) -> HashMap> { + sigs.inner.read().expect("lock poisoned").clone() + } + + /// Helper to create a server signatures map from a list of (key_id, sig) + /// pairs. + fn make_server_sigs(data: &[(&str, &str)]) -> HashMap { + let mut map = HashMap::new(); + for (key_id, sig) in data { + map.insert((*key_id).to_owned(), (*sig).to_owned()); + } + map + } + + /// Helper to create a `Signatures` object from a list of (server_name, + /// key_id, sig) tuples. + fn create_signatures(data: &[(&str, &str, &str)]) -> Signatures { + let mut map: HashMap> = HashMap::new(); + for (server_name, key_id, sig) in data { + map.entry((*server_name).to_owned()) + .or_default() + .insert((*key_id).to_owned(), (*sig).to_owned()); + } + Signatures::py_new(Some(map)) + } + + #[test] + fn test_new_empty() { + let sigs = Signatures::py_new(None); + assert!(read_inner(&sigs).is_empty()); + assert_eq!(sigs.__len__().unwrap(), 0); + } + + #[test] + fn test_new_with_data() { + let sigs = create_signatures(&[("example.com", "ed25519:key1", "sig1")]); + assert_eq!(sigs.__len__().unwrap(), 1); + assert_eq!( + sigs.get_signature("example.com", "ed25519:key1").unwrap(), + Some("sig1".to_string()) + ); + } + + #[test] + fn test_new_prunes_servers_with_no_signatures() { + let mut data = HashMap::new(); + data.insert("empty.example.com".to_string(), HashMap::new()); + data.insert( + "example.com".to_string(), + make_server_sigs(&[("ed25519:key1", "sig1")]), + ); + + let sigs = Signatures::py_new(Some(data)); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert!(inner.contains_key("example.com")); + assert!(!inner.contains_key("empty.example.com")); + } + + #[test] + fn test_add_signature() { + let sigs = Signatures::py_new(None); + sigs.add_signature( + "example.com".to_string(), + "ed25519:key1".to_string(), + "sig1".to_string(), + ) + .unwrap(); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key1")), + Some(&"sig1".to_string()) + ); + } + + #[test] + fn test_add_signature_to_existing_server() { + let sigs = create_signatures(&[("example.com", "ed25519:key1", "sig1")]); + sigs.add_signature( + "example.com".to_string(), + "ed25519:key2".to_string(), + "sig2".to_string(), + ) + .unwrap(); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key1")), + Some(&"sig1".to_string()) + ); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key2")), + Some(&"sig2".to_string()) + ); + } + + #[test] + fn test_update_signatures_clobbers_existing() { + let sigs = create_signatures(&[("example.com", "ed25519:key1", "sig1")]); + + // Create a new signatures map with a different signature for the same + // server. + let mut other = HashMap::new(); + other.insert( + "example.com".to_string(), + make_server_sigs(&[("ed25519:key2", "sig2")]), + ); + + // Update the signatures with the new map. + Python::initialize(); + Python::attach(|py| { + let value = pythonize(py, &other).unwrap(); + let value = value.cast::().unwrap(); + + sigs.update(value).unwrap(); + }); + + // Check that the old signature has been replaced with the new one. + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!(inner["example.com"].len(), 1); + assert_eq!(inner["example.com"]["ed25519:key2"], "sig2"); + } + + #[test] + fn test_serialize() { + let mut data = HashMap::new(); + data.insert( + "example.com".to_string(), + make_server_sigs(&[("ed25519:key1", "sig1")]), + ); + let sigs = Signatures::py_new(Some(data)); + + let json = serde_json::to_string(&sigs).unwrap(); + assert_eq!(json, r#"{"example.com":{"ed25519:key1":"sig1"}}"#); + } + + #[test] + fn test_serialize_empty() { + let sigs = Signatures::py_new(None); + let json = serde_json::to_string(&sigs).unwrap(); + assert_eq!(json, "{}"); + } + + #[test] + fn test_deserialize() { + let json = r#"{"example.com":{"ed25519:key1":"sig1"}}"#; + let sigs: Signatures = serde_json::from_str(json).unwrap(); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key1")), + Some(&"sig1".to_string()) + ); + } + + #[test] + fn test_deserialize_empty() { + let sigs: Signatures = serde_json::from_str("{}").unwrap(); + assert!(read_inner(&sigs).is_empty()); + } +} diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index d789c06a9c..823b6288e8 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -236,9 +236,7 @@ def event_needs_resigning( if sender.domain != server_name: return False want_key_id = verify_key.alg + ":" + verify_key.version - signed_with_current_key_id = ev.signatures.get(server_name, {}).get( - want_key_id, None - ) + signed_with_current_key_id = ev.signatures.get_signature(server_name, want_key_id) if signed_with_current_key_id: return False diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 0d4d5e0e17..36736b4559 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -120,8 +120,18 @@ class VerifyJsonRequest: ) -> "VerifyJsonRequest": """Create a VerifyJsonRequest to verify all signatures on an event object for the given server. + + Raises immediately if the event doesn't have any signatures from the + given server. """ - key_ids = list(event.signatures.get(server_name, [])) + if server_name not in event.signatures: + raise SynapseError( + 400, + f"Not signed by {server_name}", + Codes.UNAUTHORIZED, + ) + + key_ids = list(event.signatures[server_name]) return VerifyJsonRequest( server_name, # We defer creating the redacted json object, as it uses a lot more diff --git a/synapse/event_auth.py b/synapse/event_auth.py index ca528ae235..fd35da8ba0 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -128,7 +128,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: ) # Check the sender's domain has signed the event - if not event.signatures.get(sender_domain): + if sender_domain not in event.signatures: # We allow invites via 3pid to have a sender from a different # HS, as the sender must match the sender of the original # 3pid invite. This is checked further down with the @@ -141,7 +141,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: event_id_domain = get_domain_from_id(event.event_id) # Check the origin domain has signed the event - if not event.signatures.get(event_id_domain): + if event_id_domain not in event.signatures: raise AuthError(403, "Event not signed by sending server") is_invite_via_allow_rule = ( @@ -154,7 +154,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: authoriser_domain = get_domain_from_id( event.content[EventContentFields.AUTHORISING_USER] ) - if not event.signatures.get(authoriser_domain): + if authoriser_domain not in event.signatures: raise AuthError(403, "Event not signed by authorising server") diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index ac9b31cfcf..fc0f6aadbd 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -44,8 +44,12 @@ from synapse.api.constants import ( StickyEvent, ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import EventInternalMetadata -from synapse.types import JsonDict, StateKey, StrCollection +from synapse.synapse_rust.events import EventInternalMetadata, Signatures +from synapse.types import ( + JsonDict, + StateKey, + StrCollection, +) from synapse.util.caches import intern_dict from synapse.util.duration import Duration from synapse.util.frozenutils import freeze @@ -203,7 +207,7 @@ class EventBase(metaclass=abc.ABCMeta): assert room_version.event_format == self.format_version self.room_version = room_version - self.signatures = signatures + self.signatures = Signatures(signatures) self.unsigned = unsigned self.rejected_reason = rejected_reason @@ -255,7 +259,9 @@ class EventBase(metaclass=abc.ABCMeta): def get_dict(self) -> JsonDict: d = dict(self._dict) - d.update({"signatures": self.signatures, "unsigned": dict(self.unsigned)}) + d.update( + {"signatures": self.signatures.as_dict(), "unsigned": dict(self.unsigned)} + ) return d diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4032c7eca9..ed3bce69ab 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -2092,7 +2092,7 @@ class EventCreationHandler: event.unsigned.pop("room_state", None) # TODO: Make sure the signatures actually are correct. - event.signatures.update(returned_invite.signatures) + event.signatures.update(returned_invite.signatures.as_dict()) if event.content["membership"] == Membership.KNOCK: maybe_upsert_event_field( diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index 01943e1991..e46e6dc2ef 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -181,9 +181,10 @@ class RoomPolicyHandler: async def _verify_policy_server_signature( self, event: EventBase, policy_server: str, public_key: str ) -> bool: - # check the event is signed with this (via, public_key). - verify_json_req = VerifyJsonRequest.from_event(policy_server, event, 0) try: + # check the event is signed with this (via, public_key). + verify_json_req = VerifyJsonRequest.from_event(policy_server, event, 0) + key_bytes = decode_base64(public_key) verify_key = decode_verify_key_bytes(POLICY_SERVER_KEY_ID, key_bytes) # We would normally use KeyRing.verify_event_for_server but we can't here as we don't @@ -260,9 +261,7 @@ class RoomPolicyHandler: # servers need to manually fetch signatures for. This is the code that allows # those events to continue working (because they're legally sent, even if missing # the policy server signature). - event.signatures.setdefault(policy_server.server_name, {}).update( - signature.get(policy_server.server_name, {}) - ) + event.signatures.update(signature) except HttpResponseException as ex: # re-wrap HTTP errors as `SynapseError` so they can be proxied to clients directly raise ex.to_synapse_error() from ex diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index d2623f0760..c0d218398d 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -2824,8 +2824,8 @@ class EventsBackgroundUpdatesStore( # with the provided old key. if old_verify_key is not None: old_key_id = f"{old_verify_key.alg}:{old_verify_key.version}" - server_sigs = event.signatures.get(self.hs.hostname, {}) - if old_key_id not in server_sigs: + old_sig = event.signatures.get_signature(self.hs.hostname, old_key_id) + if old_sig is None: # Event wasn't signed with this key ID at all, skip. continue diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index fe0ca04420..40cf3f59f2 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -10,7 +10,7 @@ # See the GNU Affero General Public License for more details: # . -from typing import Mapping +from typing import Any, Mapping from synapse.types import JsonDict @@ -154,3 +154,32 @@ def event_visible_to_server( Returns: Whether the server is allowed to see the unredacted event. """ + +class Signatures: + """A class representing the signatures on an event.""" + + def __init__(self, signatures: Mapping[str, Mapping[str, str]] | None = None): ... + def get_signature(self, server_name: str, key_id: str) -> str | None: ... + """Get the signature for the given server name and key ID, if it exists.""" + + def __getitem__(self, server_name: str) -> Mapping[str, str]: ... + """Get the signatures for the given server name. Raises KeyError if there + are no signatures for that server.""" + + def __contains__(self, server_name: Any) -> bool: ... + """Check if there are signatures for the given server name.""" + + def __len__(self) -> int: ... + """Return the number of servers that have signatures.""" + + def add_signature(self, server_name: str, key_id: str, signature: str) -> None: ... + """Add a signature for the given server name and key ID.""" + + def update(self, signatures: Mapping[str, Mapping[str, str]]) -> None: ... + """Update the signatures with the given signatures. + + Will overwrite all existing signatures for the server names provided. + """ + + def as_dict(self) -> dict[str, dict[str, str]]: ... + """Return a copy of the signatures as a dictionary.""" diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index dde1785854..20ffed68f4 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -368,7 +368,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): ) # the auth code requires that a signature exists, but doesn't check that # signature... go figure. - join_event.signatures[other_server] = {"x": "y"} + join_event.signatures.update({other_server: {"x": "y"}}) self.get_success( self.hs.get_federation_event_handler().on_send_membership_event( From 23b8fcf85e9cb9dadb2b8ba2a29a2f717e19f790 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2026 18:51:42 +0100 Subject: [PATCH 04/42] Port `Event.unsigned` field to Rust (#19708) Similar to #19706, let's port the `unsigned` field into a Rust class. This does change things a bit in that we now define exactly what unsigned fields that are allowed to be added to an event, and what actually gets persisted. This should be a noop though, as we carefully filter out what unsigned fields we allow in from federation, for example As a side effect of this cleanup, I think this fixes handling `unsigned.age` on events received over federation. --- changelog.d/19708.misc | 1 + rust/src/events/mod.rs | 2 + rust/src/events/unsigned.rs | 429 ++++++++++++++++++ synapse/events/__init__.py | 26 +- synapse/events/utils.py | 3 +- synapse/federation/federation_base.py | 32 +- synapse/federation/federation_client.py | 7 +- synapse/federation/federation_server.py | 17 +- synapse/handlers/message.py | 1 - synapse/storage/databases/main/events.py | 2 +- .../storage/databases/main/events_worker.py | 16 +- synapse/synapse_rust/events.pyi | 32 +- tests/events/test_utils.py | 32 +- tests/federation/test_federation_server.py | 10 +- 14 files changed, 568 insertions(+), 42 deletions(-) create mode 100644 changelog.d/19708.misc create mode 100644 rust/src/events/unsigned.rs diff --git a/changelog.d/19708.misc b/changelog.d/19708.misc new file mode 100644 index 0000000000..308c2b04d0 --- /dev/null +++ b/changelog.d/19708.misc @@ -0,0 +1 @@ +Port `Event.unsigned` field to Rust. diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index e42eb97739..5f505abb91 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -28,12 +28,14 @@ use pyo3::{ pub mod filter; mod internal_metadata; pub mod signatures; +pub mod unsigned; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; m.add_submodule(&child_module)?; diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs new file mode 100644 index 0000000000..c41ed7e6e1 --- /dev/null +++ b/rust/src/events/unsigned.rs @@ -0,0 +1,429 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::{Arc, RwLock, RwLockReadGuard}; + +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError, PyTypeError}, + pyclass, pymethods, + types::{PyAnyMethods, PyList, PyListMethods, PyMapping}, + Bound, IntoPyObjectExt, PyAny, PyResult, Python, +}; +use pythonize::{depythonize, pythonize}; +use serde::{Deserialize, Serialize}; + +#[pyclass(frozen, skip_from_py_object)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(transparent)] +pub struct Unsigned { + inner: Arc>, +} + +/// The fields in the unsigned data of an event that are persisted in the +/// database. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +struct PersistedUnsignedFields { + #[serde(skip_serializing_if = "Option::is_none")] + age_ts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + replaces_state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + invite_room_state: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + knock_room_state: Option>, +} + +/// The inner representation of the unsigned data of an event, which includes +/// both the fields that are persisted in the database and the fields that are +/// only used in memory. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct UnsignedInner { + #[serde(flatten)] + persisted_fields: PersistedUnsignedFields, + #[serde(skip_serializing_if = "Option::is_none")] + prev_content: Option>, // We use Box to minimise stack space + #[serde(skip_serializing_if = "Option::is_none")] + prev_sender: Option, +} + +/// The fields that exist on the unsigned data of an event. +/// +/// This is used when converting from python to rust, to ensure that if we add a +/// new field we don't forget to add it to all the necessary places. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UnsignedField { + AgeTs, + ReplacesState, + InviteRoomState, + KnockRoomState, + PrevContent, + PrevSender, +} + +impl std::str::FromStr for UnsignedField { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "age_ts" => Ok(Self::AgeTs), + "replaces_state" => Ok(Self::ReplacesState), + "invite_room_state" => Ok(Self::InviteRoomState), + "knock_room_state" => Ok(Self::KnockRoomState), + "prev_content" => Ok(Self::PrevContent), + "prev_sender" => Ok(Self::PrevSender), + _ => Err(()), + } + } +} + +impl Unsigned { + fn py_read(&self) -> PyResult> { + self.inner + .read() + .map_err(|_| PyRuntimeError::new_err("Unsigned lock poisoned")) + } + + fn py_write(&self) -> PyResult> { + self.inner + .write() + .map_err(|_| PyRuntimeError::new_err("Unsigned lock poisoned")) + } +} + +#[pymethods] +impl Unsigned { + #[new] + fn py_new(unsigned: Bound<'_, PyMapping>) -> PyResult { + let inner = depythonize(&unsigned)?; + + Ok(Self { + inner: Arc::new(RwLock::new(inner)), + }) + } + + fn __getitem__<'py>( + &self, + py: Python<'py>, + key: Bound<'_, PyAny>, + ) -> PyResult> { + let key = key + .extract::<&str>() + .map_err(|_| PyTypeError::new_err("Unsigned keys must be strings"))?; + + let field: UnsignedField = key + .parse() + .map_err(|_| PyKeyError::new_err(format!("Unsigned has no key '{key}'")))?; + + let unsigned = self.py_read()?; + + match field { + UnsignedField::AgeTs => Ok(unsigned + .persisted_fields + .age_ts + .ok_or_else(|| PyKeyError::new_err("age_ts"))? + .into_bound_py_any(py)?), + UnsignedField::ReplacesState => Ok((unsigned.persisted_fields.replaces_state) + .as_ref() + .ok_or_else(|| PyKeyError::new_err("replaces_state"))? + .into_bound_py_any(py)?), + UnsignedField::InviteRoomState => Ok(room_state_to_py( + py, + unsigned + .persisted_fields + .invite_room_state + .as_ref() + .ok_or_else(|| PyKeyError::new_err("invite_room_state"))?, + )?), + UnsignedField::KnockRoomState => Ok(room_state_to_py( + py, + unsigned + .persisted_fields + .knock_room_state + .as_ref() + .ok_or_else(|| PyKeyError::new_err("knock_room_state"))?, + )?), + UnsignedField::PrevContent => Ok(pythonize( + py, + unsigned + .prev_content + .as_ref() + .ok_or_else(|| PyKeyError::new_err("prev_content"))?, + )?), + UnsignedField::PrevSender => Ok((unsigned.prev_sender) + .as_ref() + .ok_or_else(|| PyKeyError::new_err("prev_sender"))? + .into_bound_py_any(py)?), + } + } + + fn __contains__(&self, key: Bound<'_, PyAny>) -> PyResult { + let Ok(key) = key.extract::<&str>() else { + return Ok(false); + }; + + let Ok(field) = key.parse::() else { + return Ok(false); + }; + + let unsigned = self.py_read()?; + + let exists = match field { + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts.is_some(), + UnsignedField::ReplacesState => unsigned.persisted_fields.replaces_state.is_some(), + UnsignedField::InviteRoomState => unsigned.persisted_fields.invite_room_state.is_some(), + UnsignedField::KnockRoomState => unsigned.persisted_fields.knock_room_state.is_some(), + UnsignedField::PrevContent => unsigned.prev_content.is_some(), + UnsignedField::PrevSender => unsigned.prev_sender.is_some(), + }; + + Ok(exists) + } + + fn __setitem__(&self, key: Bound<'_, PyAny>, value: Bound<'_, PyAny>) -> PyResult<()> { + let key = key + .extract::<&str>() + .map_err(|_| PyTypeError::new_err("Unsigned keys must be strings"))?; + + let field: UnsignedField = key + .parse() + .map_err(|_| PyKeyError::new_err(format!("Unsigned has no key '{key}'")))?; + + let mut unsigned = self.py_write()?; + + match field { + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = Some(value.extract()?), + UnsignedField::ReplacesState => { + unsigned.persisted_fields.replaces_state = Some(value.extract()?) + } + UnsignedField::InviteRoomState => { + unsigned.persisted_fields.invite_room_state = Some(room_state_from_py(value)?) + } + UnsignedField::KnockRoomState => { + unsigned.persisted_fields.knock_room_state = Some(room_state_from_py(value)?) + } + UnsignedField::PrevContent => { + unsigned.prev_content = Some(Box::new(depythonize(&value)?)) + } + UnsignedField::PrevSender => unsigned.prev_sender = Some(value.extract()?), + } + + Ok(()) + } + + fn __delitem__(&self, key: Bound<'_, PyAny>) -> PyResult<()> { + let key = key + .extract::<&str>() + .map_err(|_| PyTypeError::new_err("Unsigned keys must be strings"))?; + + let field: UnsignedField = key + .parse() + .map_err(|_| PyKeyError::new_err(format!("Unsigned has no key '{key}'")))?; + + let mut unsigned = self.py_write()?; + + match field { + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = None, + UnsignedField::ReplacesState => unsigned.persisted_fields.replaces_state = None, + UnsignedField::InviteRoomState => unsigned.persisted_fields.invite_room_state = None, + UnsignedField::KnockRoomState => unsigned.persisted_fields.knock_room_state = None, + UnsignedField::PrevContent => unsigned.prev_content = None, + UnsignedField::PrevSender => unsigned.prev_sender = None, + } + + Ok(()) + } + + #[pyo3(signature = (key, default=None))] + fn get<'py>( + &self, + py: Python<'py>, + key: Bound<'py, PyAny>, + default: Option>, + ) -> PyResult>> { + match self.__getitem__(py, key) { + Ok(value) => Ok(Some(value)), + Err(err) => { + if err.is_instance_of::(py) { + Ok(default) + } else { + Err(err) + } + } + } + } + + fn for_persistence<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, &self.py_read()?.persisted_fields)?) + } + + fn for_event<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, &*self.py_read()?)?) + } +} + +fn room_state_to_py<'py>( + py: Python<'py>, + state: &[serde_json::Value], +) -> PyResult> { + let py_list = PyList::empty(py); + + for item in state { + py_list.append(pythonize(py, item)?)?; + } + + py_list.into_bound_py_any(py) +} + +fn room_state_from_py(value: Bound<'_, PyAny>) -> PyResult> { + let py_list = value.cast::()?; + + let mut state = Vec::with_capacity(py_list.len()); + for item in py_list.iter() { + state.push(pythonize::depythonize(&item)?); + } + + Ok(state) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn test_unsigned_field_from_str_valid() { + assert_eq!("age_ts".parse(), Ok(UnsignedField::AgeTs)); + assert_eq!("replaces_state".parse(), Ok(UnsignedField::ReplacesState)); + assert_eq!( + "invite_room_state".parse(), + Ok(UnsignedField::InviteRoomState) + ); + assert_eq!( + "knock_room_state".parse(), + Ok(UnsignedField::KnockRoomState) + ); + assert_eq!("prev_content".parse(), Ok(UnsignedField::PrevContent)); + assert_eq!("prev_sender".parse(), Ok(UnsignedField::PrevSender)); + } + + #[test] + fn test_unsigned_field_from_str_invalid() { + assert_eq!("".parse::(), Err(())); + assert_eq!("unknown".parse::(), Err(())); + // Case-sensitive: upper-case should not match. + assert_eq!("AGE_TS".parse::(), Err(())); + // Must be an exact match, no whitespace. + assert_eq!(" age_ts".parse::(), Err(())); + } + + #[test] + fn test_persisted_fields_serialize_empty_is_empty_object() { + let fields = PersistedUnsignedFields::default(); + let json = serde_json::to_value(&fields).unwrap(); + assert_eq!(json, json!({})); + } + + #[test] + fn test_persisted_fields_serialize_populated() { + let fields = PersistedUnsignedFields { + age_ts: Some(1234), + replaces_state: Some("$prev:example.com".to_string()), + invite_room_state: Some(vec![json!({"type": "m.room.name"})]), + knock_room_state: Some(vec![json!({"type": "m.room.topic"})]), + }; + let json = serde_json::to_value(&fields).unwrap(); + assert_eq!( + json, + json!({ + "age_ts": 1234, + "replaces_state": "$prev:example.com", + "invite_room_state": [{"type": "m.room.name"}], + "knock_room_state": [{"type": "m.room.topic"}], + }) + ); + } + + #[test] + fn test_unsigned_inner_flattens_persisted_fields() { + let inner = UnsignedInner { + persisted_fields: PersistedUnsignedFields { + age_ts: Some(99), + ..Default::default() + }, + prev_content: Some(Box::new(json!({"body": "hi"}))), + prev_sender: Some("@alice:example.com".to_string()), + }; + + let json = serde_json::to_value(&inner).unwrap(); + assert_eq!( + json, + json!({ + "age_ts": 99, + "prev_content": {"body": "hi"}, + "prev_sender": "@alice:example.com", + }) + ); + } + + #[test] + fn test_unsigned_inner_roundtrip() { + let original = UnsignedInner { + persisted_fields: PersistedUnsignedFields { + age_ts: Some(10), + replaces_state: Some("$state:example.com".to_string()), + invite_room_state: None, + knock_room_state: None, + }, + prev_content: Some(Box::new(json!({"membership": "join"}))), + prev_sender: None, + }; + + let json = serde_json::to_string(&original).unwrap(); + let roundtripped: UnsignedInner = serde_json::from_str(&json).unwrap(); + + assert_eq!(roundtripped.persisted_fields.age_ts, Some(10)); + assert_eq!( + roundtripped.persisted_fields.replaces_state.as_deref(), + Some("$state:example.com") + ); + assert_eq!( + roundtripped.prev_content.as_deref(), + Some(&json!({"membership": "join"})) + ); + assert_eq!(roundtripped.prev_sender, None); + } + + #[test] + fn test_unsigned_serializes_transparently() { + // `Unsigned` is `#[serde(transparent)]` over its inner, so serializing + // an empty default should yield an empty object rather than a wrapper. + let unsigned = Unsigned::default(); + let json = serde_json::to_value(&unsigned).unwrap(); + assert_eq!(json, json!({})); + } + + #[test] + fn test_unsigned_deserialize_from_flat_object() { + let json = json!({ + "age_ts": 5, + "prev_sender": "@bob:example.com", + }); + let unsigned: Unsigned = serde_json::from_value(json).unwrap(); + let inner = unsigned.inner.read().unwrap(); + assert_eq!(inner.persisted_fields.age_ts, Some(5)); + assert_eq!(inner.prev_sender.as_deref(), Some("@bob:example.com")); + } +} diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index fc0f6aadbd..0f850d19b1 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -44,7 +44,7 @@ from synapse.api.constants import ( StickyEvent, ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import EventInternalMetadata, Signatures +from synapse.synapse_rust.events import EventInternalMetadata, Signatures, Unsigned from synapse.types import ( JsonDict, StateKey, @@ -208,7 +208,7 @@ class EventBase(metaclass=abc.ABCMeta): self.room_version = room_version self.signatures = Signatures(signatures) - self.unsigned = unsigned + self.unsigned = Unsigned(unsigned) self.rejected_reason = rejected_reason self._dict = event_dict @@ -258,9 +258,25 @@ class EventBase(metaclass=abc.ABCMeta): return self._dict.get("state_key") def get_dict(self) -> JsonDict: + """Convert the event to a dictionary suitable for serialisation.""" d = dict(self._dict) d.update( - {"signatures": self.signatures.as_dict(), "unsigned": dict(self.unsigned)} + { + "signatures": self.signatures.as_dict(), + "unsigned": self.unsigned.for_event(), + } + ) + + return d + + def get_dict_for_persistence(self) -> JsonDict: + """Convert the event to a dictionary suitable for persistence.""" + d = dict(self._dict) + d.update( + { + "signatures": self.signatures.as_dict(), + "unsigned": self.unsigned.for_persistence(), + } ) return d @@ -401,7 +417,7 @@ class FrozenEvent(EventBase): for name, sigs in event_dict.pop("signatures", {}).items() } - unsigned = dict(event_dict.pop("unsigned", {})) + unsigned = event_dict.pop("unsigned", {}) # We intern these strings because they turn up a lot (especially when # caching). @@ -455,7 +471,7 @@ class FrozenEventV2(EventBase): assert "event_id" not in event_dict - unsigned = dict(event_dict.pop("unsigned", {})) + unsigned = event_dict.pop("unsigned", {}) # We intern these strings because they turn up a lot (especially when # caching). diff --git a/synapse/events/utils.py b/synapse/events/utils.py index f038fb5578..926c81b83d 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -47,6 +47,7 @@ from synapse.api.constants import ( from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersion from synapse.logging.opentracing import SynapseTags, set_tag, trace +from synapse.synapse_rust.events import Unsigned from synapse.types import JsonDict, Requester from . import EventBase, FrozenEventV2, StrippedStateEvent, make_event_from_dict @@ -987,7 +988,7 @@ def validate_canonicaljson(value: Any) -> None: def maybe_upsert_event_field( - event: EventBase, container: JsonDict, key: str, value: object + event: EventBase, container: Unsigned, key: str, value: object ) -> bool: """Upsert an event field, but only if this doesn't make the event too large. diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index fe0710a0bf..1631f021ca 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -307,20 +307,27 @@ def _is_invite_via_3pid(event: EventBase) -> bool: def parse_events_from_pdu_json( - pdus_json: Sequence[JsonDict], room_version: RoomVersion + pdus_json: Sequence[JsonDict], + room_version: RoomVersion, + received_time: int | None = None, ) -> list[EventBase]: return [ - event_from_pdu_json(pdu_json, room_version) + event_from_pdu_json(pdu_json, room_version, received_time=received_time) for pdu_json in filter_pdus_for_valid_depth(pdus_json) ] -def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventBase: +def event_from_pdu_json( + pdu_json: JsonDict, room_version: RoomVersion, received_time: int | None = None +) -> EventBase: """Construct an EventBase from an event json received over federation Args: pdu_json: pdu as received over federation room_version: The version of the room this event belongs to + received_time: timestamp in ms that the event was received at. If + `None` then any `age` field in the `unsigned` block will be + dropped. Raises: SynapseError: if the pdu is missing required fields or is otherwise @@ -333,6 +340,25 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB if "unsigned" in pdu_json: _strip_unsigned_values(pdu_json) + # Handle the `age` field, which is sent by some servers as part of the + # `unsigned` block. We convert this into an `age_ts` field, which is + # what Synapse uses internally. We also remove the `age` field to avoid + # confusion. + # + # c.f. https://github.com/matrix-org/synapse/issues/8429 + unsigned = pdu_json["unsigned"] + age = unsigned.pop("age", None) + + # We check that the `age` is actually an int before using it below. We + # don't error here as the `age` a) doesn't affect the validity of the + # event, and b) is best effort anyway. + if not isinstance(age, int): + age = None + + unsigned.pop("age_ts", None) + if received_time is not None and age is not None: + unsigned["age_ts"] = received_time - int(age) + depth = pdu_json["depth"] if type(depth) is not int: # noqa: E721 raise SynapseError(400, "Depth %r not an intger" % (depth,), Codes.BAD_JSON) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 78a1900c73..2b5ef5fbac 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1574,10 +1574,15 @@ class FederationClient(FederationBase): min_depth=min_depth, timeout=timeout, ) + received_time = self._clock.time_msec() room_version = await self.store.get_room_version(room_id) - events = parse_events_from_pdu_json(content.get("events", []), room_version) + events = parse_events_from_pdu_json( + content.get("events", []), + room_version, + received_time=received_time, + ) signed_events = await self._check_sigs_and_hash_for_pulled_events_and_fetch( destination, events, room_version=room_version diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 1bbe144422..6069287975 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -451,16 +451,6 @@ class FederationServer(FederationBase): newest_pdu_ts = 0 for p in transaction.pdus: - # FIXME (richardv): I don't think this works: - # https://github.com/matrix-org/synapse/issues/8429 - if "unsigned" in p: - unsigned = p["unsigned"] - if "age" in unsigned: - p["age"] = unsigned["age"] - if "age" in p: - p["age_ts"] = request_time - int(p["age"]) - del p["age"] - # We try and pull out an event ID so that if later checks fail we # can log something sensible. We don't mandate an event ID here in # case future event formats get rid of the key. @@ -488,10 +478,15 @@ class FederationServer(FederationBase): continue try: - event = event_from_pdu_json(p, room_version) + event = event_from_pdu_json(p, room_version, received_time=request_time) except SynapseError as e: logger.info("Ignoring PDU for failing to deserialize: %s", e) continue + except Exception as e: + # We catch all exceptions here as we don't want a single bad + # event to cause us to fail the whole transaction. + logger.exception("Error deserializing PDU: %s", e) + continue pdus_by_room.setdefault(room_id, []).append(event) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index ed3bce69ab..0687c9fa79 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -2089,7 +2089,6 @@ class EventCreationHandler: returned_invite = await federation_handler.send_invite( invitee.domain, event ) - event.unsigned.pop("room_state", None) # TODO: Make sure the signatures actually are correct. event.signatures.update(returned_invite.signatures.as_dict()) diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 12c918eca6..5f6b03a988 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -2711,7 +2711,7 @@ class PersistEventsStore: return def event_dict(event: EventBase) -> JsonDict: - d = event.get_dict() + d = event.get_dict_for_persistence() d.pop("redacted", None) d.pop("redacted_because", None) return d diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index cc79b8042b..6f26bd17ce 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -779,14 +779,26 @@ class EventsWorkerStore(SQLBaseStore): events.append(event) if get_prev_content: - if "replaces_state" in event.unsigned: + # The `event` here might be in the cache, and so might have + # already had the `prev_content` and `prev_sender` fields added + # to its unsigned. + # + # We check if a) we should add the previous content, and b) if + # we have already added it. + replaces_state = "replaces_state" in event.unsigned + has_prev = ( + "prev_content" in event.unsigned and "prev_sender" in event.unsigned + ) + if replaces_state and not has_prev: prev = await self.get_event( event.unsigned["replaces_state"], get_prev_content=False, allow_none=True, ) if prev: - event.unsigned = dict(event.unsigned) + # This mutates the cached event, but that's fine as the + # previous content/sender will be the same for all + # requests for this event. event.unsigned["prev_content"] = prev.content event.unsigned["prev_sender"] = prev.sender diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 40cf3f59f2..5ae2bb880a 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -12,7 +12,7 @@ from typing import Any, Mapping -from synapse.types import JsonDict +from synapse.types import JsonDict, JsonMapping class EventInternalMetadata: def __init__(self, internal_metadata_dict: JsonDict): ... @@ -183,3 +183,33 @@ class Signatures: def as_dict(self) -> dict[str, dict[str, str]]: ... """Return a copy of the signatures as a dictionary.""" + +class Unsigned: + """A class representing the unsigned data of an event.""" + + def __init__(self, unsigned_dict: JsonMapping): ... + def __getitem__(self, key: str) -> Any: ... + """Get the value for the given key. + + Raises KeyError if the key is unset or not recognised.""" + + def __setitem__(self, key: str, value: Any) -> None: ... + """Set the value for the given key. + + Raises KeyError if the key is not recognised.""" + + def __delitem__(self, key: str) -> None: ... + """Delete the value for the given key. + + Raises KeyError if the key is unset or not recognised.""" + + def __contains__(self, key: Any) -> bool: ... + def get(self, key: str, default: Any = None) -> Any: ... + """Get the value for the given key, or ``default`` if the key is unset.""" + + def for_persistence(self) -> JsonDict: ... + """Return a dict of the fields that should be persisted to the database.""" + + def for_event(self) -> JsonDict: ... + """Return a dict of all unsigned fields, including those only kept in + memory, suitable for inclusion in an event.""" diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 12ef42866d..a402dd70d1 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -67,25 +67,31 @@ def MockEvent(**kwargs: Any) -> EventBase: class TestMaybeUpsertEventField(stdlib_unittest.TestCase): def test_update_okay(self) -> None: event = make_event_from_dict({"event_id": "$1234"}) - success = maybe_upsert_event_field(event, event.unsigned, "key", "value") + success = maybe_upsert_event_field( + event, event.unsigned, "replaces_state", "value" + ) self.assertTrue(success) - self.assertEqual(event.unsigned["key"], "value") + self.assertEqual(event.unsigned["replaces_state"], "value") def test_update_not_okay(self) -> None: event = make_event_from_dict({"event_id": "$1234"}) LARGE_STRING = "a" * 100_000 - success = maybe_upsert_event_field(event, event.unsigned, "key", LARGE_STRING) + success = maybe_upsert_event_field( + event, event.unsigned, "replaces_state", LARGE_STRING + ) self.assertFalse(success) - self.assertNotIn("key", event.unsigned) + self.assertNotIn("replaces_state", event.unsigned) def test_update_not_okay_leaves_original_value(self) -> None: event = make_event_from_dict( - {"event_id": "$1234", "unsigned": {"key": "value"}} + {"event_id": "$1234", "unsigned": {"replaces_state": "value"}} ) LARGE_STRING = "a" * 100_000 - success = maybe_upsert_event_field(event, event.unsigned, "key", LARGE_STRING) + success = maybe_upsert_event_field( + event, event.unsigned, "replaces_state", LARGE_STRING + ) self.assertFalse(success) - self.assertEqual(event.unsigned["key"], "value") + self.assertEqual(event.unsigned["replaces_state"], "value") class PruneEventTestCase(stdlib_unittest.TestCase): @@ -623,7 +629,7 @@ class CloneEventTestCase(stdlib_unittest.TestCase): { "type": "A", "event_id": "$test:domain", - "unsigned": {"a": 1, "b": 2}, + "unsigned": {"age_ts": 1, "replaces_state": "2"}, }, RoomVersions.V1, {"txn_id": "txn"}, @@ -634,10 +640,14 @@ class CloneEventTestCase(stdlib_unittest.TestCase): self.assertEqual(original.internal_metadata.instance_name, "worker1") cloned = clone_event(original) - cloned.unsigned["b"] = 3 + cloned.unsigned["age_ts"] = 3 - self.assertEqual(original.unsigned, {"a": 1, "b": 2}) - self.assertEqual(cloned.unsigned, {"a": 1, "b": 3}) + self.assertEqual( + original.unsigned.for_event(), {"age_ts": 1, "replaces_state": "2"} + ) + self.assertEqual( + cloned.unsigned.for_event(), {"age_ts": 3, "replaces_state": "2"} + ) self.assertEqual(cloned.internal_metadata.stream_ordering, 1234) self.assertEqual(cloned.internal_metadata.instance_name, "worker1") self.assertEqual(cloned.internal_metadata.txn_id, "txn") diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index a40e0b0680..7b1a2a5adc 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -754,9 +754,9 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): }, } - filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1) - self.assertIn("age", filtered_event2.unsigned) - self.assertEqual(14, filtered_event2.unsigned["age"]) + filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1, received_time=20) + self.assertIn("age_ts", filtered_event2.unsigned) + self.assertEqual(6, filtered_event2.unsigned["age_ts"]) self.assertNotIn("more warez", filtered_event2.unsigned) # Invite_room_state is allowed in events of type m.room.member self.assertIn("invite_room_state", filtered_event2.unsigned) @@ -779,8 +779,8 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): "invite_room_state": [], }, } - filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1) - self.assertIn("age", filtered_event3.unsigned) + filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1, received_time=20) + self.assertIn("age_ts", filtered_event3.unsigned) # Invite_room_state field is only permitted in event type m.room.member self.assertNotIn("invite_room_state", filtered_event3.unsigned) self.assertNotIn("more warez", filtered_event3.unsigned) From 0eefdbcb9535d6595ca2b419217404acf57623ac Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 5 May 2026 08:40:17 -0500 Subject: [PATCH 05/42] fix: Cap `WorkerLock` timeout intervals to 60 seconds (#19394) Fixes the symptoms of https://github.com/element-hq/synapse/issues/19315 / https://github.com/element-hq/synapse/issues/19588 but not the underlying reason causing the number to grow so large in the first place. ``` ValueError: Exceeds the limit (4300 digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit ``` Copied from the original pull request on [Famedly's Synapse repo](https://github.com/famedly/synapse/pull/221) (with some edits): Basing the time interval around a 5 seconds leaves a big window of waiting especially as this window is doubled each retry, when another worker could be making progress but can not. Right now, the retry interval in seconds looks like `[0.2, 5, 10, 20, 40, 80, 160, 320, (continues to double)]` after which logging should start about excessive times and (relatively quickly) end up with an extremely large retry interval with an unrealistic expectation past the heat death of the universe. 1 year in seconds = 31,536,000. With this change, retry intervals in seconds should look more like: ``` [ 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 60, < never goes higher than this ] ``` Logging about excessive wait times will start at 10 minutes.
Previous breakdown when we were using 15 minutes ``` [ 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 102.4, # 1.7 minutes 204.8, # 3.41 minutes 409.6, # 6.83 minutes 819.2, # 13.65 minutes < logging about excessive times will start here, 13th iteration 900, # 15 minutes < never goes higher than this ] ```
Further suggested work in this area could be to define the cap, the retry interval starting point and the multiplier depending on how frequently this lock should be checked. See data below for reasons why. Increasing the jitter range may also be a good idea --------- Co-authored-by: Eric Eastwood (cherry picked from commit 3f58bc50dfba5768ee43ce48c5e74c25ba0b078a) --- changelog.d/19394.bugfix | 1 + synapse/handlers/worker_lock.py | 117 +++++++++++++++++++------- tests/handlers/test_worker_lock.py | 130 +++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 32 deletions(-) create mode 100644 changelog.d/19394.bugfix diff --git a/changelog.d/19394.bugfix b/changelog.d/19394.bugfix new file mode 100644 index 0000000000..4ca92cfb32 --- /dev/null +++ b/changelog.d/19394.bugfix @@ -0,0 +1 @@ +Capped the `WorkerLock` time out interval to a maximum of 60 seconds to prevent dealing with excessively long numbers. Contributed by Famedly. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 1537a18cc0..51be3b5084 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -54,6 +54,9 @@ logger = logging.getLogger(__name__) # will not disappear under our feet as long as we don't delete the room. NEW_EVENT_DURING_PURGE_LOCK_NAME = "new_event_during_purge_lock" +WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=60) +WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) + class WorkerLocksHandler: """A class for waiting on taking out locks, rather than using the storage @@ -206,9 +209,10 @@ class WaitingLock: lock_name: str lock_key: str write: bool | None + start_ts_ms: int = 0 deferred: "defer.Deferred[None]" = attr.Factory(defer.Deferred) _inner_lock: Lock | None = None - _retry_interval: float = 0.1 + _timeout_interval: float = 0.1 _lock_span: "opentracing.Scope" = attr.Factory( lambda: start_active_span("WaitingLock.lock") ) @@ -220,6 +224,7 @@ class WaitingLock: self.deferred.callback(None) async def __aenter__(self) -> None: + self.start_ts_ms = self.clock.time_msec() self._lock_span.__enter__() with start_active_span("WaitingLock.waiting_for_lock"): @@ -240,19 +245,44 @@ class WaitingLock: break try: - # Wait until the we get notified the lock might have been + # Wait until the notification that the lock might have been # released (by the deferred being resolved). We also - # periodically wake up in case the lock was released but we + # periodically wake up in case the lock was released, but we # weren't notified. with PreserveLoggingContext(): - timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=timeout, + timeout=self._timeout_interval, clock=self.clock, ) - except Exception: - pass + except defer.TimeoutError: + # Only increment the timeout value if this was an actual timeout + # (defer.TimeoutError) + self._increment_timeout_interval() + + now_ms = self.clock.time_msec() + time_spent_trying_to_lock = Duration( + milliseconds=now_ms - self.start_ts_ms + ) + if ( + time_spent_trying_to_lock.as_millis() + > WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION.as_millis() + ): + logger.warning( + "(WaitingLock (%s, %s)) Time spent waiting to acquire lock " + "is getting excessive: %ss. There may be a deadlock.", + self.lock_name, + self.lock_key, + time_spent_trying_to_lock.as_secs(), + ) + + except Exception as e: + logger.warning( + "Caught an exception while waiting on WaitingLock(lock_name=%s, lock_key=%s): %r", + self.lock_name, + self.lock_key, + e, + ) return await self._inner_lock.__aenter__() @@ -273,15 +303,14 @@ class WaitingLock: return r - def _get_next_retry_interval(self) -> float: - next = self._retry_interval - self._retry_interval = max(5, next * 2) - if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations - logger.warning( - "Lock timeout is getting excessive: %ss. There may be a deadlock.", - self._retry_interval, - ) - return next * random.uniform(0.9, 1.1) + def _increment_timeout_interval(self) -> float: + next_interval = self._timeout_interval + next_interval = min(WORKER_LOCK_MAX_RETRY_INTERVAL.as_secs(), next_interval * 2) + + # The jitter value is maintained for the timeout, to help avoid a "thundering + # herd" situation when all locks may time out at the same time. + self._timeout_interval = next_interval * random.uniform(0.9, 1.1) + return self._timeout_interval @attr.s(auto_attribs=True, eq=False) @@ -294,10 +323,11 @@ class WaitingMultiLock: store: LockStore handler: WorkerLocksHandler + start_ts_ms: int = 0 deferred: "defer.Deferred[None]" = attr.Factory(defer.Deferred) _inner_lock_cm: AsyncContextManager | None = None - _retry_interval: float = 0.1 + _timeout_interval: float = 0.1 _lock_span: "opentracing.Scope" = attr.Factory( lambda: start_active_span("WaitingLock.lock") ) @@ -309,6 +339,7 @@ class WaitingMultiLock: self.deferred.callback(None) async def __aenter__(self) -> None: + self.start_ts_ms = self.clock.time_msec() self._lock_span.__enter__() with start_active_span("WaitingLock.waiting_for_lock"): @@ -324,19 +355,42 @@ class WaitingMultiLock: break try: - # Wait until the we get notified the lock might have been + # Wait until the notification that the lock might have been # released (by the deferred being resolved). We also - # periodically wake up in case the lock was released but we + # periodically wake up in case the lock was released, but we # weren't notified. with PreserveLoggingContext(): - timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=timeout, + timeout=self._timeout_interval, clock=self.clock, ) - except Exception: - pass + except defer.TimeoutError: + # Only increment the timeout value if this was an actual timeout + # (defer.TimeoutError) + self._increment_timeout_interval() + + now_ms = self.clock.time_msec() + time_spent_trying_to_lock = Duration( + milliseconds=now_ms - self.start_ts_ms + ) + if ( + time_spent_trying_to_lock.as_millis() + > WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION.as_millis() + ): + logger.warning( + "(WaitingMultiLock (%r)) Time spent waiting to acquire lock " + "is getting excessive: %ss. There may be a deadlock.", + self.lock_names, + time_spent_trying_to_lock.as_secs(), + ) + + except Exception as e: + logger.warning( + "Caught an exception while waiting on WaitingMultiLock(lock_names=%r): %r", + self.lock_names, + e, + ) assert self._inner_lock_cm await self._inner_lock_cm.__aenter__() @@ -360,12 +414,11 @@ class WaitingMultiLock: return r - def _get_next_retry_interval(self) -> float: - next = self._retry_interval - self._retry_interval = max(5, next * 2) - if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations - logger.warning( - "Lock timeout is getting excessive: %ss. There may be a deadlock.", - self._retry_interval, - ) - return next * random.uniform(0.9, 1.1) + def _increment_timeout_interval(self) -> float: + next_interval = self._timeout_interval + next_interval = min(WORKER_LOCK_MAX_RETRY_INTERVAL.as_secs(), next_interval * 2) + + # The jitter value is maintained for the timeout, to help avoid a "thundering + # herd" situation when all locks may time out at the same time. + self._timeout_interval = next_interval * random.uniform(0.9, 1.1) + return self._timeout_interval diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index 61ff51ff92..74201f4151 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -26,7 +26,9 @@ from twisted.internet import defer from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer +from synapse.storage.databases.main.lock import _RENEWAL_INTERVAL from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase @@ -40,6 +42,7 @@ class WorkerLockTestCase(unittest.HomeserverTestCase): self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer ) -> None: self.worker_lock_handler = self.hs.get_worker_locks_handler() + self.store = self.hs.get_datastores().main def test_wait_for_lock_locally(self) -> None: """Test waiting for a lock on a single worker""" @@ -56,6 +59,66 @@ class WorkerLockTestCase(unittest.HomeserverTestCase): self.get_success(d2) self.get_success(lock2.__aexit__(None, None, None)) + def test_timeouts_for_lock_locally(self) -> None: + """ + Test that we regularly retry to reacquire locks. + + This is a regression test to make sure the lock retry time doesn't balloon to a value + so large it can't even be printed reliably anymore. + """ + + # Create and acquire the first lock + lock1 = self.worker_lock_handler.acquire_lock("name", "key") + self.get_success(lock1.__aenter__()) + + # Create and try to acquire the second lock + lock2 = self.worker_lock_handler.acquire_lock("name", "key") + d2 = defer.ensureDeferred(lock2.__aenter__()) + # Make sure we haven't acquired the lock yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Advance time by an hour (some duration that would previously cause our timeout + # to balloon if it weren't constrained). Max back-off (saturate) + # + # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an + # internal background looping call that runs every 30 seconds + # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value + # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # drop if this renewal is not allowed to run, which sours the test. + # self.pump(amount=Duration(hours=1)) + self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) + + # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Release the first lock (`lock1`). The second lock(`lock2`) should be + # automatically acquired by the `pump()` inside `get_success()` + self.get_success(lock1.__aexit__(None, None, None)) + + # We should now have the lock + self.successResultOf(d2) + + def _pump_by( + self, + *, + amount: Duration = Duration(seconds=0), + by: Duration = Duration(seconds=0.1), + ) -> None: + """ + Like `self.pump()` but you can specify the time increment to advance with until + you reach the time amount. + + Unlike `self.pump()`, this doesn't multiply the time at all. + + Args: + amount: The amount of time to advance + by: The time increment in seconds to advance time by until we reach the `amount` + """ + end_time_s = self.reactor.seconds() + amount.as_secs() + + while self.reactor.seconds() < end_time_s: + self.reactor.advance(by.as_secs()) + def test_lock_contention(self) -> None: """Test lock contention when a lot of locks wait on a single worker""" nb_locks_to_test = 500 @@ -124,3 +187,70 @@ class WorkerLockWorkersTestCase(BaseMultiWorkerStreamTestCase): self.get_success(d2) self.get_success(lock2.__aexit__(None, None, None)) + + def test_timeouts_for_lock_worker(self) -> None: + """ + Test that we regularly retry to reacquire locks. + + This is a regression test to make sure the lock retry time doesn't balloon to a value + so large it can't even be printed reliably anymore. + """ + worker = self.make_worker_hs( + "synapse.app.generic_worker", + extra_config={ + "redis": {"enabled": True}, + }, + ) + worker_lock_handler = worker.get_worker_locks_handler() + + # Create and acquire the first lock on the main process + lock1 = self.main_worker_lock_handler.acquire_lock("name", "key") + self.get_success(lock1.__aenter__()) + + # Create and try to acquire the second lock on the worker + lock2 = worker_lock_handler.acquire_lock("name", "key") + d2 = defer.ensureDeferred(lock2.__aenter__()) + # Make sure we haven't acquired the lock yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Advance time by an hour (some duration that would previously cause our timeout + # to balloon if it weren't constrained). Max back-off (saturate) + # + # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an + # internal background looping call that runs every 30 seconds + # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value + # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # drop if this renewal is not allowed to run, which sours the test. + # self.pump(amount=Duration(hours=1)) + self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) + + # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Release the first lock (`lock1`). The second lock(`lock2`) should be + # automatically acquired by the `pump()` inside `get_success()` + self.get_success(lock1.__aexit__(None, None, None)) + + # We should now have the lock + self.successResultOf(d2) + + def _pump_by( + self, + *, + amount: Duration = Duration(seconds=0), + by: Duration = Duration(seconds=0.1), + ) -> None: + """ + Like `self.pump()` but you can specify the time increment to advance with until + you reach the time amount. + + Unlike `self.pump()`, this doesn't multiply the time at all. + + Args: + amount: The amount of time to advance + by: The time increment in seconds to advance time by until we reach the `amount` + """ + end_time_s = self.reactor.seconds() + amount.as_secs() + + while self.reactor.seconds() < end_time_s: + self.reactor.advance(by.as_secs()) From 2d48851438d41bf6e248c7a743c8ffd61c2e1ba3 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 5 May 2026 17:12:11 +0100 Subject: [PATCH 06/42] Prevent pagination ending when a page is full of rejected events (ELEMENTSEC-2025-1636) Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v Reviewed-on: https://github.com/element-hq/synapse-private/pull/117 --- synapse/handlers/pagination.py | 11 +-- synapse/storage/databases/main/stream.py | 13 +++- tests/rest/client/test_rooms.py | 86 +++++++++++++++++++++++- 3 files changed, 101 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 8cbe4b63c8..2bc7efeb5e 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -566,7 +566,7 @@ class PaginationHandler: ( events, next_key, - _, + limited, ) = await self.store.paginate_room_events_by_topological_ordering( room_id=room_id, from_key=from_token.room_key, @@ -645,7 +645,7 @@ class PaginationHandler: ( events, next_key, - _, + limited, ) = await self.store.paginate_room_events_by_topological_ordering( room_id=room_id, from_key=from_token.room_key, @@ -668,11 +668,12 @@ class PaginationHandler: next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key) - # if no events are returned from pagination, that implies - # we have reached the end of the available events. + # if no events are returned from pagination (this page is empty) + # and there aren't any more pages (not limited), + # that implies we have reached the end of the available events. # In that case we do not return end, to tell the client # there is no need for further queries. - if not events: + if not limited and not events: return GetMessagesResult( messages_chunk=[], bundled_aggregations={}, diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 8fa1e2e5a9..7d14f9f4d8 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -2425,12 +2425,19 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): event_filter: If provided filters the events to those that match the filter. Returns: - The results as a list of events, a token that points to the end of - the result set, and a boolean to indicate if there were more events - but we hit the limit. If no events are returned then the end of the + - The results as a list of events; + - a token that points to the end of the result set; and + - a boolean to indicate if there were more events + but we hit the limit (`limited`) + + If no events are returned and `limited` is false, then the end of the stream has been reached (i.e. there are no events between `from_key` and `to_key`). + When `limited` is true, that means that more pagination can be attempted. + Note that `limited` can be true even if no events are returned, + because rejected events are filtered out after the limit check. + When Direction.FORWARDS: from_key < x <= to_key, (ascending order) When Direction.BACKWARDS: from_key >= x > to_key, (descending order) """ diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 221121007d..61e7e87f62 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -66,7 +66,10 @@ from synapse.util.stringutils import random_string from tests import unittest from tests.http.server._base import make_request_with_cancellation_test from tests.storage.test_stream import PaginationTestCase -from tests.test_utils.event_injection import create_event +from tests.test_utils.event_injection import ( + create_event, + inject_event, +) from tests.unittest import override_config from tests.utils import default_config @@ -2371,6 +2374,87 @@ class RoomMessageListTestCase(RoomBase): channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body ) + def test_room_messages_paginate_through_rejected_events( + self, + ) -> None: + """Test that pagination continues past a batch of rejected events. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v + + Synapse before 1.152.1 had a bug meaning that a batch full of only + rejected events would cause `/messages` to not return any more + pagination tokens, falsely signalling the end of backpagination. + """ + # Send an early message that should not be filtered. + early_event_id = self.helper.send(self.room_id, "early message")["event_id"] + + # Inject a batch of events and mark them as rejected in the database. + # We create more events than a single pagination request would fetch, + # so that one page of backward pagination request would only see rejected events. + for _ in range(3): + event = self.get_success( + inject_event( + self.hs, + room_id=self.room_id, + sender=self.user_id, + type=EventTypes.Message, + content={"body": "filtered event", "msgtype": "m.text"}, + ) + ) + self.get_success( + self.hs.get_datastores().main.db_pool.runInteraction( + "mark_rejected", + self.hs.get_datastores().main.mark_event_rejected_txn, + event.event_id, + "testing", + ) + ) + + # Send a message after all the rejected events. + latest_event_id = self.helper.send(self.room_id, "latest message")["event_id"] + + # Start backpaginating. + channel = self.make_request( + "GET", f"/rooms/{self.room_id}/messages?dir=b&limit=2" + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + events_in_page = [e["event_id"] for e in channel.json_body["chunk"]] + end_token: str | None = channel.json_body["end"] + + self.assertEqual( + events_in_page, + [latest_event_id], + "The latest event should be included in the first page we see whilst backpaginating", + ) + + event_ids_in_pages: list[list[str]] = [events_in_page] + + # Bound the number of backpagination attempts to 2 + for _ in range(2): + channel = self.make_request( + "GET", f"/rooms/{self.room_id}/messages?from={end_token}&dir=b&limit=2" + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + events_in_page = [e["event_id"] for e in channel.json_body["chunk"]] + event_ids_in_pages.append(events_in_page) + + if early_event_id in events_in_page: + # We have found the event we were looking for + return + + self.assertIn( + "end", + channel.json_body, + f"No `end` token received. Did not find {early_event_id} whilst backpaginating ({latest_event_id = }, {event_ids_in_pages = })", + ) + # Use the end_token in the next iteration + end_token = channel.json_body["end"] + + self.fail( + f"Exhausted backpagination attempts. Did not find {early_event_id} whilst backpaginating ({latest_event_id = }, {event_ids_in_pages = })" + ) + class RoomMessageFilterTestCase(RoomBase): """Tests /rooms/$room_id/messages REST events.""" From d97b5b9e21681792d78c5d58243fc483c664b66e Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 7 May 2026 13:49:49 +0100 Subject: [PATCH 07/42] 1.152.1 --- CHANGES.md | 8 ++++++++ changelog.d/19394.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/19394.bugfix diff --git a/CHANGES.md b/CHANGES.md index 3425bcca8c..d9b3f8b2c1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +# Synapse 1.152.1 (2026-05-07) + +## Security Fixes + +- Prevent CPU starvation (Denial of Service) under worker lock contention, additionally capping the `WorkerLock` time out interval to a maximum of 60 seconds. Contributed by Famedly. ([\#19394](https://github.com/element-hq/synapse/issues/19394), ELEMENTSEC-2026-1706, [GHSA-8q93-326v-3m7g](https://github.com/element-hq/synapse/security/advisories/GHSA-8q93-326v-3m7g), CVE pending) +- Prevent pagination ending when a page is full of rejected events. (ELEMENTSEC-2025-1636, [GHSA-6qf2-7x63-mm6v](https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v), CVE pending) + + # Synapse 1.152.0 (2026-04-28) No significant changes since 1.152.0rc1. diff --git a/changelog.d/19394.bugfix b/changelog.d/19394.bugfix deleted file mode 100644 index 4ca92cfb32..0000000000 --- a/changelog.d/19394.bugfix +++ /dev/null @@ -1 +0,0 @@ -Capped the `WorkerLock` time out interval to a maximum of 60 seconds to prevent dealing with excessively long numbers. Contributed by Famedly. diff --git a/debian/changelog b/debian/changelog index ff9dfe3e13..cfefe953e3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.152.1) stable; urgency=medium + + * New Synapse release 1.152.1. + + -- Synapse Packaging team Thu, 07 May 2026 13:29:05 +0100 + matrix-synapse-py3 (1.152.0) stable; urgency=medium * New Synapse release 1.152.0. diff --git a/pyproject.toml b/pyproject.toml index 9509609ff3..e9ae6e204d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.152.0" +version = "1.152.1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 2829a146d32e472fcd2885ed32cb3bbea6f86690 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 7 May 2026 10:36:25 -0500 Subject: [PATCH 08/42] Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds (#19755) Better to retry more quickly than have workers wait around. 5 seconds is still a reasonable gap in time to not overwhelm anything. This matters most in cross-worker scenarios. When locks are on the same worker, when the lock holder releases, we signal to other locks (with the same name/key) that they should try reacquiring the lock immediately. But locks on other workers only re-check based on their retry `_timeout_interval`. Updating to 5 seconds to match the previous intentions based on the [flawed code](https://github.com/element-hq/synapse/blob/6100f6e4f7fb0c72f1ae2802683ebc811c0e3a77/synapse/handlers/worker_lock.py#L278). We can assume they were trying to have 5 seconds as the max value to retry. Spawning from https://github.com/element-hq/synapse/pull/19394#discussion_r3168458070 --- changelog.d/19755.misc | 1 + synapse/handlers/worker_lock.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19755.misc diff --git a/changelog.d/19755.misc b/changelog.d/19755.misc new file mode 100644 index 0000000000..6ad478e531 --- /dev/null +++ b/changelog.d/19755.misc @@ -0,0 +1 @@ +Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 51be3b5084..57792ea53c 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -54,7 +54,18 @@ logger = logging.getLogger(__name__) # will not disappear under our feet as long as we don't delete the room. NEW_EVENT_DURING_PURGE_LOCK_NAME = "new_event_during_purge_lock" -WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=60) +WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=5) +""" +The maximum wait time before retrying to acquire the lock. + +Better to retry more quickly than have workers wait around. 5 seconds is still a +reasonable gap in time to not overwhelm the CPU/Database. + +This matters most in cross-worker scenarios. When locks are on the same worker, when the +lock holder releases, we signal to other locks (with the same name/key) that they +should try reacquiring the lock immediately. But locks on other workers only re-check +based on their retry `_timeout_interval`. +""" WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) From 4911296fb5e21649aa7d205b2ad9fcc43016d831 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 7 May 2026 10:38:56 -0500 Subject: [PATCH 09/42] Force keyword-only args for `Duration` (prevent footgun) (#19756) So people have to specify which time unit they want to use. Spawning from https://github.com/element-hq/synapse/pull/19394#discussion_r3188418426 --- changelog.d/19756.misc | 1 + synapse/util/duration.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 changelog.d/19756.misc diff --git a/changelog.d/19756.misc b/changelog.d/19756.misc new file mode 100644 index 0000000000..2450505b53 --- /dev/null +++ b/changelog.d/19756.misc @@ -0,0 +1 @@ +Force keyword-only args for `Duration` (prevent footgun) so people have to specify which time unit they want to us. diff --git a/synapse/util/duration.py b/synapse/util/duration.py index 135b980852..a1abe944b5 100644 --- a/synapse/util/duration.py +++ b/synapse/util/duration.py @@ -32,6 +32,33 @@ class Duration(timedelta): ``` """ + # Using `__new__` (instead of `__init__`) because that's what `timedelta` uses + def __new__( + cls, + # The whole goal of overriding `__new__` is to require keyword-only arguments. + # Without this, `Duration(5)` would create a duration represnting 5 *days* + # (timedelta's default), but callers almost certainly want to specify which unit + # like seconds or hours. + *, + days: float = 0, + seconds: float = 0, + microseconds: float = 0, + milliseconds: float = 0, + minutes: float = 0, + hours: float = 0, + weeks: float = 0, + ) -> "Duration": + return super().__new__( + cls, + days=days, + seconds=seconds, + microseconds=microseconds, + milliseconds=milliseconds, + minutes=minutes, + hours=hours, + weeks=weeks, + ) + def as_millis(self) -> int: """Returns the duration in milliseconds.""" return int(self / _ONE_MILLISECOND) From 8dbbc4000b347ed1e4bca494e36dc11c37dd865d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 8 May 2026 06:20:25 -0500 Subject: [PATCH 10/42] Commit stray Rust change that keeps popping up (`rust/src/canonical_json.rs`) (#19763) (introduced in https://github.com/element-hq/synapse/pull/19739) Seems like some automatic change from `poetry run ./scripts-dev/lint.sh` --- changelog.d/19763.misc | 1 + rust/src/canonical_json.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19763.misc diff --git a/changelog.d/19763.misc b/changelog.d/19763.misc new file mode 100644 index 0000000000..c9e8277e01 --- /dev/null +++ b/changelog.d/19763.misc @@ -0,0 +1 @@ +Lint and format `rust/src/canonical_json.rs`. diff --git a/rust/src/canonical_json.rs b/rust/src/canonical_json.rs index ff1fcd3ee4..4abd001847 100644 --- a/rust/src/canonical_json.rs +++ b/rust/src/canonical_json.rs @@ -824,7 +824,7 @@ mod tests { // Serialize the keys in the reverse order. for (c, _) in ascii_order.iter().rev() { - map_serializer.serialize_entry(c.into(), &1).unwrap(); + map_serializer.serialize_entry(c, &1).unwrap(); } SerializeMap::end(map_serializer).unwrap(); From 0e508ba80f157e64aab01bb5fca56a9225d4f540 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 8 May 2026 13:22:15 +0100 Subject: [PATCH 11/42] 1.153.0rc1 --- CHANGES.md | 32 +++++++++++++++++++++++++++++++ changelog.d/18475.feature | 1 - changelog.d/19394.bugfix | 1 - changelog.d/19398.bugfix | 1 - changelog.d/19611.bugfix | 1 - changelog.d/19706.misc | 1 - changelog.d/19708.misc | 1 - changelog.d/19711.doc | 1 - changelog.d/19714.bugfix | 1 - changelog.d/19720.feature | 1 - changelog.d/19722.feature | 1 - changelog.d/19727.bugfix | 1 - changelog.d/19737.feature | 1 - changelog.d/19739.misc | 1 - changelog.d/19742.bugfix | 1 - changelog.d/19743.misc | 1 - changelog.d/19755.misc | 1 - changelog.d/19756.misc | 1 - changelog.d/19763.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 22 files changed, 40 insertions(+), 20 deletions(-) delete mode 100644 changelog.d/18475.feature delete mode 100644 changelog.d/19394.bugfix delete mode 100644 changelog.d/19398.bugfix delete mode 100644 changelog.d/19611.bugfix delete mode 100644 changelog.d/19706.misc delete mode 100644 changelog.d/19708.misc delete mode 100644 changelog.d/19711.doc delete mode 100644 changelog.d/19714.bugfix delete mode 100644 changelog.d/19720.feature delete mode 100644 changelog.d/19722.feature delete mode 100644 changelog.d/19727.bugfix delete mode 100644 changelog.d/19737.feature delete mode 100644 changelog.d/19739.misc delete mode 100644 changelog.d/19742.bugfix delete mode 100644 changelog.d/19743.misc delete mode 100644 changelog.d/19755.misc delete mode 100644 changelog.d/19756.misc delete mode 100644 changelog.d/19763.misc diff --git a/CHANGES.md b/CHANGES.md index d9b3f8b2c1..da26f17606 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,35 @@ +# Synapse 1.153.0rc1 (2026-05-08) + +## Features + +- Make ACLs apply to EDUs per [MSC4163](https://github.com/matrix-org/matrix-spec-proposals/pull/4163). ([\#18475](https://github.com/element-hq/synapse/issues/18475)) +- Stabilize [MSC3266: Room summary API](https://github.com/matrix-org/matrix-spec-proposals/pull/3266), removing the experimental config flag `msc3266_enabled`. Contributed by @dasha-uwu. ([\#19720](https://github.com/element-hq/synapse/issues/19720)) +- Partial [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) implementation: `m.room.create` is now a required part of stripped `invite_state`/`knock_state` . Contributed by @FrenchGithubUser @Famedly. ([\#19722](https://github.com/element-hq/synapse/issues/19722)) +- Expose `tombstoned` and `replacement_room` in room details on admin API endpoint `GET /_synapse/admin/v1/rooms/`. Contributed by Noah Markert. ([\#19737](https://github.com/element-hq/synapse/issues/19737)) + +## Bugfixes + +- Allow self-requested user erasure (upon account deactivation) to succeed even if Synapse has disabled profile changes. Contributed by Famedly. ([\#19398](https://github.com/element-hq/synapse/issues/19398)) +- Fix Synapse not backfilling new history when attempting to use a pagination token near a backward extremity. ([\#19611](https://github.com/element-hq/synapse/issues/19611)) +- Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714)) +- Fix a bug where when upgrading a room to room version 12, the power level event in the old room got temporarily mutated to remove the user upgrading the room's power. ([\#19727](https://github.com/element-hq/synapse/issues/19727)) +- Fix packaging for Fedora and EPEL caused by unnecessary bumping `authlib` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. ([\#19742](https://github.com/element-hq/synapse/issues/19742)) + +## Improved Documentation + +- Add warning about known problems when configuring `use_frozen_dicts`. ([\#19711](https://github.com/element-hq/synapse/issues/19711)) + +## Internal Changes + +- Port `Event.signatures` field to Rust. ([\#19706](https://github.com/element-hq/synapse/issues/19706)) +- Port `Event.unsigned` field to Rust. ([\#19708](https://github.com/element-hq/synapse/issues/19708)) +- Add a Rust canonical JSON serializer. ([\#19739](https://github.com/element-hq/synapse/issues/19739)) +- Configure Dependabot to only update Python dependencies in the lockfile, unless widening upper bounds. ([\#19743](https://github.com/element-hq/synapse/issues/19743)) +- Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. ([\#19755](https://github.com/element-hq/synapse/issues/19755)) +- Force keyword-only arguments for `Duration` so time units have to be specified. ([\#19756](https://github.com/element-hq/synapse/issues/19756)) +- Lint and format `rust/src/canonical_json.rs`. ([\#19763](https://github.com/element-hq/synapse/issues/19763)) + + # Synapse 1.152.1 (2026-05-07) ## Security Fixes diff --git a/changelog.d/18475.feature b/changelog.d/18475.feature deleted file mode 100644 index 06c13db43e..0000000000 --- a/changelog.d/18475.feature +++ /dev/null @@ -1 +0,0 @@ -Make ACLs apply to EDUs per [MSC4163](https://github.com/matrix-org/matrix-spec-proposals/pull/4163). diff --git a/changelog.d/19394.bugfix b/changelog.d/19394.bugfix deleted file mode 100644 index 4ca92cfb32..0000000000 --- a/changelog.d/19394.bugfix +++ /dev/null @@ -1 +0,0 @@ -Capped the `WorkerLock` time out interval to a maximum of 60 seconds to prevent dealing with excessively long numbers. Contributed by Famedly. diff --git a/changelog.d/19398.bugfix b/changelog.d/19398.bugfix deleted file mode 100644 index 07679b31ae..0000000000 --- a/changelog.d/19398.bugfix +++ /dev/null @@ -1 +0,0 @@ -Allow user requested erasure to succeed even if Synapse has disabled profile changes. Contributed by Famedly. diff --git a/changelog.d/19611.bugfix b/changelog.d/19611.bugfix deleted file mode 100644 index 4952fd00db..0000000000 --- a/changelog.d/19611.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix Synapse not backfilling new history when attempting to use a pagination token near a backward extremity. diff --git a/changelog.d/19706.misc b/changelog.d/19706.misc deleted file mode 100644 index 205abd09d4..0000000000 --- a/changelog.d/19706.misc +++ /dev/null @@ -1 +0,0 @@ -Port `Event.signatures` field to Rust. diff --git a/changelog.d/19708.misc b/changelog.d/19708.misc deleted file mode 100644 index 308c2b04d0..0000000000 --- a/changelog.d/19708.misc +++ /dev/null @@ -1 +0,0 @@ -Port `Event.unsigned` field to Rust. diff --git a/changelog.d/19711.doc b/changelog.d/19711.doc deleted file mode 100644 index d00ee6a908..0000000000 --- a/changelog.d/19711.doc +++ /dev/null @@ -1 +0,0 @@ -Add warning about known problems when configuring `use_frozen_dicts`. diff --git a/changelog.d/19714.bugfix b/changelog.d/19714.bugfix deleted file mode 100644 index 6aba7b21a6..0000000000 --- a/changelog.d/19714.bugfix +++ /dev/null @@ -1 +0,0 @@ -Have SSS return a new response immediately if a room subscription have changed and produced a new response. diff --git a/changelog.d/19720.feature b/changelog.d/19720.feature deleted file mode 100644 index 97a1d35de5..0000000000 --- a/changelog.d/19720.feature +++ /dev/null @@ -1 +0,0 @@ -Stabilize MSC3266, removing the experimental config flag `msc3266_enabled`. Add support for stable room summary endpoints. Contributed by @dasha-uwu. diff --git a/changelog.d/19722.feature b/changelog.d/19722.feature deleted file mode 100644 index 30104b7e74..0000000000 --- a/changelog.d/19722.feature +++ /dev/null @@ -1 +0,0 @@ -Partial [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) implementation: `m.room.create` is now a required part of stripped `invite_state`/`knock_state` . Contributed by @FrenchGithubUser @Famedly. diff --git a/changelog.d/19727.bugfix b/changelog.d/19727.bugfix deleted file mode 100644 index a535bd6aa4..0000000000 --- a/changelog.d/19727.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where when upgrading a room to v12 the power level event in the old room got mutated to remove the user upgrading the room's power. diff --git a/changelog.d/19737.feature b/changelog.d/19737.feature deleted file mode 100644 index 13bf2405df..0000000000 --- a/changelog.d/19737.feature +++ /dev/null @@ -1 +0,0 @@ -Exposes `tombstoned` and `replacement_room` in room details on admin API endpoint `GET /_synapse/admin/v1/rooms/`. Contributed by Noah Markert. diff --git a/changelog.d/19739.misc b/changelog.d/19739.misc deleted file mode 100644 index 24562b24fc..0000000000 --- a/changelog.d/19739.misc +++ /dev/null @@ -1 +0,0 @@ -Add a Rust canonical JSON serializer. diff --git a/changelog.d/19742.bugfix b/changelog.d/19742.bugfix deleted file mode 100644 index 342769b65b..0000000000 --- a/changelog.d/19742.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix packaging for Fedora and EPEL caused by unnecessary bumping `authlib` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. diff --git a/changelog.d/19743.misc b/changelog.d/19743.misc deleted file mode 100644 index 35c4841386..0000000000 --- a/changelog.d/19743.misc +++ /dev/null @@ -1 +0,0 @@ -Configure Dependabot to only update Python dependencies in the lockfile, unless widening upper bounds. diff --git a/changelog.d/19755.misc b/changelog.d/19755.misc deleted file mode 100644 index 6ad478e531..0000000000 --- a/changelog.d/19755.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. diff --git a/changelog.d/19756.misc b/changelog.d/19756.misc deleted file mode 100644 index 2450505b53..0000000000 --- a/changelog.d/19756.misc +++ /dev/null @@ -1 +0,0 @@ -Force keyword-only args for `Duration` (prevent footgun) so people have to specify which time unit they want to us. diff --git a/changelog.d/19763.misc b/changelog.d/19763.misc deleted file mode 100644 index c9e8277e01..0000000000 --- a/changelog.d/19763.misc +++ /dev/null @@ -1 +0,0 @@ -Lint and format `rust/src/canonical_json.rs`. diff --git a/debian/changelog b/debian/changelog index cfefe953e3..6aa3735603 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.153.0~rc1) stable; urgency=medium + + * New Synapse release 1.153.0rc1. + + -- Synapse Packaging team Fri, 08 May 2026 13:05:08 +0100 + matrix-synapse-py3 (1.152.1) stable; urgency=medium * New Synapse release 1.152.1. diff --git a/pyproject.toml b/pyproject.toml index 7ead67c8f5..b456adfc60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.152.1" +version = "1.153.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 49207028d1..8888d2c673 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.152/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.153/synapse-config.schema.json type: object properties: modules: From eb2ae9d3da6eb765836e83c37781a2fbba98479e Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 8 May 2026 14:03:41 +0100 Subject: [PATCH 12/42] Tweak changelog --- CHANGES.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index da26f17606..e7bd8f351a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,11 +23,10 @@ - Port `Event.signatures` field to Rust. ([\#19706](https://github.com/element-hq/synapse/issues/19706)) - Port `Event.unsigned` field to Rust. ([\#19708](https://github.com/element-hq/synapse/issues/19708)) -- Add a Rust canonical JSON serializer. ([\#19739](https://github.com/element-hq/synapse/issues/19739)) +- Add a Rust canonical JSON serializer. ([\#19739](https://github.com/element-hq/synapse/issues/19739), [\#19763](https://github.com/element-hq/synapse/issues/19763)) - Configure Dependabot to only update Python dependencies in the lockfile, unless widening upper bounds. ([\#19743](https://github.com/element-hq/synapse/issues/19743)) - Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. ([\#19755](https://github.com/element-hq/synapse/issues/19755)) - Force keyword-only arguments for `Duration` so time units have to be specified. ([\#19756](https://github.com/element-hq/synapse/issues/19756)) -- Lint and format `rust/src/canonical_json.rs`. ([\#19763](https://github.com/element-hq/synapse/issues/19763)) # Synapse 1.152.1 (2026-05-07) From c430c16df47229f8ecef6783739accc042fcafbe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 8 May 2026 14:19:03 +0100 Subject: [PATCH 13/42] Port event content to Rust (#19725) Based on #19708. This is on the path to porting the entire event class to Rust, as `event.content` will then return the new Rust class `JsonObject`. This PR adds a pure Rust `JsonObject` class that is a `Mapping` representing a json-style object. It uses `serde_json::Value` as its in-memory representation and `pythonize` for conversion when a field is looked up on the object. I'm not thrilled with the name, but couldn't think of a better one. This also adds `JsonObject` handling to the JSON serialisation functions we use, as well as to the `freeze(..)` function. Reviewable commit-by-commit. --- changelog.d/19725.misc | 1 + rust/src/events/json_object.rs | 488 +++++++++++++++++++++++ rust/src/events/mod.rs | 12 +- synapse/__init__.py | 9 +- synapse/events/__init__.py | 53 +-- synapse/events/utils.py | 2 +- synapse/events/validator.py | 4 +- synapse/handlers/room.py | 9 +- synapse/handlers/stats.py | 4 +- synapse/push/bulk_push_rule_evaluator.py | 4 +- synapse/synapse_rust/events.pyi | 11 +- synapse/util/events.py | 4 +- synapse/util/frozenutils.py | 5 + synapse/util/json.py | 30 +- tests/crypto/test_event_signing.py | 7 +- tests/module_api/test_api.py | 4 +- tests/rest/client/test_rooms.py | 4 +- tests/server.py | 3 +- tests/synapse_rust/test_json_object.py | 149 +++++++ tests/test_state.py | 5 +- tests/test_utils/__init__.py | 4 +- 21 files changed, 749 insertions(+), 63 deletions(-) create mode 100644 changelog.d/19725.misc create mode 100644 rust/src/events/json_object.rs create mode 100644 tests/synapse_rust/test_json_object.py diff --git a/changelog.d/19725.misc b/changelog.d/19725.misc new file mode 100644 index 0000000000..b320f42b9c --- /dev/null +++ b/changelog.d/19725.misc @@ -0,0 +1 @@ +Port `Event.content` field to Rust. diff --git a/rust/src/events/json_object.rs b/rust/src/events/json_object.rs new file mode 100644 index 0000000000..2c4be1c87b --- /dev/null +++ b/rust/src/events/json_object.rs @@ -0,0 +1,488 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::{collections::BTreeMap, sync::Arc}; + +use pyo3::{ + exceptions::{PyKeyError, PyTypeError}, + pyclass, pymethods, + types::{ + PyAnyMethods, PyIterator, PyList, PyListMethods, PyMapping, PySet, PySetMethods, PyTuple, + }, + Bound, IntoPyObject, IntoPyObjectExt, Py, PyAny, PyResult, Python, +}; +use pythonize::{depythonize, pythonize}; +use serde::{Deserialize, Serialize}; + +/// A generic class for representing immutable JSON objects. +/// +/// This is used for representing the `content` field of an event. +/// +/// The basic architecture here is to optimize for two things: +/// 1. Fast access of top-level keys (e.g. `event.content["key"]`) +/// 2. Pure Rust implementation. +#[derive(Serialize, Deserialize, Clone, Default)] +#[pyclass(mapping, frozen)] +#[serde(transparent)] +pub struct JsonObject { + object: Arc, serde_json::Value>>, +} + +#[pymethods] +impl JsonObject { + #[new] + #[pyo3(signature = (content = None))] + fn new<'a, 'py>(content: Option<&'a Bound<'py, PyAny>>) -> PyResult { + let Some(content) = content else { + // If no content is provided, default to an empty object. + return Ok(Self::default()); + }; + + if let Ok(content) = content.cast::() { + // If the content is already a JsonObject, we can just clone the + // underlying map (this is safe as the object is immutable). + return Ok(JsonObject { + object: content.get().object.clone(), + }); + } + + let Ok(content) = content.cast::() else { + return Err(PyTypeError::new_err("'content' must be a mapping")); + }; + + // Use pythonize to try and convert from a mapping. + let content = depythonize(content)?; + Ok(Self { + object: Arc::new(content), + }) + } + + fn __len__(&self) -> usize { + self.object.len() + } + + fn __contains__(&self, key: &Bound<'_, PyAny>) -> bool { + // Match dict semantics: a non-string key is simply "not in" the + // mapping, rather than raising TypeError. + let Ok(key_str) = key.extract::<&str>() else { + return false; + }; + self.object.contains_key(key_str) + } + + fn __getitem__<'py>( + &self, + py: Python<'py>, + key: Bound<'_, PyAny>, + ) -> PyResult> { + // We only ever store string keys, so any non-string lookup is a miss. + // Raise KeyError (not TypeError) to match dict's behaviour. + let Ok(key_str) = key.extract::<&str>() else { + return Err(PyKeyError::new_err(key.unbind())); + }; + let Some(value) = self.object.get(key_str) else { + return Err(PyKeyError::new_err(key.unbind())); + }; + Ok(pythonize(py, value)?) + } + + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // The easiest way to get an iterator over the keys is to create a + // temporary list and call `iter()` on it. This is not the most + // efficient approach, but is much less boilerplate than implementing a + // custom iterator type. Since the keys are typically small in number + // this should be fine in practice. + let list = PyList::new(py, self.object.keys().map(Box::as_ref))?; + PyIterator::from_object(&list) + } + + // The view classes below each hold a `JsonObject` clone. This is cheap + // because the underlying map is behind an `Arc`, and lets the view outlive + // the originating object (matching dict_keys/values/items semantics in + // Python, which also keep the dict alive). + + fn keys(&self) -> JsonObjectKeysView { + JsonObjectKeysView { + object: self.clone(), + } + } + + fn values(&self) -> JsonObjectValuesView { + JsonObjectValuesView { + object: self.clone(), + } + } + + fn items(&self) -> JsonObjectItemsView { + JsonObjectItemsView { + object: self.clone(), + } + } + + #[pyo3(signature = (key, default=None))] + fn get<'py>( + &self, + py: Python<'py>, + key: Bound<'_, PyAny>, + default: Option>, + ) -> PyResult> { + // Non-string keys can never match, so treat them as a miss and return + // the caller-supplied default rather than raising. + let Ok(key_str) = key.extract::<&str>() else { + return Ok(default.into_pyobject(py)?); + }; + match self.object.get(key_str) { + Some(value) => Ok(pythonize(py, value)?), + None => Ok(default.into_pyobject(py)?), + } + } + + fn __eq__(&self, other: Bound<'_, PyAny>) -> bool { + // We support equality against any Python mapping (e.g. plain dicts), + // so callers can swap a JsonObject in without rewriting comparisons. + let Ok(mapping) = other.cast::() else { + return false; + }; + + let Ok(other_len) = mapping.len() else { + return false; + }; + + if other_len != self.object.len() { + return false; + } + + // We know the "other" is a mapping with the same number of fields as + // us. So we can convert it into a JsonObject and compare the underlying + // maps. + let Ok(other_dict) = depythonize(&other) else { + return false; + }; + + *self.object == other_dict + } + + // Since we implement comparisons with other types, we need to disable + // hashing to avoid violating the invariant that equal objects must have the + // same hash. + // + // Alternatively, we could only allow comparisons with other JsonObjects and + // allow hashing, but a) its nicer to be able to compare with arbitrary + // mappings and b) we don't really need hashing for these objects. + #[classattr] + const __hash__: Option> = None; + + fn __str__(&self) -> String { + serde_json::to_string(&self.object).expect("Value should be serializable") + } + + fn __repr__(&self) -> String { + format!("JsonObject({})", self.__str__()) + } +} + +/// Helper class returned by `JsonObject.keys()` to act as a view into the keys +/// of the object. +/// +/// This needs to both be iterable *and* operate like a set. +#[pyclass(frozen)] +#[derive(Clone)] +pub struct JsonObjectKeysView { + object: JsonObject, +} + +#[pymethods] +impl JsonObjectKeysView { + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // Create the iterator by making a temporary python list of the keys and + // calling `iter()` on it. + let list = PyList::new(py, self.object.object.keys().map(Box::as_ref))?; + PyIterator::from_object(&list) + } + + fn __len__(&self) -> usize { + self.object.__len__() + } + + fn __contains__(&self, key: &Bound<'_, PyAny>) -> bool { + self.object.__contains__(key) + } + + fn __eq__(&self, other: Bound<'_, PyAny>) -> bool { + let other_len = match other.len() { + Ok(len) => len, + Err(_) => return false, + }; + + if self.object.__len__() != other_len { + return false; + } + + for key in self.object.object.keys() { + if !matches!(other.contains(key.as_ref()), Ok(true)) { + return false; + } + } + + true + } + + // The set operators below match the behaviour of `dict.keys()` in Python: + // they accept any object that supports `__contains__` (for `&`) or is + // iterable (for `|`, `-`, `^`), not just sets. Each returns a fresh + // `PySet` so the caller gets a normal mutable Python set back. + // + // The `__r*__` variants are reflected operators, called by Python when + // the left-hand operand doesn't know how to combine with us. Since these + // operations are commutative for sets (or symmetric in the case of `^`), + // they just delegate. The asymmetric ops (`-`) need a separate impl. + + fn __and__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // Iterate our (typically small) key set and probe `other`, which may + // be any container supporting `__contains__`. + let mut result = Vec::new(); + + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + result.push(key.as_ref()); + } + } + + PySet::new(py, &result) + } + + fn __rand__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + self.__and__(py, other) + } + + fn __or__<'py>(&self, py: Python<'py>, other: Bound<'_, PyAny>) -> PyResult> { + // Union needs to enumerate both sides, so the right operand must be + // iterable (a bare `__contains__` is not enough). + let Ok(other_iter) = other.try_iter() else { + return Err(PyTypeError::new_err("Right operand must be iterable")); + }; + + let result = PySet::new(py, self.object.object.keys().map(Box::as_ref))?; + + // PySet handles dedup, so we can blindly add every element from the + // other iterable. + for item in other_iter { + let item = item?; + result.add(item)?; + } + + Ok(result) + } + + fn __ror__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + self.__or__(py, other) + } + + fn __sub__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // `self - other`: keep our keys that are not in `other`. Only `other` + // needs to support `__contains__` here. + let mut result = Vec::new(); + + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + continue; + } + result.push(key.as_ref()); + } + + PySet::new(py, &result) + } + + fn __rsub__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // `other - self`: we need to enumerate `other`, so it must be + // iterable. Not symmetric with `__sub__`, hence a separate impl. + let Ok(other_iter) = other.try_iter() else { + return Err(PyTypeError::new_err("Left operand must be iterable")); + }; + + let result = PySet::empty(py)?; + + for item in other_iter { + let item = item?; + if self.object.__contains__(&item) { + continue; + } + result.add(item)?; + } + + Ok(result) + } + + fn __xor__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // Symmetric difference: elements in exactly one side. Implemented as + // two filtered passes — one over our keys, one over `other`. + let Ok(other_iter) = other.try_iter() else { + return Err(PyTypeError::new_err("Right operand must be iterable")); + }; + + let result = PySet::empty(py)?; + + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + continue; + } + result.add(key.as_ref())?; + } + + for item in other_iter { + let item = item?; + if self.object.__contains__(&item) { + continue; + } + result.add(item)?; + } + + Ok(result) + } + + fn __rxor__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + self.__xor__(py, other) + } + + fn isdisjoint(&self, other: Bound<'_, PyAny>) -> bool { + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + return false; + } + } + + true + } +} + +/// Helper class returned by `JsonObject.values()` to act as a view into the +/// values of the object. +#[pyclass(frozen)] +#[derive(Clone)] +pub struct JsonObjectValuesView { + object: JsonObject, +} + +#[pymethods] +impl JsonObjectValuesView { + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // Create the iterator by making a temporary python list of the keys and + // calling `iter()` on it. + let list = PyList::empty(py); + for v in self.object.object.values() { + let py_value = pythonize(py, v)?.into_bound_py_any(py)?; + list.append(py_value)?; + } + + PyIterator::from_object(&list) + } + + fn __len__(&self) -> usize { + self.object.__len__() + } + + fn __contains__(&self, other: Bound<'_, PyAny>) -> bool { + // We compare by JSON equality rather than Python identity: convert + // the candidate into a `serde_json::Value` once and scan our values. + // Anything that fails to depythonize cannot match by definition. + let other_value: serde_json::Value = match depythonize(&other) { + Ok(v) => v, + Err(_) => return false, + }; + self.object.object.values().any(|v| *v == other_value) + } +} + +/// Helper class returned by `JsonObject.items()` to act as a view into the +/// items of the object. +/// +/// Technically this should be a set-like view according to Python semantics, +/// unless the values are unhashable. Since the values are immutable we could +/// support it, but it's more work and nobody seems to actually use the set +/// operations on `dict_items` in practice. +#[pyclass(frozen)] +#[derive(Clone)] +pub struct JsonObjectItemsView { + object: JsonObject, +} + +#[pymethods] +impl JsonObjectItemsView { + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // Create the iterator by making a temporary python list of the keys and + // calling `iter()` on it. + let list = PyList::empty(py); + for (k, v) in self.object.object.iter() { + let py_key = k.as_ref().into_bound_py_any(py)?; + let py_value = pythonize(py, v)?.into_bound_py_any(py)?; + let item = PyTuple::new(py, [py_key, py_value])?; + list.append(item)?; + } + + PyIterator::from_object(&list) + } + + fn __len__(&self) -> usize { + self.object.__len__() + } + + fn __contains__(&self, other: Bound<'_, PyAny>) -> bool { + // `(key, value) in items` — only a 2-tuple can possibly match. We + // look the key up directly (avoiding a full scan) and then compare + // the stored value against `value` using JSON equality. + let Ok((key, value)) = other.extract::<(Bound<'_, PyAny>, Bound<'_, PyAny>)>() else { + return false; + }; + let Ok(key_str) = key.extract::<&str>() else { + return false; + }; + let Some(stored) = self.object.object.get(key_str) else { + return false; + }; + let other_value: serde_json::Value = match depythonize(&value) { + Ok(v) => v, + Err(_) => return false, + }; + *stored == other_value + } +} diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 5f505abb91..e60cdb7078 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -21,21 +21,31 @@ //! Classes for representing Events. use pyo3::{ - types::{PyAnyMethods, PyModule, PyModuleMethods}, + types::{PyAnyMethods, PyMapping, PyModule, PyModuleMethods}, wrap_pyfunction, Bound, PyResult, Python, }; pub mod filter; mod internal_metadata; +mod json_object; pub mod signatures; pub mod unsigned; +use json_object::JsonObject; + /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + // Register the `JsonObject` class as a `Mapping` so that `isinstance` works. + PyMapping::register::(py)?; + let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; m.add_submodule(&child_module)?; diff --git a/synapse/__init__.py b/synapse/__init__.py index 2bed060878..3acfc1a0d7 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -65,7 +65,8 @@ try: except ImportError: pass -# Teach canonicaljson how to serialise immutabledicts. + +# Teach canonicaljson how to serialise immutabledicts and JsonObjects. try: from canonicaljson import register_preserialisation_callback from immutabledict import immutabledict @@ -79,6 +80,12 @@ try: return dict(d) register_preserialisation_callback(immutabledict, _immutabledict_cb) + + # Teach canonicaljson how to serialise JsonObjects, which is just to + # convert them to dicts. + from synapse.synapse_rust.events import JsonObject # noqa: E402 + + register_preserialisation_callback(JsonObject, lambda obj: dict(obj)) except ImportError: pass diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 0f850d19b1..5be0298c30 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -44,9 +44,15 @@ from synapse.api.constants import ( StickyEvent, ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import EventInternalMetadata, Signatures, Unsigned +from synapse.synapse_rust.events import ( + EventInternalMetadata, + JsonObject, + Signatures, + Unsigned, +) from synapse.types import ( JsonDict, + JsonMapping, StateKey, StrCollection, ) @@ -206,17 +212,29 @@ class EventBase(metaclass=abc.ABCMeta): ): assert room_version.event_format == self.format_version + if "content" in event_dict: + event_dict["content"] = JsonObject(event_dict["content"]) + + # We intern these strings because they turn up a lot (especially when + # caching). + event_dict = intern_dict(event_dict) + + if USE_FROZEN_DICTS: + frozen_dict = freeze(event_dict) + else: + frozen_dict = event_dict + self.room_version = room_version self.signatures = Signatures(signatures) self.unsigned = Unsigned(unsigned) self.rejected_reason = rejected_reason - self._dict = event_dict + self._dict = frozen_dict self.internal_metadata = EventInternalMetadata(internal_metadata_dict) depth: DictProperty[int] = DictProperty("depth") - content: DictProperty[JsonDict] = DictProperty("content") + content: DictProperty[JsonMapping] = DictProperty("content") hashes: DictProperty[dict[str, str]] = DictProperty("hashes") origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts") sender: DictProperty[str] = DictProperty("sender") @@ -259,7 +277,14 @@ class EventBase(metaclass=abc.ABCMeta): def get_dict(self) -> JsonDict: """Convert the event to a dictionary suitable for serialisation.""" + d = dict(self._dict) + if "content" in d: + # Convert the content (which is a JsonObject) back to a dict. Json + # serialization should handle JsonObjects fine, but for sanities + # sake we want `get_dict()` and `get_pdu_json()` to return plain + # dicts. + d["content"] = dict(self.content) d.update( { "signatures": self.signatures.as_dict(), @@ -419,19 +444,10 @@ class FrozenEvent(EventBase): unsigned = event_dict.pop("unsigned", {}) - # We intern these strings because they turn up a lot (especially when - # caching). - event_dict = intern_dict(event_dict) - - if USE_FROZEN_DICTS: - frozen_dict = freeze(event_dict) - else: - frozen_dict = event_dict - self._event_id = event_dict["event_id"] super().__init__( - frozen_dict, + event_dict, room_version=room_version, signatures=signatures, unsigned=unsigned, @@ -473,19 +489,10 @@ class FrozenEventV2(EventBase): unsigned = event_dict.pop("unsigned", {}) - # We intern these strings because they turn up a lot (especially when - # caching). - event_dict = intern_dict(event_dict) - - if USE_FROZEN_DICTS: - frozen_dict = freeze(event_dict) - else: - frozen_dict = event_dict - self._event_id: str | None = None super().__init__( - frozen_dict, + event_dict, room_version=room_version, signatures=signatures, unsigned=unsigned, diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 926c81b83d..adbede7f16 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -1032,7 +1032,7 @@ def strip_event(event: EventBase) -> JsonDict: return { "type": event.type, "state_key": event.state_key, - "content": event.content, + "content": dict(event.content), "sender": event.sender, } diff --git a/synapse/events/validator.py b/synapse/events/validator.py index ff22b2287f..d1b5152d77 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -42,7 +42,7 @@ from synapse.events.utils import ( ) from synapse.http.servlet import validate_json_object from synapse.storage.controllers.state import server_acl_evaluator_from_event -from synapse.types import EventID, JsonDict, RoomID, StrCollection, UserID +from synapse.types import EventID, JsonDict, JsonMapping, RoomID, StrCollection, UserID from synapse.types.rest import RequestBodyModel @@ -245,7 +245,7 @@ class EventValidator: self._ensure_state_event(event) - def _ensure_strings(self, d: JsonDict, keys: StrCollection) -> None: + def _ensure_strings(self, d: JsonMapping, keys: StrCollection) -> None: for s in keys: if s not in d: raise SynapseError(400, "'%s' not in content" % (s,)) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6a5c76c667..13647caa2a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -706,12 +706,12 @@ class RoomCreationHandler: spam_check = await self._spam_checker_module_callbacks.user_may_create_room( user_id, { - "creation_content": creation_content, + "creation_content": dict(creation_content), "initial_state": [ { "type": state_key[0], "state_key": state_key[1], - "content": event_content, + "content": dict(event_content), } for state_key, event_content in initial_state.items() ], @@ -1437,7 +1437,7 @@ class RoomCreationHandler: room_config: JsonDict, invite_list: list[str], initial_state: MutableStateMap, - creation_content: JsonDict, + creation_content: JsonMapping, room_alias: RoomAlias | None = None, power_level_content_override: JsonDict | None = None, creator_join_profile: JsonDict | None = None, @@ -1508,7 +1508,7 @@ class RoomCreationHandler: async def create_event( etype: str, - content: JsonDict, + content: JsonMapping, for_batch: bool, **kwargs: Any, ) -> tuple[EventBase, synapse.events.snapshot.UnpersistedEventContextBase]: @@ -1561,6 +1561,7 @@ class RoomCreationHandler: if creation_event_with_context is None: # MSC2175 removes the creator field from the create event. if not room_version.implicit_room_creator: + creation_content = dict(creation_content) creation_content["creator"] = creator_id creation_event, unpersisted_creation_context = await create_event( EventTypes.Create, creation_content, False diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index c87b5f854a..4fd69262b2 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -31,7 +31,7 @@ from typing import ( from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions from synapse.storage.databases.main.state_deltas import StateDelta -from synapse.types import JsonDict +from synapse.types import JsonMapping from synapse.util.duration import Duration from synapse.util.events import get_plain_text_topic_from_event_content @@ -195,7 +195,7 @@ class StatsHandler: ) continue - event_content: JsonDict = {} + event_content: JsonMapping = {} if delta.event_id is not None: event = await self.store.get_event(delta.event_id, allow_none=True) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 7cf89200a8..03dd341744 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -51,7 +51,7 @@ from synapse.storage.databases.main.roommember import EventIdMembership from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import ProfileInfo from synapse.synapse_rust.push import FilteredPushRules, PushRuleEvaluator -from synapse.types import JsonValue +from synapse.types import JsonMapping, JsonValue from synapse.types.state import StateFilter from synapse.util import unwrapFirstError from synapse.util.async_helpers import gather_results @@ -231,7 +231,7 @@ class BulkPushRuleEvaluator: event: EventBase, context: EventContext, event_id_to_event: Mapping[str, EventBase], - ) -> tuple[dict, int | None]: + ) -> tuple[JsonMapping, int | None]: """ Given an event and an event context, get the power level event relevant to the event and the power level of the sender of the event. diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 5ae2bb880a..5b55d47f0d 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -10,7 +10,7 @@ # See the GNU Affero General Public License for more details: # . -from typing import Any, Mapping +from typing import Any, Iterator, Mapping from synapse.types import JsonDict, JsonMapping @@ -213,3 +213,12 @@ class Unsigned: def for_event(self) -> JsonDict: ... """Return a dict of all unsigned fields, including those only kept in memory, suitable for inclusion in an event.""" + +class JsonObject(Mapping[str, Any]): + """Immutable JSON object mapping.""" + + def __init__(self, content_dict: JsonMapping | None = None): ... + def __len__(self) -> int: ... + def __getitem__(self, key: str) -> Any: ... + def __iter__(self) -> Iterator[str]: ... + def __eq__(self, other: object) -> bool: ... diff --git a/synapse/util/events.py b/synapse/util/events.py index 19eca1c1ae..e7c1c83a37 100644 --- a/synapse/util/events.py +++ b/synapse/util/events.py @@ -17,7 +17,7 @@ from typing import Any from pydantic import Field, StrictStr, ValidationError, field_validator -from synapse.types import JsonDict +from synapse.types import JsonMapping from synapse.util.pydantic_models import ParseModel from synapse.util.stringutils import random_string @@ -103,7 +103,7 @@ class TopicContent(ParseModel): return None -def get_plain_text_topic_from_event_content(content: JsonDict) -> str | None: +def get_plain_text_topic_from_event_content(content: JsonMapping) -> str | None: """ Given the `content` of an `m.room.topic` event, returns the plain-text topic representation. Prefers pulling plain-text from the newer `m.topic` field if diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 0bc27410c6..92c03690f2 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -23,6 +23,8 @@ from typing import Any from immutabledict import immutabledict +from synapse.synapse_rust.events import JsonObject + def freeze(o: Any) -> Any: if isinstance(o, dict): @@ -31,6 +33,9 @@ def freeze(o: Any) -> Any: if isinstance(o, immutabledict): return o + if isinstance(o, JsonObject): + return o + if isinstance(o, (bytes, str)): return o diff --git a/synapse/util/json.py b/synapse/util/json.py index b1091704a8..8f8d731c6d 100644 --- a/synapse/util/json.py +++ b/synapse/util/json.py @@ -20,24 +20,30 @@ from typing import ( from immutabledict import immutabledict +from synapse.synapse_rust.events import JsonObject + def _reject_invalid_json(val: Any) -> None: """Do not allow Infinity, -Infinity, or NaN values in JSON.""" raise ValueError("Invalid JSON value: '%s'" % val) -def _handle_immutabledict(obj: Any) -> dict[Any, Any]: - """Helper for json_encoder. Makes immutabledicts serializable by returning - the underlying dict +def _handle_extra_mappings(obj: Any) -> dict[Any, Any]: + """Helper for json_encoder. Makes immutabledicts and JsonObjects + serializable """ - if type(obj) is immutabledict: - # fishing the protected dict out of the object is a bit nasty, - # but we don't really want the overhead of copying the dict. - try: - # Safety: we catch the AttributeError immediately below. - return obj._dict - except AttributeError: - # If all else fails, resort to making a copy of the immutabledict + match obj: + case immutabledict(): + # fishing the protected dict out of the object is a bit nasty, + # but we don't really want the overhead of copying the dict. + try: + # Safety: we catch the AttributeError immediately below. + return obj._dict + except AttributeError: + # If all else fails, resort to making a copy of the immutabledict + return dict(obj) + case JsonObject(): + # Just convert to a dict. return dict(obj) raise TypeError( "Object of type %s is not JSON serializable" % obj.__class__.__name__ @@ -49,7 +55,7 @@ def _handle_immutabledict(obj: Any) -> dict[Any, Any]: # * produces valid JSON (no NaNs etc) # * reduces redundant whitespace json_encoder = json.JSONEncoder( - allow_nan=False, separators=(",", ":"), default=_handle_immutabledict + allow_nan=False, separators=(",", ":"), default=_handle_extra_mappings ) # Create a custom decoder to reject Python extensions to JSON. diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 334ff64bc2..6ec0f64ffc 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -62,6 +62,7 @@ class EventSigningTestCase(unittest.TestCase): "origin_server_ts": 1000000, "signatures": {}, "type": "X", + "content": {}, "unsigned": {"age_ts": 1000000}, } @@ -74,7 +75,7 @@ class EventSigningTestCase(unittest.TestCase): self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) self.assertEqual( - event.hashes["sha256"], "A6Nco6sqoy18PPfPDVdYvoowfc0PVBk9g9OiyT3ncRM" + event.hashes["sha256"], "mq4QfPPpC+QsBd6eqfVsmJIEz8uvMSVK0+AU67PLESk" ) self.assertTrue(hasattr(event, "signatures")) @@ -82,8 +83,8 @@ class EventSigningTestCase(unittest.TestCase): self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEqual( event.signatures[HOSTNAME][KEY_NAME], - "PBc48yDVszWB9TRaB/+CZC1B+pDAC10F8zll006j+NN" - "fe4PEMWcVuLaG63LFTK9e4rwJE8iLZMPtCKhDTXhpAQ", + "18rGIkd4JJXxw9m+1j3BtN+TmqmLip4VHvFbyXLngpB" + "LXOqbxlQViQABRzep2cODQ2aa5FnFgz+Llt2P03WiAw", ) def test_sign_message(self) -> None: diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index f1b20a12ec..2ba5da3b95 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -265,7 +265,7 @@ class ModuleApiTestCase(BaseModuleApiTestCase): self.assertEqual(event.type, "m.room.message") self.assertEqual(event.room_id, room_id) self.assertFalse(hasattr(event, "state_key")) - self.assertDictEqual(event.content, content) + self.assertDictEqual(dict(event.content), content) expected_requester = create_requester( user_id, authenticated_entity=self.hs.hostname @@ -301,7 +301,7 @@ class ModuleApiTestCase(BaseModuleApiTestCase): self.assertEqual(event.type, "m.room.power_levels") self.assertEqual(event.room_id, room_id) self.assertEqual(event.state_key, "") - self.assertDictEqual(event.content, content) + self.assertDictEqual(dict(event.content), content) # Check that the event was sent self.event_creation_handler.create_and_send_nonmember_event.assert_called_with( diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 61e7e87f62..28872fa06c 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -59,7 +59,7 @@ from synapse.rest.client import ( sync, ) from synapse.server import HomeServer -from synapse.types import JsonDict, RoomAlias, UserID, create_requester +from synapse.types import JsonDict, JsonMapping, RoomAlias, UserID, create_requester from synapse.util.clock import Clock from synapse.util.stringutils import random_string @@ -1859,7 +1859,7 @@ class RoomMessagesTestCase(RoomBase): mock_return_value: str | bool | Codes | tuple[Codes, JsonDict] | bool = ( "NOT_SPAM" ) - mock_content: JsonDict | None = None + mock_content: JsonMapping | None = None async def check_event_for_spam( self, diff --git a/tests/server.py b/tests/server.py index 55860701da..20fcc42081 100644 --- a/tests/server.py +++ b/tests/server.py @@ -101,6 +101,7 @@ from synapse.storage.engines import BaseDatabaseEngine, create_engine from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock +from synapse.util.json import json_encoder from tests.utils import ( LEAVE_DB, @@ -422,7 +423,7 @@ def make_request( path = b"/" + path if isinstance(content, dict): - content = json.dumps(content).encode("utf8") + content = json_encoder.encode(content).encode("utf8") if isinstance(content, str): content = content.encode("utf8") diff --git a/tests/synapse_rust/test_json_object.py b/tests/synapse_rust/test_json_object.py new file mode 100644 index 0000000000..77b188eee0 --- /dev/null +++ b/tests/synapse_rust/test_json_object.py @@ -0,0 +1,149 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.synapse_rust.events import JsonObject +from synapse.util import MutableOverlayMapping + +from tests import unittest + + +class JsonObjectMappingTestCase(unittest.TestCase): + def test_new_and_basic_mapping_behavior(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(len(obj), 2) + self.assertTrue("a" in obj) + self.assertTrue("b" in obj) + self.assertFalse("c" in obj) + self.assertFalse(123 in obj) # type: ignore[comparison-overlap] + + def test_getitem_and_key_errors(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(obj["a"], 1) + + with self.assertRaises(KeyError): + _ = obj["missing"] + + with self.assertRaises(KeyError): + _ = obj[10] # type: ignore[index] + + def test_iter_keys_values_items(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + iterator = iter(obj) + first = next(iterator) + second = next(iterator) + self.assertCountEqual((first, second), ("a", "b")) + with self.assertRaises(StopIteration): + next(iterator) + + self.assertCountEqual(list(obj.keys()), ["a", "b"]) + self.assertCountEqual(list(obj.values()), [1, 2]) + self.assertCountEqual(list(obj.items()), [("a", 1), ("b", 2)]) + + def test_keys_set_like_behavior(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + # Test 'and' operator. + self.assertEqual(obj.keys() & {"a"}, {"a"}) + self.assertEqual({"a"} & obj.keys(), {"a"}) + self.assertEqual(obj.keys() & {"c"}, set()) + self.assertEqual({"c"} & obj.keys(), set()) + + # Test 'or' operator. + self.assertEqual(obj.keys() | {"a"}, {"a", "b"}) + self.assertEqual({"a"} | obj.keys(), {"a", "b"}) + self.assertEqual(obj.keys() | {"c"}, {"a", "b", "c"}) + self.assertEqual({"c"} | obj.keys(), {"a", "b", "c"}) + + # Test 'xor' operator. + self.assertEqual(obj.keys() ^ {"a"}, {"b"}) + self.assertEqual({"a"} ^ obj.keys(), {"b"}) + self.assertEqual(obj.keys() ^ {"c"}, {"a", "b", "c"}) + self.assertEqual({"c"} ^ obj.keys(), {"a", "b", "c"}) + + # Test 'sub' operator. + self.assertEqual(obj.keys() - {"a"}, {"b"}) + self.assertEqual({"a"} - obj.keys(), set()) + self.assertEqual(obj.keys() - {"c"}, {"a", "b"}) + self.assertEqual({"c"} - obj.keys(), {"c"}) + + def test_values_view(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + values = obj.values() + + self.assertEqual(len(values), 2) + self.assertCountEqual(list(values), [1, 2]) + + self.assertIn(1, values) + self.assertIn(2, values) + self.assertNotIn(3, values) + self.assertNotIn("a", values) + self.assertNotIn(object(), values) + + # Iterating twice should yield the same values. + self.assertCountEqual(list(values), [1, 2]) + + def test_items_view(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + items = obj.items() + + self.assertEqual(len(items), 2) + self.assertCountEqual(list(items), [("a", 1), ("b", 2)]) + + self.assertIn(("a", 1), items) + self.assertIn(("b", 2), items) + self.assertNotIn(("a", 2), items) + self.assertNotIn(("c", 1), items) + self.assertNotIn("a", items) + self.assertNotIn(("a", 1, "extra"), items) + + # Iterating twice should yield the same items. + self.assertCountEqual(list(items), [("a", 1), ("b", 2)]) + + def test_get(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(obj.get("a"), 1) + self.assertEqual(obj.get("missing", "fallback"), "fallback") + self.assertEqual(obj.get(5, "fallback"), "fallback") # type: ignore[call-overload] + + def test_eq(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(obj, {"a": 1, "b": 2}) + self.assertNotEqual(obj, {"a": 1}) + self.assertNotEqual(obj, ["a", "b"]) + + def test_str_and_repr(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(str(obj), r'{"a":1,"b":2}') + self.assertEqual(repr(obj), r'JsonObject({"a":1,"b":2})') + + def test_json_object_constructor(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + # Passing in an existing JsonObject should work. + obj2 = JsonObject(obj) + self.assertEqual(obj2, {"a": 1, "b": 2}) + + # Other mapping types should also work. + obj3 = JsonObject(MutableOverlayMapping({"a": 1, "b": 2})) + self.assertEqual(obj3, {"a": 1, "b": 2}) + + # Test that passing a non-mapping raises a TypeError. + with self.assertRaises(TypeError): + JsonObject(123) # type: ignore[arg-type] diff --git a/tests/test_state.py b/tests/test_state.py index 7df95ebf8b..0ca88aef74 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -35,7 +35,7 @@ from synapse.api.room_versions import RoomVersions from synapse.events import EventBase, make_event_from_dict from synapse.events.snapshot import EventContext from synapse.state import StateHandler, StateResolutionHandler, _make_state_cache_entry -from synapse.types import MutableStateMap, StateMap +from synapse.types import JsonDict, MutableStateMap, StateMap from synapse.types.state import StateFilter from synapse.util.macaroons import MacaroonGenerator @@ -67,13 +67,14 @@ def create_event( else: name = "<%s, %s>" % (type, event_id) - d = { + d: JsonDict = { "event_id": event_id, "type": type, "sender": "@user_id:example.com", "room_id": "!room_id:example.com", "depth": depth, "prev_events": prev_events or [], + "content": {}, } if state_key is not None: diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py index 0df5a4e6c3..4170768208 100644 --- a/tests/test_utils/__init__.py +++ b/tests/test_utils/__init__.py @@ -24,7 +24,6 @@ Utilities for running the unit tests """ import base64 -import json import sys import warnings from binascii import unhexlify @@ -41,6 +40,7 @@ from twisted.web.http_headers import Headers from twisted.web.iweb import IResponse from synapse.types import JsonSerializable +from synapse.util.json import json_encoder if TYPE_CHECKING: from sys import UnraisableHookArgs @@ -127,7 +127,7 @@ class FakeResponse: # type: ignore[misc] @classmethod def json(cls, *, code: int = 200, payload: JsonSerializable) -> "FakeResponse": headers = Headers({"Content-Type": ["application/json"]}) - body = json.dumps(payload).encode("utf-8") + body = json_encoder.encode(payload).encode("utf-8") return cls(code=code, body=body, headers=headers) From 5c87faf9e9adbea0c0d6b376f65e6a03533b2326 Mon Sep 17 00:00:00 2001 From: Will Hunt <2072976+Half-Shot@users.noreply.github.com> Date: Mon, 11 May 2026 12:39:38 +0100 Subject: [PATCH 14/42] MSC4452: Preview URL capability (#19715) Implementation of https://github.com/matrix-org/matrix-spec-proposals/pull/4452 --- changelog.d/19715.feature | 3 ++ synapse/config/experimental.py | 4 ++ synapse/config/repository.py | 3 +- synapse/rest/client/capabilities.py | 5 ++ synapse/rest/client/media.py | 20 ++++--- .../rest/media/media_repository_resource.py | 7 ++- synapse/rest/media/preview_url_resource.py | 10 +++- tests/rest/client/test_capabilities.py | 43 ++++++++++++++- tests/rest/client/test_media.py | 54 +++++++++++++++++++ 9 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 changelog.d/19715.feature diff --git a/changelog.d/19715.feature b/changelog.d/19715.feature new file mode 100644 index 0000000000..973fe66e7d --- /dev/null +++ b/changelog.d/19715.feature @@ -0,0 +1,3 @@ +Add support for "MSC4452 Preview URL capabilities API" which exposes a `io.element.msc4452.preview_url` capability. +If `experimental_features.msc4452_enabled` is `true`, the `/_matrix/(client/v1/media|media/v3)/preview_url` endpoint +now responds with a 403 status code when the capability is disabled. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index f7c452bc73..c958a278fc 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -613,3 +613,7 @@ class ExperimentalConfig(Config): # Tracked in: https://github.com/element-hq/synapse/issues/19691 # Note that this is only applicable to legacy auth, not MAS integration (OAuth 2.0). self.msc4450_enabled: bool = experimental.get("msc4450_enabled", False) + + # MSC4455: Preview URL capability + # Tracked in: https://github.com/element-hq/synapse/issues/19719 + self.msc4452_enabled: bool = experimental.get("msc4452_enabled", False) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index c87442aace..cb50d0dc1d 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -242,7 +242,8 @@ class ContentRepositoryConfig(Config): self.thumbnail_requirements = parse_thumbnail_requirements( config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES) ) - self.url_preview_enabled = config.get("url_preview_enabled", False) + self.url_preview_enabled = bool(config.get("url_preview_enabled", False)) + if self.url_preview_enabled: check_requirements("url-preview") diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 705d74dee1..2be5f5849d 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -77,6 +77,11 @@ class CapabilitiesRestServlet(RestServlet): } } + if self.config.experimental.msc4452_enabled: + response["capabilities"]["io.element.msc4452.preview_url"] = { + "enabled": self.config.media.url_preview_enabled, + } + if self.config.experimental.msc3720_enabled: response["capabilities"]["org.matrix.msc3720.account_status"] = { "enabled": True, diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index 15f58acb95..c740659cdd 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -23,7 +23,12 @@ import logging import re -from synapse.api.errors import Codes, cs_error +from synapse.api.errors import ( + Codes, + SynapseError, + UnrecognizedRequestError, + cs_error, +) from synapse.http.server import ( HttpServer, respond_with_json, @@ -79,11 +84,17 @@ class PreviewURLServlet(RestServlet): self.clock = hs.get_clock() self.media_repo = media_repo self.media_storage = media_storage - assert self.media_repo.url_previewer is not None self.url_previewer = self.media_repo.url_previewer + self.can_respond_403 = hs.config.experimental.msc4452_enabled async def on_GET(self, request: SynapseRequest) -> None: requester = await self.auth.get_user_by_req(request) + if self.url_previewer is None: + # If we have no url_previewer then it has been disabled by the server. + if self.can_respond_403: + raise SynapseError(403, "URL Previews are disabled", Codes.FORBIDDEN) + else: + raise UnrecognizedRequestError(code=404) url = parse_string(request, "url", required=True) ts = parse_integer(request, "ts") if ts is None: @@ -299,10 +310,7 @@ class DownloadResource(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: media_repo = hs.get_media_repository() - if hs.config.media.url_preview_enabled: - PreviewURLServlet(hs, media_repo, media_repo.media_storage).register( - http_server - ) + PreviewURLServlet(hs, media_repo, media_repo.media_storage).register(http_server) MediaConfigResource(hs).register(http_server) ThumbnailResource(hs, media_repo, media_repo.media_storage).register(http_server) DownloadResource(hs, media_repo).register(http_server) diff --git a/synapse/rest/media/media_repository_resource.py b/synapse/rest/media/media_repository_resource.py index 963b9de252..354144af35 100644 --- a/synapse/rest/media/media_repository_resource.py +++ b/synapse/rest/media/media_repository_resource.py @@ -106,8 +106,7 @@ class MediaRepositoryResource(JsonResource): ThumbnailResource(hs, media_repo, media_repo.media_storage).register( http_server ) - if hs.config.media.url_preview_enabled: - PreviewUrlResource(hs, media_repo, media_repo.media_storage).register( - http_server - ) + PreviewUrlResource(hs, media_repo, media_repo.media_storage).register( + http_server + ) MediaConfigResource(hs).register(http_server) diff --git a/synapse/rest/media/preview_url_resource.py b/synapse/rest/media/preview_url_resource.py index bfeff2179b..5a7bdf38f0 100644 --- a/synapse/rest/media/preview_url_resource.py +++ b/synapse/rest/media/preview_url_resource.py @@ -23,6 +23,7 @@ import re from typing import TYPE_CHECKING +from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError from synapse.http.server import respond_with_json_bytes from synapse.http.servlet import RestServlet, parse_integer, parse_string from synapse.http.site import SynapseRequest @@ -65,12 +66,17 @@ class PreviewUrlResource(RestServlet): self.clock = hs.get_clock() self.media_repo = media_repo self.media_storage = media_storage - assert self.media_repo.url_previewer is not None self.url_previewer = self.media_repo.url_previewer + self.can_respond_403 = hs.config.experimental.msc4452_enabled async def on_GET(self, request: SynapseRequest) -> None: - # XXX: if get_user_by_req fails, what should we do in an async render? requester = await self.auth.get_user_by_req(request) + if self.url_previewer is None: + # If we have no url_previewer then it has been disabled by the server. + if self.can_respond_403: + raise SynapseError(403, "URL Previews are disabled", Codes.FORBIDDEN) + else: + raise UnrecognizedRequestError(code=404) url = parse_string(request, "url", required=True) ts = parse_integer(request, "ts", default=self.clock.time_msec()) og = await self.url_previewer.preview(url, requester.user, ts) diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index 2567fee2a4..c28e0605b5 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -28,7 +28,12 @@ from synapse.server import HomeServer from synapse.util.clock import Clock from tests import unittest -from tests.unittest import override_config +from tests.unittest import override_config, skip_unless + +try: + import lxml +except ImportError: + lxml = None # type: ignore[assignment] class CapabilitiesTestCase(unittest.HomeserverTestCase): @@ -276,3 +281,39 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertFalse( capabilities["org.matrix.msc4267.forget_forced_upon_leave"]["enabled"] ) + + @override_config( + { + "url_preview_enabled": False, + "experimental_features": {"msc4452_enabled": True}, + } + ) + def test_url_previews_disabled(self) -> None: + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse(capabilities["io.element.msc4452.preview_url"]["enabled"]) + + @skip_unless(lxml is not None, "Requires lxml") + @override_config( + { + "url_preview_enabled": True, + "url_preview_ip_range_blacklist": ["127.0.0.1"], + "experimental_features": {"msc4452_enabled": True}, + }, + ) + def test_url_previews_enabled(self) -> None: + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertTrue(capabilities["io.element.msc4452.preview_url"]["enabled"]) diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index ec81b1413c..3409581c5e 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -1573,6 +1573,60 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.assertEqual(channel.code, 403, channel.result) +# We test this here because this endpoint must still work +# even if lxml is not installed. +class URLPreviewDisabledTests(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + media.register_servlets, + ] + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.register_user("user", "password") + self.tok = self.login("user", "password") + + @override_config( + { + "url_preview_enabled": False, + "experimental_features": {"msc4452_enabled": False}, + } + ) + def test_disabled_previews(self) -> None: + """Tests that disabling URL previews gives back a sane response.""" + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/preview_url?url=" + quote("http://example.com"), + access_token=self.tok, + ) + self.assertEqual(channel.code, 404, channel.result) + self.assertEqual( + channel.json_body, + {"errcode": "M_UNRECOGNIZED", "error": "Unrecognized request"}, + ) + + @override_config( + { + "url_preview_enabled": False, + "experimental_features": {"msc4452_enabled": True}, + } + ) + def test_disabled_previews_with_msc4452(self) -> None: + """Tests that disabling URL previews gives back a sane response.""" + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/preview_url?url=" + quote("http://example.com"), + access_token=self.tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual( + channel.json_body, + {"errcode": "M_FORBIDDEN", "error": "URL Previews are disabled"}, + ) + + class MediaConfigTest(unittest.HomeserverTestCase): servlets = [ media.register_servlets, From b8bd35105fb166aa40d7493a26bcbfff909174c8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 12 May 2026 10:10:09 -0500 Subject: [PATCH 15/42] Update `WorkerLock` tests to better stress the `WORKER_LOCK_MAX_RETRY_INTERVAL` (#19772) There is no behavioral change, only a change to the tests. See https://github.com/element-hq/synapse/pull/19772#discussion_r3222059105 for an explanation of why the tests needed changing (and diff comments). Follow-up to https://github.com/element-hq/synapse/pull/19394. The test discussion originally happened in https://github.com/element-hq/synapse/pull/19394#discussion_r2789673181 This is spawning from thinking about the problem again. --- changelog.d/19772.misc | 1 + synapse/handlers/worker_lock.py | 8 +-- synapse/storage/databases/main/lock.py | 14 +++-- tests/handlers/test_worker_lock.py | 73 ++++++++++++++++++++--- tests/storage/databases/main/test_lock.py | 12 ++-- 5 files changed, 83 insertions(+), 25 deletions(-) create mode 100644 changelog.d/19772.misc diff --git a/changelog.d/19772.misc b/changelog.d/19772.misc new file mode 100644 index 0000000000..939507f5c3 --- /dev/null +++ b/changelog.d/19772.misc @@ -0,0 +1 @@ +Update `WorkerLock` tests to better stress the `WORKER_LOCK_MAX_RETRY_INTERVAL`. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 57792ea53c..a37b04494b 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -61,10 +61,10 @@ The maximum wait time before retrying to acquire the lock. Better to retry more quickly than have workers wait around. 5 seconds is still a reasonable gap in time to not overwhelm the CPU/Database. -This matters most in cross-worker scenarios. When locks are on the same worker, when the -lock holder releases, we signal to other locks (with the same name/key) that they -should try reacquiring the lock immediately. But locks on other workers only re-check -based on their retry `_timeout_interval`. +This matters most when locks go stale as normally, when the lock holder releases, we +signal to other locks (with the same name/key) that they should try reacquiring the lock +immediately. But stale locks are never released and instead forcefully reaped behind the +scenes. """ WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index dd49f98366..decb74e994 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -53,9 +53,9 @@ logger = logging.getLogger(__name__) _RENEWAL_INTERVAL = Duration(seconds=30) # How long before an acquired lock times out. -_LOCK_TIMEOUT_MS = 2 * 60 * 1000 +_LOCK_TIMEOUT = Duration(minutes=2) -_LOCK_REAP_INTERVAL = Duration(milliseconds=_LOCK_TIMEOUT_MS / 10.0) +_LOCK_REAP_INTERVAL = Duration(milliseconds=_LOCK_TIMEOUT.as_millis() / 10.0) class LockStore(SQLBaseStore): @@ -63,7 +63,7 @@ class LockStore(SQLBaseStore): Locks are identified by a name and key. A lock is acquired by inserting into the `worker_locks` table if a) there is no existing row for the name/key or - b) the existing row has a `last_renewed_ts` older than `_LOCK_TIMEOUT_MS`. + b) the existing row has a `last_renewed_ts` older than `_LOCK_TIMEOUT`. When a lock is taken out the instance inserts a random `token`, the instance that holds that token holds the lock until it drops (or times out). @@ -182,7 +182,7 @@ class LockStore(SQLBaseStore): self._instance_name, token, now, - now - _LOCK_TIMEOUT_MS, + now - _LOCK_TIMEOUT.as_millis(), ), ) @@ -340,7 +340,9 @@ class LockStore(SQLBaseStore): """ def reap_stale_read_write_locks_txn(txn: LoggingTransaction) -> None: - txn.execute(delete_sql, (self.clock.time_msec() - _LOCK_TIMEOUT_MS,)) + txn.execute( + delete_sql, (self.clock.time_msec() - _LOCK_TIMEOUT.as_millis(),) + ) if txn.rowcount: logger.info("Reaped %d stale locks", txn.rowcount) @@ -489,7 +491,7 @@ class Lock: ) return ( last_renewed_ts is not None - and self._clock.time_msec() - _LOCK_TIMEOUT_MS < last_renewed_ts + and self._clock.time_msec() - _LOCK_TIMEOUT.as_millis() < last_renewed_ts ) async def __aenter__(self) -> None: diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index 74201f4151..a38adcd4d4 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -25,8 +25,13 @@ import platform from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from synapse.handlers.worker_lock import WORKER_LOCK_MAX_RETRY_INTERVAL from synapse.server import HomeServer -from synapse.storage.databases.main.lock import _RENEWAL_INTERVAL +from synapse.storage.databases.main.lock import ( + _LOCK_REAP_INTERVAL, + _LOCK_TIMEOUT, + _RENEWAL_INTERVAL, +) from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -83,7 +88,7 @@ class WorkerLockTestCase(unittest.HomeserverTestCase): # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an # internal background looping call that runs every 30 seconds # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value - # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # further out by 2 minutes (`_LOCK_TIMEOUT`). The `Lock` will prematurely # drop if this renewal is not allowed to run, which sours the test. # self.pump(amount=Duration(hours=1)) self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) @@ -91,9 +96,34 @@ class WorkerLockTestCase(unittest.HomeserverTestCase): # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) self.assertNoResult(d2) - # Release the first lock (`lock1`). The second lock(`lock2`) should be - # automatically acquired by the `pump()` inside `get_success()` - self.get_success(lock1.__aexit__(None, None, None)) + # Drop the lock without releasing it. If we just normally released the lock + # (`self.get_success(lock1.__aexit__(None, None, None))`), the + # `add_lock_released_callback`/`notify_lock_released` cycle would signal that we + # should re-aquire the lock right away (on the next reactor tick). And we want + # to avoid that as the point of this test is to stress the retry timeout + # interval and `WORKER_LOCK_MAX_RETRY_INTERVAL`. + del lock1 + + # Wait for `lock1` to go stale (it won't be renewed anymore because we deleted + # it just above) + self._pump_by( + amount=_LOCK_TIMEOUT, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock1` is reaped (found stale and forcefully drops + # the lock its holding) + self._pump_by( + amount=_LOCK_REAP_INTERVAL, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock2` tries re-acquiring the lock. Should be no + # longer than our `WORKER_LOCK_MAX_RETRY_INTERVAL`. + self._pump_by( + amount=WORKER_LOCK_MAX_RETRY_INTERVAL, + by=_RENEWAL_INTERVAL, + ) # We should now have the lock self.successResultOf(d2) @@ -219,7 +249,7 @@ class WorkerLockWorkersTestCase(BaseMultiWorkerStreamTestCase): # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an # internal background looping call that runs every 30 seconds # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value - # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # further out by 2 minutes (`_LOCK_TIMEOUT`). The `Lock` will prematurely # drop if this renewal is not allowed to run, which sours the test. # self.pump(amount=Duration(hours=1)) self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) @@ -227,9 +257,34 @@ class WorkerLockWorkersTestCase(BaseMultiWorkerStreamTestCase): # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) self.assertNoResult(d2) - # Release the first lock (`lock1`). The second lock(`lock2`) should be - # automatically acquired by the `pump()` inside `get_success()` - self.get_success(lock1.__aexit__(None, None, None)) + # Drop the lock without releasing it. If we just normally released the lock + # (`self.get_success(lock1.__aexit__(None, None, None))`), the + # `add_lock_released_callback`/`notify_lock_released` cycle would signal that we + # should re-aquire the lock right away (on the next reactor tick). And we want + # to avoid that as the point of this test is to stress the retry timeout + # interval and `WORKER_LOCK_MAX_RETRY_INTERVAL`. + del lock1 + + # Wait for `lock1` to go stale (it won't be renewed anymore because we deleted + # it just above) + self._pump_by( + amount=_LOCK_TIMEOUT, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock1` is reaped (found stale and forcefully drops + # the lock its holding) + self._pump_by( + amount=_LOCK_REAP_INTERVAL, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock2` tries re-acquiring the lock. Should be no + # longer than our `WORKER_LOCK_MAX_RETRY_INTERVAL`. + self._pump_by( + amount=WORKER_LOCK_MAX_RETRY_INTERVAL, + by=_RENEWAL_INTERVAL, + ) # We should now have the lock self.successResultOf(d2) diff --git a/tests/storage/databases/main/test_lock.py b/tests/storage/databases/main/test_lock.py index 622eb96ded..c38e0cc834 100644 --- a/tests/storage/databases/main/test_lock.py +++ b/tests/storage/databases/main/test_lock.py @@ -26,7 +26,7 @@ from twisted.internet.defer import Deferred from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer -from synapse.storage.databases.main.lock import _LOCK_TIMEOUT_MS, _RENEWAL_INTERVAL +from synapse.storage.databases.main.lock import _LOCK_TIMEOUT, _RENEWAL_INTERVAL from synapse.util.clock import Clock from tests import unittest @@ -117,7 +117,7 @@ class LockTestCase(unittest.HomeserverTestCase): self.get_success(lock.__aenter__()) # Wait for ages with the lock, we should not be able to get the lock. - self.reactor.advance(5 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(5 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) self.assertIsNone(lock2) @@ -138,7 +138,7 @@ class LockTestCase(unittest.HomeserverTestCase): lock._looping_call.stop() # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) self.assertIsNotNone(lock2) @@ -154,7 +154,7 @@ class LockTestCase(unittest.HomeserverTestCase): del lock # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) self.assertIsNotNone(lock2) @@ -402,7 +402,7 @@ class ReadWriteLockTestCase(unittest.HomeserverTestCase): lock._looping_call.stop() # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success( self.store.try_acquire_read_write_lock("name", "key", write=True) @@ -422,7 +422,7 @@ class ReadWriteLockTestCase(unittest.HomeserverTestCase): del lock # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success( self.store.try_acquire_read_write_lock("name", "key", write=True) From 5efeac44b27a98bdf73f0fddc37753d8181d5ee2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2026 11:28:06 +0100 Subject: [PATCH 16/42] Handle arbitrary sized integers in `unsigned`. (#19769) Handle arbitrary sized integers in `unsigned` (and other Rust objects that use `serde_json::Value`) --- Cargo.lock | 59 +++--------- changelog.d/19769.bugfix | 1 + rust/Cargo.toml | 14 ++- rust/src/acl/mod.rs | 2 +- rust/src/canonical_json.rs | 132 ++++++++++++++++++++++++--- rust/src/events/internal_metadata.rs | 2 +- rust/src/events/unsigned.rs | 28 +++--- rust/src/push/mod.rs | 6 +- rust/src/room_versions.rs | 13 ++- tests/synapse_rust/test_unsigned.py | 52 +++++++++++ 10 files changed, 225 insertions(+), 84 deletions(-) create mode 100644 changelog.d/19769.bugfix create mode 100644 tests/synapse_rust/test_unsigned.py diff --git a/Cargo.lock b/Cargo.lock index 832d5129fe..8285c7cf38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,12 +29,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - [[package]] name = "base64" version = "0.22.1" @@ -646,12 +640,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "indoc" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" - [[package]] name = "ipnet" version = "2.11.0" @@ -735,15 +723,6 @@ version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.3.17" @@ -821,37 +800,34 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ "anyhow", "bytes", - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -870,9 +846,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -882,9 +858,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", @@ -895,12 +871,13 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3a8f29db331e28c332c63496cfcbb822aca3d7320bc08b655d7fd0c29c50ede" +checksum = "0b79f670c9626c8b651c0581011b57b6ba6970bb69faf01a7c4c0cfc81c43f95" dependencies = [ "pyo3", "serde", + "serde_json", ] [[package]] @@ -1391,9 +1368,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.2" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "thiserror" @@ -1569,12 +1546,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - [[package]] name = "untrusted" version = "0.9.0" diff --git a/changelog.d/19769.bugfix b/changelog.d/19769.bugfix new file mode 100644 index 0000000000..05aae44311 --- /dev/null +++ b/changelog.d/19769.bugfix @@ -0,0 +1 @@ +Correctly handle arbitrary precision integers in `unsigned` field of events. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 5bdd194707..162bc98182 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,7 +30,7 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.27.2", features = [ +pyo3 = { version = "0.28.3", features = [ "macros", "anyhow", "abi3", @@ -39,12 +39,18 @@ pyo3 = { version = "0.27.2", features = [ # https://docs.rs/pyo3/latest/pyo3/bytes/index.html "bytes", ] } -pyo3-log = "0.13.1" -pythonize = "0.27.0" +pyo3-log = "0.13.3" +pythonize = { version = "0.28.0", features = ["arbitrary_precision"] } regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive", "rc"] } -serde_json = { version = "1.0.85", features = ["raw_value"] } +serde_json = { version = "1.0.85", features = [ + "raw_value", + # We need to be able to parse arbitrary precision numbers, as some numbers + # in the database may be out of range of i64 (as Python uses arbitrary + # precision integers). + "arbitrary_precision", +] } ulid = "1.1.2" icu_segmenter = "2.0.0" reqwest = { version = "0.12.15", default-features = false, features = [ diff --git a/rust/src/acl/mod.rs b/rust/src/acl/mod.rs index 57b45475fd..ab10b9f037 100644 --- a/rust/src/acl/mod.rs +++ b/rust/src/acl/mod.rs @@ -47,7 +47,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> } #[derive(Debug, Clone)] -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct ServerAclEvaluator { allow_ip_literals: bool, allow: Vec, diff --git a/rust/src/canonical_json.rs b/rust/src/canonical_json.rs index 4abd001847..94ed6c368d 100644 --- a/rust/src/canonical_json.rs +++ b/rust/src/canonical_json.rs @@ -29,7 +29,7 @@ use std::{ io::{self, Write}, }; -use serde::ser::SerializeMap; +use serde::{ser::SerializeMap, Serializer as _}; use serde::{ ser::{Error as _, SerializeStruct}, Serialize, @@ -37,7 +37,7 @@ use serde::{ use serde_json::{ ser::{Formatter, Serializer}, value::RawValue, - Value, + Number, Value, }; /// The minimum integer that can be used in canonical JSON. @@ -46,6 +46,12 @@ pub const MIN_VALID_INTEGER: i64 = -(2i64.pow(53)) + 1; /// The maximum integer that can be used in canonical JSON. pub const MAX_VALID_INTEGER: i64 = (2i64.pow(53)) - 1; +/// A token used by `serde_json` to identify its internal `Number` type when the +/// `arbitrary_precision` feature is enabled. This is a copy from serde_json's +/// internal `TOKEN` for `Number`, which unfortunately isn't exported by the +/// crate. +const SERDE_JSON_NUMBER_TOKEN: &str = "$serde_json::private::Number"; + /// Options to control how strict JSON canonicalization is. #[derive(Clone, Debug)] pub struct CanonicalizationOptions { @@ -236,7 +242,7 @@ where type SerializeMap = CanonicalSerializeMap<'a, W>; - type SerializeStruct = CanonicalSerializeMap<'a, W>; + type SerializeStruct = CanonicalSerializeStruct<'a, W>; type SerializeStructVariant = <&'a mut Serializer as serde::Serializer>::SerializeStructVariant; @@ -426,7 +432,7 @@ where fn serialize_struct( self, name: &'static str, - _len: usize, + len: usize, ) -> Result { // We want to disallow `RawValue` as we don't know if its contents is // canonical JSON. @@ -436,10 +442,7 @@ where if name == "$serde_json::private::RawValue" { return Err(Self::Error::custom("`RawValue` is not supported")); } - Ok(CanonicalSerializeMap::new( - &mut self.inner, - self.options.clone(), - )) + CanonicalSerializeStruct::new(name, len, &mut self.inner, self.options.clone()) } fn serialize_struct_variant( @@ -554,7 +557,42 @@ where } } -impl<'a, W> SerializeStruct for CanonicalSerializeMap<'a, W> +/// A helper type for [`CanonicalSerializer`] that serializes structs in +/// lexicographic order. +#[doc(hidden)] +pub struct CanonicalSerializeStruct<'a, W: Write> { + name: &'static str, + // We buffer up the key and serialized value for each field we see. + // The BTreeMap will then serialize in lexicographic order. + map: BTreeMap<&'static str, Box>, + options: CanonicalizationOptions, + // The serializer to use to write the sorted map too. + struct_serializer: + <&'a mut Serializer as serde::Serializer>::SerializeStruct, +} + +impl<'a, W> CanonicalSerializeStruct<'a, W> +where + W: Write, +{ + fn new( + name: &'static str, + len: usize, + ser: &'a mut Serializer, + options: CanonicalizationOptions, + ) -> Result { + let struct_serializer = ser.serialize_struct(name, len)?; + + Ok(Self { + name, + map: BTreeMap::new(), + options, + struct_serializer, + }) + } +} + +impl<'a, W> SerializeStruct for CanonicalSerializeStruct<'a, W> where W: Write, { @@ -566,20 +604,69 @@ where where T: Serialize + ?Sized, { - let key_string = key.to_string(); + // Check if this is the special case of `SERDE_JSON_NUMBER_TOKEN`, + // which is used when serializing numbers with the `arbitrary_precision` + // feature. If so, we can just serialize it directly without + // canonicalizing it first, as `serde_json` will have already serialized + // it in a canonical way. + if key == SERDE_JSON_NUMBER_TOKEN && self.name == SERDE_JSON_NUMBER_TOKEN { + if self.options.enforce_int_range { + // We need to check that the number is in the valid range, as + // `serde_json` won't have done this for us as we're using the + // `arbitrary_precision` feature. + + // The value here will be something that serializes to a JSON + // string containing the number, so we first serialize it to a + // Value and pull the string out, then parse it as a `Number`. + + let serde_val = serde_json::to_value(value)?; + let serde_json::Value::String(number_str) = serde_val else { + return Err(serde_json::Error::custom("invalid number")); + }; + + let number: Number = number_str + .parse() + .map_err(|_| serde_json::Error::custom("invalid number"))?; + + // Now check that the number is an integer in the valid range. + if let Some(int) = number.as_i64() { + assert_integer_in_range(int)?; + } else { + // Can't be cast to an i64, so it must be out of range. + return Err(serde_json::Error::custom("integer out of range")); + } + } + + self.struct_serializer.serialize_field(key, value)?; + return Ok(()); + } // We serialize the value canonically, then store it as a `RawValue` in // the buffer map. let value_string = to_string_canonical(value, self.options.clone())?; - self.map - .insert(key_string, RawValue::from_string(value_string)?); + self.map.insert(key, RawValue::from_string(value_string)?); Ok(()) } - fn end(self) -> Result { - self.map.serialize(self.ser)?; + fn end(mut self) -> Result { + if self.name == SERDE_JSON_NUMBER_TOKEN { + // Map must be empty in this case, as `SERDE_JSON_NUMBER_TOKEN` + // only has one field and we've handled it in `serialize_field`. + if !self.map.is_empty() { + return Err(Self::Error::custom(format!( + "unexpected fields in `{}`", + SERDE_JSON_NUMBER_TOKEN + ))); + } + } + + for (key, value) in self.map { + self.struct_serializer.serialize_field(key, &value)?; + } + + SerializeStruct::end(self.struct_serializer)?; Ok(()) } @@ -737,6 +824,23 @@ mod tests { assert!(to_string_canonical(&-(2i128.pow(60)), CanonicalizationOptions::strict()).is_err()); } + #[test] + fn bigints() { + // Create a `serde_json::Number` that is too big to be represented as an + // i64, but can be represented as a string. + let bigint_string = "10000000000000000000000000000000000000"; + let value: serde_json::Number = bigint_string.parse().unwrap(); + + // This should work with relaxed option. + assert_eq!( + to_string_canonical(&value, CanonicalizationOptions::relaxed()).unwrap(), + bigint_string + ); + + // But should fail with strict option, as it's out of range. + assert!(to_string_canonical(&value, CanonicalizationOptions::strict()).is_err()); + } + #[test] fn backwards_compatibility() { assert_eq!( diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index 6fd3d06b00..4084b8442d 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -476,7 +476,7 @@ impl EventInternalMetadataInner { } } -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] pub struct EventInternalMetadata { inner: Arc>, diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs index c41ed7e6e1..5aa56812c0 100644 --- a/rust/src/events/unsigned.rs +++ b/rust/src/events/unsigned.rs @@ -23,6 +23,7 @@ use pyo3::{ }; use pythonize::{depythonize, pythonize}; use serde::{Deserialize, Serialize}; +use serde_json::Number; #[pyclass(frozen, skip_from_py_object)] #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -36,7 +37,7 @@ pub struct Unsigned { #[derive(Debug, Clone, Serialize, Deserialize, Default)] struct PersistedUnsignedFields { #[serde(skip_serializing_if = "Option::is_none")] - age_ts: Option, + age_ts: Option, #[serde(skip_serializing_if = "Option::is_none")] replaces_state: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -129,11 +130,14 @@ impl Unsigned { let unsigned = self.py_read()?; match field { - UnsignedField::AgeTs => Ok(unsigned - .persisted_fields - .age_ts - .ok_or_else(|| PyKeyError::new_err("age_ts"))? - .into_bound_py_any(py)?), + UnsignedField::AgeTs => { + let age_ts = &unsigned + .persisted_fields + .age_ts + .as_ref() + .ok_or_else(|| PyKeyError::new_err("age_ts"))?; + Ok(pythonize(py, age_ts)?) + } UnsignedField::ReplacesState => Ok((unsigned.persisted_fields.replaces_state) .as_ref() .ok_or_else(|| PyKeyError::new_err("replaces_state"))? @@ -203,7 +207,7 @@ impl Unsigned { let mut unsigned = self.py_write()?; match field { - UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = Some(value.extract()?), + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = Some(depythonize(&value)?), UnsignedField::ReplacesState => { unsigned.persisted_fields.replaces_state = Some(value.extract()?) } @@ -339,7 +343,7 @@ mod tests { #[test] fn test_persisted_fields_serialize_populated() { let fields = PersistedUnsignedFields { - age_ts: Some(1234), + age_ts: Some(1234.into()), replaces_state: Some("$prev:example.com".to_string()), invite_room_state: Some(vec![json!({"type": "m.room.name"})]), knock_room_state: Some(vec![json!({"type": "m.room.topic"})]), @@ -360,7 +364,7 @@ mod tests { fn test_unsigned_inner_flattens_persisted_fields() { let inner = UnsignedInner { persisted_fields: PersistedUnsignedFields { - age_ts: Some(99), + age_ts: Some(99.into()), ..Default::default() }, prev_content: Some(Box::new(json!({"body": "hi"}))), @@ -382,7 +386,7 @@ mod tests { fn test_unsigned_inner_roundtrip() { let original = UnsignedInner { persisted_fields: PersistedUnsignedFields { - age_ts: Some(10), + age_ts: Some(10.into()), replaces_state: Some("$state:example.com".to_string()), invite_room_state: None, knock_room_state: None, @@ -394,7 +398,7 @@ mod tests { let json = serde_json::to_string(&original).unwrap(); let roundtripped: UnsignedInner = serde_json::from_str(&json).unwrap(); - assert_eq!(roundtripped.persisted_fields.age_ts, Some(10)); + assert_eq!(roundtripped.persisted_fields.age_ts, Some(10.into())); assert_eq!( roundtripped.persisted_fields.replaces_state.as_deref(), Some("$state:example.com") @@ -423,7 +427,7 @@ mod tests { }); let unsigned: Unsigned = serde_json::from_value(json).unwrap(); let inner = unsigned.inner.read().unwrap(); - assert_eq!(inner.persisted_fields.age_ts, Some(5)); + assert_eq!(inner.persisted_fields.age_ts, Some(5.into())); assert_eq!(inner.prev_sender.as_deref(), Some("@bob:example.com")); } } diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs index ac9b9c93e4..780d7a8cbd 100644 --- a/rust/src/push/mod.rs +++ b/rust/src/push/mod.rs @@ -104,7 +104,7 @@ fn get_base_rule_ids() -> HashSet<&'static str> { /// A single push rule for a user. #[derive(Debug, Clone)] -#[pyclass(frozen)] +#[pyclass(frozen, from_py_object)] pub struct PushRule { /// A unique ID for this rule pub rule_id: Cow<'static, str>, @@ -462,7 +462,7 @@ pub struct RelatedEventMatchTypeCondition { /// The collection of push rules for a user. #[derive(Debug, Clone, Default)] -#[pyclass(frozen)] +#[pyclass(frozen, from_py_object)] pub struct PushRules { /// Custom push rules that override a base rule. overridden_base_rules: HashMap, PushRule>, @@ -549,7 +549,7 @@ impl PushRules { /// A wrapper around `PushRules` that checks the enabled state of rules and /// filters out disabled experimental rules. #[derive(Debug, Clone, Default)] -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct FilteredPushRules { push_rules: PushRules, enabled_map: BTreeMap, diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index dbc962174d..47473cf200 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -93,7 +93,7 @@ impl PushRuleRoomFlag { } /// An object which describes the unique attributes of a room version. -#[pyclass(frozen, eq, hash, get_all)] +#[pyclass(frozen, eq, hash, get_all, skip_from_py_object)] #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RoomVersion { /// The identifier for this version. @@ -622,7 +622,7 @@ const ROOM_VERSION_MSC4242V12: RoomVersion = RoomVersion { /// Note: room versions can be added to this mapping at startup (allowing /// support for experimental room versions to be behind experimental feature /// flags). -#[pyclass(frozen, mapping)] +#[pyclass(frozen, mapping, skip_from_py_object)] #[derive(Clone)] pub struct KnownRoomVersionsMapping { // Note we use a Vec here to ensure that the order of keys is @@ -637,19 +637,22 @@ pub struct KnownRoomVersionsMapping { impl KnownRoomVersionsMapping { /// Add a new room version to the mapping, indicating that this instance /// supports it. - fn add_room_version(&self, version: RoomVersion) -> PyResult<()> { + fn add_room_version(&self, version: Bound<'_, RoomVersion>) -> PyResult<()> { let mut versions = self .versions .write() .map_err(|_| PyRuntimeError::new_err("KnownRoomVersionsMapping lock poisoned"))?; - if versions.iter().any(|v| v.identifier == version.identifier) { + if versions + .iter() + .any(|v| v.identifier == version.get().identifier) + { // We already have this room version, so we don't add it again (as // otherwise we'd end up with duplicates). return Ok(()); } - versions.push(version); + versions.push(*version.get()); Ok(()) } diff --git a/tests/synapse_rust/test_unsigned.py b/tests/synapse_rust/test_unsigned.py new file mode 100644 index 0000000000..5193188f38 --- /dev/null +++ b/tests/synapse_rust/test_unsigned.py @@ -0,0 +1,52 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.synapse_rust.events import Unsigned + +from tests import unittest + + +class UnsignedTestCase(unittest.TestCase): + def test_prev_content(self) -> None: + """Test that the prev_content field is correctly exposed as a JsonObject.""" + unsigned = Unsigned({"prev_content": {"key1": "value1", "key2": 42}}) + + self.assert_dict(unsigned["prev_content"], {"key1": "value1", "key2": 42}) + + self.assert_dict( + unsigned.for_event(), {"prev_content": {"key1": "value1", "key2": 42}} + ) + + def test_large_age_ts(self) -> None: + """Test that we can handle integers larger than 2^128, which is larger + than the maximum rust native integer size.""" + + large_int = 2**200 + unsigned = Unsigned({"age_ts": large_int}) + + self.assertEqual(unsigned["age_ts"], large_int) + + self.assert_dict(unsigned.for_event(), {"age_ts": large_int}) + + def test_large_integer_in_prev_content(self) -> None: + """Test that we can handle integers larger than 2^128 in the + prev_content field, which is a JsonObject and thus can contain arbitrary + JSON.""" + + large_int = 2**200 + unsigned = Unsigned({"prev_content": {"some_field": large_int}}) + + self.assertEqual(unsigned["prev_content"]["some_field"], large_int) + self.assert_dict( + unsigned.for_event(), {"prev_content": {"some_field": large_int}} + ) From f109c259609532ec6b0c4c9f7da4b25ddfa9aed4 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 13 May 2026 12:01:11 +0100 Subject: [PATCH 17/42] 1.153.0rc2 --- CHANGES.md | 7 +++++++ changelog.d/19769.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/19769.bugfix diff --git a/CHANGES.md b/CHANGES.md index e7bd8f351a..3a66c6dd39 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.153.0rc2 (2026-05-13) + +## Bugfixes + +- Correctly handle arbitrary precision integers in `unsigned` field of events. The bug was introduced in 1.153.0rc1. ([\#19769](https://github.com/element-hq/synapse/issues/19769)) + + # Synapse 1.153.0rc1 (2026-05-08) ## Features diff --git a/changelog.d/19769.bugfix b/changelog.d/19769.bugfix deleted file mode 100644 index 05aae44311..0000000000 --- a/changelog.d/19769.bugfix +++ /dev/null @@ -1 +0,0 @@ -Correctly handle arbitrary precision integers in `unsigned` field of events. diff --git a/debian/changelog b/debian/changelog index 6aa3735603..bbd164b2a8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.153.0~rc2) stable; urgency=medium + + * New Synapse release 1.153.0rc2. + + -- Synapse Packaging team Wed, 13 May 2026 12:00:39 +0100 + matrix-synapse-py3 (1.153.0~rc1) stable; urgency=medium * New Synapse release 1.153.0rc1. diff --git a/pyproject.toml b/pyproject.toml index b456adfc60..a7422e1987 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.153.0rc1" +version = "1.153.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 16c17f3a420242e53088337d48b1fb55a86e3a8f Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 13 May 2026 17:26:16 +0200 Subject: [PATCH 18/42] Add CVE IDs to changelog for 1.152.1. (#19778) Since this is just a change log update, I've removed the entire checklist. Please tell me if this is incorrect. --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d9b3f8b2c1..f0488cd68c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,8 +2,8 @@ ## Security Fixes -- Prevent CPU starvation (Denial of Service) under worker lock contention, additionally capping the `WorkerLock` time out interval to a maximum of 60 seconds. Contributed by Famedly. ([\#19394](https://github.com/element-hq/synapse/issues/19394), ELEMENTSEC-2026-1706, [GHSA-8q93-326v-3m7g](https://github.com/element-hq/synapse/security/advisories/GHSA-8q93-326v-3m7g), CVE pending) -- Prevent pagination ending when a page is full of rejected events. (ELEMENTSEC-2025-1636, [GHSA-6qf2-7x63-mm6v](https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v), CVE pending) +- Prevent CPU starvation (Denial of Service) under worker lock contention, additionally capping the `WorkerLock` time out interval to a maximum of 60 seconds. Contributed by Famedly. ([\#19394](https://github.com/element-hq/synapse/issues/19394), ELEMENTSEC-2026-1706, [GHSA-8q93-326v-3m7g](https://github.com/element-hq/synapse/security/advisories/GHSA-8q93-326v-3m7g), CVE-2026-45078) +- Prevent pagination ending when a page is full of rejected events. (ELEMENTSEC-2025-1636, [GHSA-6qf2-7x63-mm6v](https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v), CVE-2026-45076) # Synapse 1.152.0 (2026-04-28) From ff0420a03c462fd7a4f67ccca76b58578296d142 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 13 May 2026 12:31:38 -0500 Subject: [PATCH 19/42] Improve `We can't get valid state history.` logging (#19765) Add `event_id` so you can actually correlate everything together in the logs. --- changelog.d/19765.misc | 1 + synapse/handlers/federation_event.py | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 changelog.d/19765.misc diff --git a/changelog.d/19765.misc b/changelog.d/19765.misc new file mode 100644 index 0000000000..7ac5375c7e --- /dev/null +++ b/changelog.d/19765.misc @@ -0,0 +1 @@ +Improve Synapse logging around when someone encounters `We can't get valid state history.` so you can correlate everything by `event_id`. diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index e314180e12..7cbc52d350 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -1166,9 +1166,10 @@ class FederationEventHandler: return await self._state_handler.compute_event_context(event) logger.info( - "Event %s is missing prev_events %s: calculating state for a " - "backwards extremity", + "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Event in room %s is missing prev_events %s: " + "calculating state for a backwards extremity", event_id, + room_id, shortstr(missing_prevs), ) # Calculate the state after each of the previous events, and @@ -1186,16 +1187,21 @@ class FederationEventHandler: # Ask the remote server for the states we don't # know about - for p in missing_prevs: - logger.info("Requesting state after missing prev_event %s", p) + for missing_prev in missing_prevs: + logger.info( + "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Requesting state from %s for missing prev_event %s", + event_id, + dest, + missing_prev, + ) - with nested_logging_context(p): + with nested_logging_context(missing_prev): # note that if any of the missing prevs share missing state or # auth events, the requests to fetch those events are deduped # by the get_pdu_cache in federation_client. remote_state_map = ( await self._get_state_ids_after_missing_prev_event( - dest, room_id, p + dest, room_id, missing_prev ) ) @@ -1225,7 +1231,11 @@ class FederationEventHandler: except Exception as e: logger.warning( - "Error attempting to resolve state at missing prev_events: %s", e + "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Error attempting to resolve state from " + "%s for missing prev_events: %s", + event_id, + dest, + e, ) raise FederationError( "ERROR", From b90a0e9fe99b2f66e0bb2911ac88f22437646590 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2026 10:29:16 +0100 Subject: [PATCH 20/42] Use `StrCollection` for `prev_state_events`. (#19777) Convert `prev_state_events` to use `StrCollection` rather than requiring it to be a mutable list. None of the usages require it to be a proper list, and besides, events are immutable and therefore so should `event.prev_state_events`. --- changelog.d/19777.misc | 1 + synapse/events/__init__.py | 2 +- synapse/events/builder.py | 2 +- synapse/handlers/message.py | 9 +++++---- 4 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 changelog.d/19777.misc diff --git a/changelog.d/19777.misc b/changelog.d/19777.misc new file mode 100644 index 0000000000..cd049f8153 --- /dev/null +++ b/changelog.d/19777.misc @@ -0,0 +1 @@ +Use `StrCollection` for `prev_state_events`. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 5be0298c30..fcb169e079 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -618,7 +618,7 @@ class FrozenEventVMSC4242(FrozenEventV4): """FrozenEventVMSC4242, which differs from FrozenEventV4 only in the addition of prev_state_events""" format_version = EventFormatVersions.ROOM_VMSC4242 - prev_state_events: DictProperty[list[str]] = DictProperty("prev_state_events") + prev_state_events: DictProperty[StrCollection] = DictProperty("prev_state_events") def __init__( self, diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 78eb98e1e5..742bb82da9 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -132,7 +132,7 @@ class EventBuilder: prev_event_ids: list[str], auth_event_ids: list[str] | None, depth: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, ) -> EventBase: """Transform into a fully signed and hashed event diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 0687c9fa79..56c639f5ef 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -79,6 +79,7 @@ from synapse.types import ( Requester, RoomAlias, StateMap, + StrCollection, StreamToken, UserID, create_requester, @@ -589,7 +590,7 @@ class EventCreationHandler: state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, delay_id: str | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """ @@ -982,7 +983,7 @@ class EventCreationHandler: ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: """ @@ -1127,7 +1128,7 @@ class EventCreationHandler: ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: room_id = event_dict["room_id"] @@ -1253,7 +1254,7 @@ class EventCreationHandler: state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """Create a new event for a local client. If bool for_batch is true, will create an event using the prev_event_ids, and will create an event context for From ace8447037a2d206fc1385b2d4969e6fb7445aa0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2026 11:21:00 +0100 Subject: [PATCH 21/42] Tidy up Rust RoomVersion structs (#19766) This is in prep for using the room versions more from Rust. Main changes: - Change it so each room version is defined as a delta to the last one. This is a cosmetic change that makes it easier to ensure the room version definitions are correct (as they're defined as deltas from previous versions). - Move constants to `RoomVersion` constants, like `RoomVersion::V1`, for convenience. - Change visibility of various attributes. --- changelog.d/19766.misc | 1 + rust/src/events/json_object.rs | 8 +- rust/src/room_versions.rs | 733 +++++++++++---------------------- 3 files changed, 246 insertions(+), 496 deletions(-) create mode 100644 changelog.d/19766.misc diff --git a/changelog.d/19766.misc b/changelog.d/19766.misc new file mode 100644 index 0000000000..699852daa2 --- /dev/null +++ b/changelog.d/19766.misc @@ -0,0 +1 @@ +Tidy up Rust `RoomVersion` structs. diff --git a/rust/src/events/json_object.rs b/rust/src/events/json_object.rs index 2c4be1c87b..0ab54e8dc5 100644 --- a/rust/src/events/json_object.rs +++ b/rust/src/events/json_object.rs @@ -34,7 +34,7 @@ use serde::{Deserialize, Serialize}; /// 1. Fast access of top-level keys (e.g. `event.content["key"]`) /// 2. Pure Rust implementation. #[derive(Serialize, Deserialize, Clone, Default)] -#[pyclass(mapping, frozen)] +#[pyclass(mapping, frozen, skip_from_py_object)] #[serde(transparent)] pub struct JsonObject { object: Arc, serde_json::Value>>, @@ -197,7 +197,7 @@ impl JsonObject { /// of the object. /// /// This needs to both be iterable *and* operate like a set. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] pub struct JsonObjectKeysView { object: JsonObject, @@ -397,7 +397,7 @@ impl JsonObjectKeysView { /// Helper class returned by `JsonObject.values()` to act as a view into the /// values of the object. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] pub struct JsonObjectValuesView { object: JsonObject, @@ -440,7 +440,7 @@ impl JsonObjectValuesView { /// unless the values are unhashable. Since the values are immutable we could /// support it, but it's more work and nobody seems to actually use the set /// operations on `dict_items` in practice. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] pub struct JsonObjectItemsView { object: JsonObject, diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index 47473cf200..2860bfdbd7 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -14,7 +14,11 @@ //! Rust implementation of room version definitions. -use std::sync::{Arc, LazyLock, RwLock}; +use std::{ + fmt::Display, + str::FromStr, + sync::{Arc, LazyLock, RwLock}, +}; use pyo3::{ exceptions::{PyKeyError, PyRuntimeError}, @@ -30,47 +34,47 @@ use pyo3::{ /// versions that they were used or introduced in. /// The concept of an 'event format version' is specific to Synapse (the /// specification does not mention this term.) -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct EventFormatVersions {} #[pymethods] impl EventFormatVersions { /// $id:server event id format: used for room v1 and v2 #[classattr] - const ROOM_V1_V2: i32 = 1; + pub const ROOM_V1_V2: i32 = 1; /// MSC1659-style $hash event id format: used for room v3 #[classattr] - const ROOM_V3: i32 = 2; + pub const ROOM_V3: i32 = 2; /// MSC1884-style $hash format: introduced for room v4 #[classattr] - const ROOM_V4_PLUS: i32 = 3; + pub const ROOM_V4_PLUS: i32 = 3; /// MSC4291 room IDs as hashes: introduced for room HydraV11 #[classattr] - const ROOM_V11_HYDRA_PLUS: i32 = 4; + pub const ROOM_V11_HYDRA_PLUS: i32 = 4; /// MSC4242 state DAGs: adds prev_state_events, removes auth_events #[classattr] - const ROOM_VMSC4242: i32 = 5; + pub const ROOM_VMSC4242: i32 = 5; } /// Enum to identify the state resolution algorithms. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct StateResolutionVersions {} #[pymethods] impl StateResolutionVersions { /// Room v1 state res #[classattr] - const V1: i32 = 1; + pub const V1: i32 = 1; /// MSC1442 state res: room v2 and later #[classattr] - const V2: i32 = 2; + pub const V2: i32 = 2; /// MSC4297 state res #[classattr] - const V2_1: i32 = 3; + pub const V2_1: i32 = 3; } /// Room disposition constants. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct RoomDisposition {} #[pymethods] @@ -94,18 +98,18 @@ impl PushRuleRoomFlag { /// An object which describes the unique attributes of a room version. #[pyclass(frozen, eq, hash, get_all, skip_from_py_object)] -#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RoomVersion { /// The identifier for this version. pub identifier: &'static str, - /// One of the RoomDisposition constants. + /// One of the [`RoomDisposition`] constants. pub disposition: &'static str, - /// One of the EventFormatVersions constants. + /// One of the [`EventFormatVersions`] constants. pub event_format: i32, - /// One of the StateResolutionVersions constants. + /// One of the [`StateResolutionVersions`] constants. pub state_res: i32, pub enforce_key_validity: bool, - /// Before MSC2432, m.room.aliases had special auth rules and redaction rules. + /// Before MSC2432, `m.room.aliases` had special auth rules and redaction rules. pub special_case_aliases_auth: bool, /// Strictly enforce canonicaljson, do not allow: /// * Integers outside the range of [-2^53 + 1, 2^53 - 1] @@ -159,459 +163,204 @@ pub struct RoomVersion { pub msc4242_state_dags: bool, } -const ROOM_VERSION_V1: RoomVersion = RoomVersion { - identifier: "1", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V1_V2, - state_res: StateResolutionVersions::V1, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; +impl RoomVersion { + pub const V1: RoomVersion = RoomVersion { + identifier: "1", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V1_V2, + state_res: StateResolutionVersions::V1, + enforce_key_validity: false, + special_case_aliases_auth: true, + strict_canonicaljson: false, + limit_notifications_power_levels: false, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: false, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, + }; -const ROOM_VERSION_V2: RoomVersion = RoomVersion { - identifier: "2", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V1_V2, - state_res: StateResolutionVersions::V2, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + pub const V2: RoomVersion = RoomVersion { + identifier: "2", + state_res: StateResolutionVersions::V2, + ..Self::V1 + }; -const ROOM_VERSION_V3: RoomVersion = RoomVersion { - identifier: "3", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V3, - state_res: StateResolutionVersions::V2, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + pub const V3: RoomVersion = RoomVersion { + identifier: "3", + event_format: EventFormatVersions::ROOM_V3, + ..Self::V2 + }; -const ROOM_VERSION_V4: RoomVersion = RoomVersion { - identifier: "4", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + pub const V4: RoomVersion = RoomVersion { + identifier: "4", + event_format: EventFormatVersions::ROOM_V4_PLUS, + ..Self::V3 + }; -const ROOM_VERSION_V5: RoomVersion = RoomVersion { - identifier: "5", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + pub const V5: RoomVersion = RoomVersion { + identifier: "5", + enforce_key_validity: true, + ..Self::V4 + }; -const ROOM_VERSION_V6: RoomVersion = RoomVersion { - identifier: "6", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + pub const V6: RoomVersion = RoomVersion { + identifier: "6", + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + ..Self::V5 + }; -const ROOM_VERSION_V7: RoomVersion = RoomVersion { - identifier: "7", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + pub const V7: RoomVersion = RoomVersion { + identifier: "7", + knock_join_rule: true, + ..Self::V6 + }; -const ROOM_VERSION_V8: RoomVersion = RoomVersion { - identifier: "8", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: false, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + pub const V8: RoomVersion = RoomVersion { + identifier: "8", + restricted_join_rule: true, + ..Self::V7 + }; -const ROOM_VERSION_V9: RoomVersion = RoomVersion { - identifier: "9", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + pub const V9: RoomVersion = RoomVersion { + identifier: "9", + restricted_join_rule_fix: true, + ..Self::V8 + }; -const ROOM_VERSION_V10: RoomVersion = RoomVersion { - identifier: "10", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + pub const V10: RoomVersion = RoomVersion { + identifier: "10", + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + ..Self::V9 + }; -/// MSC3389 (Redaction changes for events with a relation) based on room version "10". -const ROOM_VERSION_MSC3389V10: RoomVersion = RoomVersion { - identifier: "org.matrix.msc3389.10", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: true, // Changed from v10 - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; + /// MSC3389 (Redaction changes for events with a relation) based on room version "10". + pub const MSC3389V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3389.10", + disposition: RoomDisposition::UNSTABLE, + msc3389_relation_redactions: true, + strict_event_byte_limits_room_versions: true, + ..Self::V10 + }; -/// MSC1767 (Extensible Events) based on room version "10". -const ROOM_VERSION_MSC1767V10: RoomVersion = RoomVersion { - identifier: "org.matrix.msc1767.10", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[PushRuleRoomFlag::EXTENSIBLE_EVENTS], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + /// MSC1767 (Extensible Events) based on room version "10". + pub const MSC1767V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc1767.10", + disposition: RoomDisposition::UNSTABLE, + msc3931_push_features: &[PushRuleRoomFlag::EXTENSIBLE_EVENTS], + ..Self::V10 + }; -/// MSC3757 (Restricting who can overwrite a state event) based on room version "10". -const ROOM_VERSION_MSC3757V10: RoomVersion = RoomVersion { - identifier: "org.matrix.msc3757.10", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: true, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; + /// MSC3757 (Restricting who can overwrite a state event) based on room version "10". + pub const MSC3757V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3757.10", + disposition: RoomDisposition::UNSTABLE, + msc3757_enabled: true, + ..Self::V10 + }; -const ROOM_VERSION_V11: RoomVersion = RoomVersion { - identifier: "11", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: true, // Changed from v10 - msc4242_state_dags: false, -}; + pub const V11: RoomVersion = RoomVersion { + identifier: "11", + implicit_room_creator: true, // Used by MSC3820 + updated_redaction_rules: true, // Used by MSC3820 + strict_event_byte_limits_room_versions: true, + ..Self::V10 + }; -/// MSC3757 (Restricting who can overwrite a state event) based on room version "11". -const ROOM_VERSION_MSC3757V11: RoomVersion = RoomVersion { - identifier: "org.matrix.msc3757.11", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: true, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; + /// MSC3757 (Restricting who can overwrite a state event) based on room version "11". + pub const MSC3757V11: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3757.11", + disposition: RoomDisposition::UNSTABLE, + msc3757_enabled: true, + ..Self::V11 + }; -const ROOM_VERSION_HYDRA_V11: RoomVersion = RoomVersion { - identifier: "org.matrix.hydra.11", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, - state_res: StateResolutionVersions::V2_1, // Changed from v11 - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: true, // Changed from v11 - msc4291_room_ids_as_hashes: true, // Changed from v11 - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; + pub const HYDRA_V11: RoomVersion = RoomVersion { + identifier: "org.matrix.hydra.11", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, + state_res: StateResolutionVersions::V2_1, + msc4289_creator_power_enabled: true, + msc4291_room_ids_as_hashes: true, + ..Self::V11 + }; -const ROOM_VERSION_V12: RoomVersion = RoomVersion { - identifier: "12", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, - state_res: StateResolutionVersions::V2_1, // Changed from v11 - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: true, // Changed from v11 - msc4291_room_ids_as_hashes: true, // Changed from v11 - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; + pub const V12: RoomVersion = RoomVersion { + identifier: "12", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, + state_res: StateResolutionVersions::V2_1, + msc4289_creator_power_enabled: true, + msc4291_room_ids_as_hashes: true, + ..Self::V11 + }; -const ROOM_VERSION_MSC4242V12: RoomVersion = RoomVersion { - identifier: "org.matrix.msc4242.12", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_VMSC4242, - state_res: StateResolutionVersions::V2_1, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, - updated_redaction_rules: true, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: true, - msc4291_room_ids_as_hashes: true, - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: true, -}; + pub const MSC4242V12: RoomVersion = RoomVersion { + identifier: "org.matrix.msc4242.12", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_VMSC4242, + msc4242_state_dags: true, + ..Self::V12 + }; +} + +impl Display for RoomVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.identifier.fmt(f) + } +} + +impl FromStr for &'static RoomVersion { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "1" => Ok(&RoomVersion::V1), + "2" => Ok(&RoomVersion::V2), + "3" => Ok(&RoomVersion::V3), + "4" => Ok(&RoomVersion::V4), + "5" => Ok(&RoomVersion::V5), + "6" => Ok(&RoomVersion::V6), + "7" => Ok(&RoomVersion::V7), + "8" => Ok(&RoomVersion::V8), + "9" => Ok(&RoomVersion::V9), + "10" => Ok(&RoomVersion::V10), + "11" => Ok(&RoomVersion::V11), + "12" => Ok(&RoomVersion::V12), + "org.matrix.msc1767.10" => Ok(&RoomVersion::MSC1767V10), + "org.matrix.msc3389.10" => Ok(&RoomVersion::MSC3389V10), + "org.matrix.msc3757.10" => Ok(&RoomVersion::MSC3757V10), + "org.matrix.msc3757.11" => Ok(&RoomVersion::MSC3757V11), + "org.matrix.hydra.11" => Ok(&RoomVersion::HYDRA_V11), + "org.matrix.msc4242.12" => Ok(&RoomVersion::MSC4242V12), + _ => Err(anyhow::anyhow!("Unknown room version: {}", s)), + } + } +} + +impl<'py> IntoPyObject<'py> for &RoomVersion { + type Target = RoomVersion; + + type Output = Bound<'py, RoomVersion>; + + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.clone().into_pyobject(py) + } +} /// Helper class for managing the known room versions, and providing dict-like /// access to them for Python. @@ -652,7 +401,7 @@ impl KnownRoomVersionsMapping { return Ok(()); } - versions.push(*version.get()); + versions.push(version.get().clone()); Ok(()) } @@ -666,7 +415,7 @@ impl KnownRoomVersionsMapping { versions .iter() .find(|v| v.identifier == key) - .copied() + .cloned() .ok_or_else(|| PyKeyError::new_err(key.to_string())) } @@ -698,7 +447,7 @@ impl KnownRoomVersionsMapping { // like a Set *and* has a stable ordering). We don't depend on this, so // for simplicity we just return a list of the items. let versions = self.versions.read().unwrap(); - Ok(versions.iter().map(|v| (v.identifier, *v)).collect()) + Ok(versions.iter().map(|v| (v.identifier, v.clone())).collect()) } #[pyo3(signature = (key, default=None))] @@ -715,8 +464,8 @@ impl KnownRoomVersionsMapping { }; let versions = self.versions.read().unwrap(); - if let Some(version) = versions.iter().find(|v| v.identifier == key).copied() { - return Ok(Some(version.into_bound_py_any(py)?)); + if let Some(version) = versions.iter().find(|v| v.identifier == key) { + return Ok(Some(version.clone().into_bound_py_any(py)?)); } Ok(default) @@ -757,21 +506,21 @@ impl<'py> IntoPyObject<'py> for &KnownRoomVersionsMapping { /// support all experimental room versions. static KNOWN_ROOM_VERSIONS: LazyLock = LazyLock::new(|| { let vec = vec![ - ROOM_VERSION_V1, - ROOM_VERSION_V2, - ROOM_VERSION_V3, - ROOM_VERSION_V4, - ROOM_VERSION_V5, - ROOM_VERSION_V6, - ROOM_VERSION_V7, - ROOM_VERSION_V8, - ROOM_VERSION_V9, - ROOM_VERSION_V10, - ROOM_VERSION_V11, - ROOM_VERSION_V12, - ROOM_VERSION_MSC3757V10, - ROOM_VERSION_MSC3757V11, - ROOM_VERSION_HYDRA_V11, + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + RoomVersion::MSC3757V10, + RoomVersion::MSC3757V11, + RoomVersion::HYDRA_V11, ]; KnownRoomVersionsMapping { @@ -783,82 +532,82 @@ static KNOWN_ROOM_VERSIONS: LazyLock = LazyLock::new(| /// /// This should contain all room versions that we know about. #[pyclass(frozen)] -pub struct RoomVersions {} +struct RoomVersions {} #[pymethods] #[allow(non_snake_case)] impl RoomVersions { #[classattr] fn V1(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V1.into_py_any(py) + RoomVersion::V1.into_py_any(py) } #[classattr] fn V2(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V2.into_py_any(py) + RoomVersion::V2.into_py_any(py) } #[classattr] fn V3(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V3.into_py_any(py) + RoomVersion::V3.into_py_any(py) } #[classattr] fn V4(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V4.into_py_any(py) + RoomVersion::V4.into_py_any(py) } #[classattr] fn V5(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V5.into_py_any(py) + RoomVersion::V5.into_py_any(py) } #[classattr] fn V6(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V6.into_py_any(py) + RoomVersion::V6.into_py_any(py) } #[classattr] fn V7(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V7.into_py_any(py) + RoomVersion::V7.into_py_any(py) } #[classattr] fn V8(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V8.into_py_any(py) + RoomVersion::V8.into_py_any(py) } #[classattr] fn V9(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V9.into_py_any(py) + RoomVersion::V9.into_py_any(py) } #[classattr] fn V10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V10.into_py_any(py) + RoomVersion::V10.into_py_any(py) } #[classattr] fn MSC1767v10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC1767V10.into_py_any(py) + RoomVersion::MSC1767V10.into_py_any(py) } #[classattr] fn MSC3389v10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC3389V10.into_py_any(py) + RoomVersion::MSC3389V10.into_py_any(py) } #[classattr] fn MSC3757v10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC3757V10.into_py_any(py) + RoomVersion::MSC3757V10.into_py_any(py) } #[classattr] fn V11(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V11.into_py_any(py) + RoomVersion::V11.into_py_any(py) } #[classattr] fn MSC3757v11(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC3757V11.into_py_any(py) + RoomVersion::MSC3757V11.into_py_any(py) } #[classattr] fn HydraV11(py: Python<'_>) -> PyResult> { - ROOM_VERSION_HYDRA_V11.into_py_any(py) + RoomVersion::HYDRA_V11.into_py_any(py) } #[classattr] fn V12(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V12.into_py_any(py) + RoomVersion::V12.into_py_any(py) } #[classattr] fn MSC4242v12(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC4242V12.into_py_any(py) + RoomVersion::MSC4242V12.into_py_any(py) } } From 86a1e73ef4cb28640727a3fd1cf77ca8ba052454 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2026 19:02:33 +0100 Subject: [PATCH 22/42] Consolidate MSC4242 state DAG checks via a `TypeIs` helper (#19774) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reason for the change is to make it easier to support these checks when porting event class to Rust. Previously, code that needed to access `prev_state_events` had to combine a `room_version.msc4242_state_dags` boolean check with an `isinstance(event, FrozenEventVMSC4242)` cast (or `cast()`) for the type checker. Introduce `supports_msc4242_state_dag()` in a new `synapse/events/py_protocol.py` which does both in one step via `TypeIs[MSC4242Event]`, removing the need to import the concrete `FrozenEventVMSC4242` class at every call site. `MSC4242Event` is an `EventBase` subclass used purely for type narrowing — it's marked with a metaclass that rejects `isinstance()` to make accidental runtime use loud. No behavioural change: callers continue to gate on the same room version flag and access the same `prev_state_events` attribute. --- changelog.d/19774.misc | 1 + synapse/event_auth.py | 10 +-- synapse/events/py_protocol.py | 88 +++++++++++++++++++ synapse/handlers/admin.py | 6 +- synapse/handlers/message.py | 8 +- synapse/state/__init__.py | 5 +- synapse/storage/controllers/persist_events.py | 23 ++--- synapse/storage/databases/main/events.py | 9 +- tests/events/test_py_protocol.py | 82 +++++++++++++++++ tests/storage/test_msc4242_state_dag.py | 13 +-- 10 files changed, 210 insertions(+), 35 deletions(-) create mode 100644 changelog.d/19774.misc create mode 100644 synapse/events/py_protocol.py create mode 100644 tests/events/test_py_protocol.py diff --git a/changelog.d/19774.misc b/changelog.d/19774.misc new file mode 100644 index 0000000000..5a2cb4d800 --- /dev/null +++ b/changelog.d/19774.misc @@ -0,0 +1 @@ +Refactor MSC4242 state DAG checks behind a single `TypeIs` helper to avoid scattered `isinstance` casts. diff --git a/synapse/event_auth.py b/synapse/event_auth.py index fd35da8ba0..d9750651fa 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -61,7 +61,8 @@ from synapse.api.room_versions import ( EventFormatVersions, RoomVersion, ) -from synapse.events import FrozenEventVMSC4242, is_creator +from synapse.events import is_creator +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.state import CREATE_KEY from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( @@ -187,8 +188,8 @@ async def check_state_independent_auth_rules( return # State DAGs 2. Considering the event's prev_state_events: - if event.room_version.msc4242_state_dags: - prev_state_events_ids = set(cast(FrozenEventVMSC4242, event).prev_state_events) + if supports_msc4242_state_dag(event): + prev_state_events_ids = set(event.prev_state_events) # Fetch all of the `prev_state_events` prev_state_events = {} # Try to load the `prev_state_events` from `batched_auth_events` initially as @@ -515,8 +516,7 @@ def _check_create(event: "EventBase") -> None: raise AuthError(403, "Create event has prev events") # State DAGs 1.2 If it has any prev_state_events, reject. - if event.room_version.msc4242_state_dags: - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event): if len(event.prev_state_events) > 0: raise AuthError(403, "Create event has prev state events") diff --git a/synapse/events/py_protocol.py b/synapse/events/py_protocol.py new file mode 100644 index 0000000000..d9ac8c066f --- /dev/null +++ b/synapse/events/py_protocol.py @@ -0,0 +1,88 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +"""Type-narrowing helpers for `EventBase`. + +`EventBase` subclasses are split by room version (e.g. `FrozenEventV4`, +`FrozenEventVMSC4242`), and certain attributes — such as `prev_state_events` +on MSC4242 events — only exist on a subset of those subclasses. Branching on +`room_version.` at runtime tells us *which* subclass we have, but the +type checker can't see that link without an `isinstance` cast at every call +site. + +This module provides "marker" subclasses of `EventBase` (`MSC4242Event`, +etc.) paired with `TypeIs`-returning predicates (`supports_msc4242_state_dag`, +etc.). A single call to the predicate both performs the room-version check +and narrows the type — replacing the `if room_version.foo: assert +isinstance(event, FrozenEventV...)` idiom. + +The marker classes are *type-only*: their metaclass raises on `isinstance` +so they cannot be misused as real runtime classes. Add new markers and +predicates here when a new room-version feature gates access to additional +attributes. +""" + +import abc +from typing import ( + TYPE_CHECKING, + Sequence, +) + +from typing_extensions import TypeIs + +from synapse.events import EventBase + +if TYPE_CHECKING: + from synapse.events.snapshot import EventContext, EventPersistencePair + + +class _DisableIsInstance(abc.ABCMeta): + """Metaclass which disables isinstance checks on classes which use it, by + making isinstance() raise NotImplementedError. + + This is used to prevent isinstance checks on EventProtocol, which is a + helper class used for type narrowing of EventBase objects, but which should + not be used for isinstance checks itself (as its purely type annotation + rather than a real class). + """ + + def __instancecheck__(cls, instance: object) -> bool: + raise NotImplementedError("Instance cannot be used.") + + +class EventProtocol(EventBase, metaclass=_DisableIsInstance): + """Helper subclass that allows type narrowing for `EventBase` objects.""" + + +class MSC4242Event(EventProtocol): + """Marker protocol for events in MSC4242 rooms. This allows us to narrow the + type of events.""" + + prev_state_events: list[str] + + +def supports_msc4242_state_dag(event: EventBase) -> TypeIs[MSC4242Event]: + """Returns true if the given event is in a room that supports state DAGs + (MSC4242)""" + + return event.room_version.msc4242_state_dags + + +def all_supports_msc4242_state_dag( + obj: Sequence["EventPersistencePair"], +) -> TypeIs[Sequence[tuple[MSC4242Event, "EventContext"]]]: + """Returns true if the given sequence of events are all in a room that + supports state DAGs (MSC4242)""" + + return all(event.room_version.msc4242_state_dags for event, _ in obj) diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 51a752472f..0b1251f9e3 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -32,7 +32,8 @@ import attr from synapse.api.constants import Direction, EventTypes, Membership from synapse.api.errors import SynapseError -from synapse.events import EventBase, FrozenEventVMSC4242 +from synapse.events import EventBase +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.utils import FilteredEvent from synapse.types import ( JsonMapping, @@ -495,8 +496,7 @@ class AdminHandler: try: prev_state_events = None - if room_version.msc4242_state_dags: - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event): prev_state_events = event.prev_state_events assert prev_state_events is not None, ( "Parent event of redaction has no `prev_state_events` which should be impossible as `prev_state_events` is a required field in MSC4242 rooms" diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 56c639f5ef..bade219ddf 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -53,8 +53,9 @@ from synapse.api.errors import ( from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.api.urls import ConsentURIBuilder from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase, FrozenEventVMSC4242, relation_from_event +from synapse.events import EventBase, relation_from_event from synapse.events.builder import EventBuilder +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import ( EventContext, EventPersistencePair, @@ -1603,8 +1604,7 @@ class EventCreationHandler: auth_event = event_id_to_event.get(event_id) if auth_event: batched_auth_events[event_id] = auth_event - if event.room_version.msc4242_state_dags: - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event): # State DAG rooms will check that the prev_state_events are not rejected. # To do that, we need to make sure we pass in the prev_state_events as # batched_auth_events, else we will fail the event due to the @@ -1873,7 +1873,7 @@ class EventCreationHandler: state_entry = await self.state.resolve_state_groups_for_events( event.room_id, event_ids=event.prev_state_events - if isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event) else event.prev_event_ids(), ) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 2f0e3f2c3e..b22f91b37c 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -37,7 +37,8 @@ from prometheus_client import Counter, Histogram from synapse.api.constants import EventTypes from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, StateResolutionVersions -from synapse.events import EventBase, FrozenEventVMSC4242 +from synapse.events import EventBase +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import ( EventContext, UnpersistedEventContext, @@ -315,7 +316,7 @@ class StateHandler: # might redundantly recalculate the state for this event later.) prev_event_ids = frozenset( event.prev_state_events - if isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event) else event.prev_event_ids() ) incomplete_prev_events = await self.store.get_partial_state_events( diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 7cc6a39639..62e84f5ac5 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -34,6 +34,7 @@ from typing import ( Generator, Generic, Iterable, + Sequence, TypeVar, cast, ) @@ -46,7 +47,14 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.api.errors import SynapseError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS -from synapse.events import EventBase, FrozenEventVMSC4242, event_exists_in_state_dag +from synapse.events import ( + EventBase, + event_exists_in_state_dag, +) +from synapse.events.py_protocol import ( + MSC4242Event, + all_supports_msc4242_state_dag, +) from synapse.events.snapshot import EventContext, EventPersistencePair from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable @@ -637,7 +645,6 @@ class EventsPersistenceStorageController: # Get the room version for the first event. This room version is the same for all events # as events_and_contexts is all for one room. assert len(events_and_contexts) > 0 - room_version = events_and_contexts[0][0].room_version for chunk in chunks: # We can't easily parallelize these since different chunks @@ -648,22 +655,18 @@ class EventsPersistenceStorageController: new_state_dag_extrems = None if not backfilled: - if room_version.msc4242_state_dags: + if all_supports_msc4242_state_dag(chunk): with Measure( self._clock, name="_process_state_dag_forward_extremities_and_state_delta", server_name=self.server_name, ): - assert all( - isinstance(ev, FrozenEventVMSC4242) for ev, _ in chunk - ) ( new_forward_extremities, # for prev_events state_delta_for_room, # for state groups new_state_dag_extrems, # for prev_state_events ) = await self._process_state_dag_forward_extremities_and_state_delta( - room_id, - cast(list[tuple[FrozenEventVMSC4242, EventContext]], chunk), + room_id, chunk ) else: with Measure( @@ -840,7 +843,7 @@ class EventsPersistenceStorageController: async def _process_state_dag_forward_extremities_and_state_delta( self, room_id: str, - event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + event_contexts: Sequence[tuple[MSC4242Event, EventContext]], ) -> tuple[set[str] | None, DeltaState | None, set[str] | None]: """Process the forwards extremities for state DAG rooms. Returns: @@ -933,7 +936,7 @@ class EventsPersistenceStorageController: self, room_id: str, existing_fwd_extrems: frozenset[str], - event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + event_contexts: Sequence[tuple[MSC4242Event, EventContext]], ) -> set[str]: """Calculate the new state dag forward extremities. Modifies existing_fwd_extrems. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 5f6b03a988..5aab0067fc 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -48,12 +48,12 @@ from synapse.api.errors import PartialStateConflictError from synapse.api.room_versions import RoomVersions from synapse.events import ( EventBase, - FrozenEventVMSC4242, StrippedStateEvent, event_exists_in_state_dag, is_creator, relation_from_event, ) +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.events.snapshot import EventPersistencePair from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import trace @@ -2897,10 +2897,7 @@ class PersistEventsStore: self._handle_event_relations(txn, event) - if event.room_version.msc4242_state_dags and event_exists_in_state_dag( - event - ): - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event) and event_exists_in_state_dag(event): self._store_state_dag_edges(txn, event) # Store the labels for this event. @@ -2980,7 +2977,7 @@ class PersistEventsStore: txn.call_after(local_prefill) def _store_state_dag_edges( - self, txn: LoggingTransaction, event: FrozenEventVMSC4242 + self, txn: LoggingTransaction, event: MSC4242Event ) -> None: # the create event has no edge but we still need to persist it as get_state_dag just # yanks all rows in this table. It's a bit gross to store NULL as the prev_state_event_id diff --git a/tests/events/test_py_protocol.py b/tests/events/test_py_protocol.py new file mode 100644 index 0000000000..306e3c1704 --- /dev/null +++ b/tests/events/test_py_protocol.py @@ -0,0 +1,82 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +from unittest.mock import Mock + +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.events import EventBase, FrozenEvent, make_event_from_dict +from synapse.events.py_protocol import ( + EventProtocol, + MSC4242Event, + all_supports_msc4242_state_dag, + supports_msc4242_state_dag, +) + +from tests.unittest import TestCase + + +def _make_event(room_version: RoomVersion) -> EventBase: + """Helper to make an EventBase with the given room version.""" + event_dict = { + "content": {}, + "sender": "@user:example.com", + "type": "m.room.message", + "room_id": "!room:example.com", + } + if room_version.msc4242_state_dags: + event_dict["prev_state_events"] = [] + return make_event_from_dict(event_dict, room_version=room_version) + + +class TestMetaClass(TestCase): + def test_is_instance(self) -> None: + """Test that isinstance checks on EventProtocol raise + NotImplementedError, but that isinstance checks on EventBase and + FrozenEvent still work as normal. + """ + # EventBase and FrozenEvent should work as normal + self.assertFalse(isinstance(object(), EventBase)) + self.assertFalse(isinstance(object(), FrozenEvent)) + + with self.assertRaises(NotImplementedError): + isinstance(object(), EventProtocol) + + with self.assertRaises(NotImplementedError): + isinstance(object(), MSC4242Event) + + +class SupportsMSC4242StateDagTestCase(TestCase): + def test_single_event_msc4242(self) -> None: + """A single event in an MSC4242 room is recognised.""" + ev = _make_event(RoomVersions.MSC4242v12) + self.assertTrue(supports_msc4242_state_dag(ev)) + + def test_single_event_non_msc4242(self) -> None: + """A single event in a non-MSC4242 room is not recognised.""" + ev = _make_event(RoomVersions.V11) + self.assertFalse(supports_msc4242_state_dag(ev)) + + def test_sequence_all_msc4242(self) -> None: + """A sequence of MSC4242 (event, context) pairs is recognised.""" + pairs = [(_make_event(RoomVersions.MSC4242v12), Mock()) for _ in range(3)] + self.assertTrue(all_supports_msc4242_state_dag(pairs)) + + def test_sequence_mixed(self) -> None: + """A sequence containing any non-MSC4242 event is not recognised.""" + pairs = [ + (_make_event(RoomVersions.MSC4242v12), Mock()), + (_make_event(RoomVersions.V11), Mock()), + ] + self.assertFalse(all_supports_msc4242_state_dag(pairs)) diff --git a/tests/storage/test_msc4242_state_dag.py b/tests/storage/test_msc4242_state_dag.py index 8775e5c8eb..6720c1d659 100644 --- a/tests/storage/test_msc4242_state_dag.py +++ b/tests/storage/test_msc4242_state_dag.py @@ -19,7 +19,10 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events import FrozenEventVMSC4242, make_event_from_dict +from synapse.events import ( + make_event_from_dict, +) +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.events.snapshot import EventContext from synapse.rest.client import room from synapse.server import HomeServer @@ -152,7 +155,7 @@ class MSC4242EventPersistenceStateDagsStoreTestCase(HomeserverTestCase): id: str, prev_state_events: list[str], rejected: bool = False, - ) -> tuple[FrozenEventVMSC4242, EventContext]: + ) -> tuple[MSC4242Event, EventContext]: ev = make_event_from_dict( { "prev_state_events": prev_state_events, @@ -166,8 +169,8 @@ class MSC4242EventPersistenceStateDagsStoreTestCase(HomeserverTestCase): }, room_version=RoomVersions.MSC4242v12, ) - assert isinstance(ev, FrozenEventVMSC4242) - ev._event_id = id + ev._event_id = id # type: ignore[attr-defined] + assert supports_msc4242_state_dag(ev) ctx = Mock() ctx.rejected = rejected return ev, ctx @@ -175,7 +178,7 @@ class MSC4242EventPersistenceStateDagsStoreTestCase(HomeserverTestCase): def _test( self, current_fwds: list[str], - new_events: list[tuple[FrozenEventVMSC4242, EventContext]], + new_events: list[tuple[MSC4242Event, EventContext]], want_new_extrems: set[str], want_raises: bool = False, ) -> None: From b233892a13a6b21648ffe3e333373004216d73a3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 14 May 2026 14:53:16 -0500 Subject: [PATCH 23/42] Update `wait_for_stream_token(...)` patterns and fix sync fetching with unbounded token (#19644) Spawning from trying to find the proper way to wait for a token, see https://github.com/element-hq/synapse/pull/19558#discussion_r2977673208 - Update `wait_for_stream_token(...)` patterns so validation/sanitization is handled upstream in usage. - Fix sync waiting for bounded token but using unbounded token to fetch data. Noticed while working on adding the new method. Part of https://github.com/element-hq/synapse/issues/19647 --- changelog.d/19644.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 17 +++ synapse/handlers/sync.py | 9 ++ synapse/notifier.py | 45 ++++++- synapse/streams/events.py | 10 ++ synapse/types/__init__.py | 3 + tests/handlers/test_sync.py | 14 ++- tests/test_notifier.py | 136 ++++++++++++++++++++++ 8 files changed, 228 insertions(+), 7 deletions(-) create mode 100644 changelog.d/19644.bugfix create mode 100644 tests/test_notifier.py diff --git a/changelog.d/19644.bugfix b/changelog.d/19644.bugfix new file mode 100644 index 0000000000..73ab4bc63e --- /dev/null +++ b/changelog.d/19644.bugfix @@ -0,0 +1 @@ +Fix long-standing but niche bug with sync where it could attempt to fetch data with flawed invalid future tokens. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index a3443b300c..2e26bfc43a 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -150,6 +150,23 @@ class SlidingSyncHandler: # events or future events if the user is nefariously, manually modifying the # token. if from_token is not None: + # Work around a bug where older Synapse versions gave out tokens "from the + # future", i.e. that are ahead of the tokens persisted in the DB. This could + # also happen if a user is intentionally messing with the token so this also + # acts as sanitization/validation. + # + # If the token has positions ahead of our persisted positions in the + # database (invalid), then we simply use our max persisted position (recover + # gracefully); instead of waiting for a position that may never come around. + # + # FIXME: For Sliding Sync, instead of bounding the token, we should detect + # the invalid future position and raise a `M_UNKNOWN_POS` error. + from_token = SlidingSyncStreamToken( + stream_token=await self.event_sources.bound_future_token( + from_token.stream_token + ), + connection_position=from_token.connection_position, + ) # We need to make sure this worker has caught up with the token. If # this returns false, it means we timed out waiting, and we should # just return an empty response. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c88f703ae9..bca51b64b7 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -414,6 +414,15 @@ class SyncHandler: context.tag = sync_label if since_token is not None: + # Work around a bug where older Synapse versions gave out tokens "from the + # future", i.e. that are ahead of the tokens persisted in the DB. This could + # also happen if a user is intentionally messing with the token so this also + # acts as sanitization/validation. + # + # If the token has positions ahead of our persisted positions in the + # database (invalid), then we simply use our max persisted position (recover + # gracefully); instead of waiting for a position that may never come around. + since_token = await self.event_sources.bound_future_token(since_token) # We need to make sure this worker has caught up with the token. If # this returns false it means we timed out waiting, and we should # just return an empty response. diff --git a/synapse/notifier.py b/synapse/notifier.py index f1cec74462..6a057ac09f 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -832,15 +832,49 @@ class Notifier: return result async def wait_for_stream_token(self, stream_token: StreamToken) -> bool: - """Wait for this worker to catch up with the given stream token.""" + """ + Wait for this worker to catch up with the given stream token. + + This is important to ensure that the worker has a proper view of the world + before trying to serve a request. For example, one worker can return a response + with some `next_batch` token, but then the next request goes to another worker + which is behind; if the worker assembles a response up to the token, it could be + missing data in the gap between where it's behind and the requested token. + + ### Inavlid future tokens + + We assume the token has already been validated/sanitized before being passed to + this function to ensure it's not some invalid future token. We consider a token + invalid, if the token has positions ahead of our persisted positions in the + database. This is important as we we don't want to wait for the stream to + advance in those cases (as it may never do so) (it's a waste of time for the + user and server). + + Previously, we would sanitize and `bound_future_token(...)` within this function + but that leads to bad patterns upstream where people can continue to use the + unbounded token. + + While it was possible for older Synapse versions to erroneously give out invalid + future tokens, this is no longer the case and its considered a Synapse + programming error if this ever happens. Validation/sanitization is still + necessary as a user can intentionally mess with numbers in the tokens being + provided. + + Args: + stream_token: The token to wait for. We assume the token has already been + validated/sanitized to ensure it's not some invalid future token (has a + stream position ahead of what is in the DB). (see details above) + + Returns: + True when this worker has caught up + False when we timed out waiting + """ current_token = self.event_sources.get_current_token() + # Return early if we are already caught up if stream_token.is_before_or_eq(current_token): return True - # Work around a bug where older Synapse versions gave out tokens "from - # the future", i.e. that are ahead of the tokens persisted in the DB. - stream_token = await self.event_sources.bound_future_token(stream_token) - + # Start waiting until we've caught up to the `stream_token` start = self.clock.time_msec() logged = False while True: @@ -850,6 +884,7 @@ class Notifier: now = self.clock.time_msec() + # Timed out if now - start > 10_000: return False diff --git a/synapse/streams/events.py b/synapse/streams/events.py index d2720fb959..f5677a2082 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -149,6 +149,16 @@ class EventSources: ].get_max_allocated_token() if max_token < token_value.get_max_stream_pos(): + # Log *something* as we consider this as a Synapse programming error + # (assuming no malicious user manipulation of the token) (we shouldn't be + # handing out future tokens). + # + # We don't assert as the whole point of bounding is so that we can recover + # gracefully. + # + # Old versions of Synapse could advance streams without persisting anything in + # the DB (fixed in https://github.com/element-hq/synapse/pull/17229) and on + # restart, those updates would be lost. logger.error( "Bounding token from the future '%s': token: %s, bound: %s", key, diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 02889795bb..8b005ef84d 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -685,6 +685,9 @@ class AbstractMultiWriterStreamToken(metaclass=abc.ABCMeta): def bound_stream_token(self, max_stream: int) -> "Self": """Bound the stream positions to a maximum value""" + # Shortcut if we're already under the bound + if self.get_max_stream_pos() <= max_stream: + return self min_pos = min(self.stream, max_stream) return type(self)( diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index b9dee1c954..e4a4d59aeb 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -51,6 +51,7 @@ from synapse.types import ( create_requester, ) from synapse.util.clock import Clock +from synapse.util.duration import Duration import tests.unittest import tests.utils @@ -1058,13 +1059,22 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): ) # This should block waiting for the presence stream to update - self.pump() + # + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop iterate. + self.reactor.advance(Duration(seconds=2).as_secs()) + # It should still not be done yet self.assertFalse(sync_d.called) # Marking the stream ID as persisted should unblock the request. self.get_success(ctx_mgr.__aexit__(None, None, None)) - self.get_success(sync_d, by=1.0) + # Advance time to make another iteration of `wait_for_stream_token(...)` sleep + # loop so it sees that we're finally caught up now. + self.reactor.advance(Duration(seconds=1).as_secs()) + + # Done waiting + self.get_success(sync_d) @parameterized.expand( [(key,) for key in StreamKeyType.__members__.values()], diff --git a/tests/test_notifier.py b/tests/test_notifier.py new file mode 100644 index 0000000000..c65134e832 --- /dev/null +++ b/tests/test_notifier.py @@ -0,0 +1,136 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import logging + +from twisted.internet import defer +from twisted.internet.testing import MemoryReactor + +from synapse.server import HomeServer +from synapse.types import MultiWriterStreamToken, StreamKeyType, StreamToken +from synapse.util.clock import Clock +from synapse.util.duration import Duration + +import tests.unittest + +logger = logging.getLogger(__name__) + + +class NotifierTestCase(tests.unittest.HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = self.hs.get_datastores().main + self.notifier = self.hs.get_notifier() + + def test_wait_for_stream_token_with_caught_up_token(self) -> None: + """ + Test `wait_for_stream_token` when we receive a token that we are caught up to. + """ + # Create a token + receipt_id_gen = self.store.get_receipts_stream_id_gen() + receipt_token = MultiWriterStreamToken.from_generator(receipt_id_gen) + token = StreamToken.START.copy_and_replace(StreamKeyType.RECEIPT, receipt_token) + + # Function under test + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + + # Done waiting and caught-up (True) + wait_result = self.get_success(wait_d) + self.assertEqual(wait_result, True) + + def test_wait_for_stream_token_with_future_sync_token(self) -> None: + """ + Test `wait_for_stream_token` when we receive a token that is ahead of our + current token, we'll wait until the stream position advances. + + This can happen if replication streams start lagging, and the client's + previous sync request was serviced by a worker ahead of ours. + """ + # We simulate a lagging stream by getting a stream ID from the ID gen + # and then waiting to mark it as "persisted". + receipt_id_gen = self.store.get_receipts_stream_id_gen() + ctx_mgr = receipt_id_gen.get_next() + receipt_stream_id = self.get_success(ctx_mgr.__aenter__()) + + # Create the new token based on the stream ID above. + current_receipt_token = MultiWriterStreamToken.from_generator(receipt_id_gen) + receipt_token = current_receipt_token.copy_and_advance( + MultiWriterStreamToken(stream=receipt_stream_id) + ) + token = StreamToken.START.copy_and_advance(StreamKeyType.RECEIPT, receipt_token) + + # Function under test + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + + # This should block waiting for the stream to update + # + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop iterate. + self.reactor.advance(Duration(seconds=2).as_secs()) + # It should still not be done yet + self.assertFalse(wait_d.called) + + # Marking the stream ID as persisted should unblock the request. + self.get_success(ctx_mgr.__aexit__(None, None, None)) + + # Advance time to make another iteration of + # `wait_for_stream_token(...)` sleep loop so it sees that we're + # finally caught up now. + self.reactor.advance(Duration(seconds=1).as_secs()) + + # Done waiting and caught-up (True) + wait_result = self.get_success(wait_d) + self.assertEqual(wait_result, True) + + def test_wait_for_stream_token_with_future_sync_token_timeout( + self, + ) -> None: + """ + Test `wait_for_stream_token` when we receive a token that is ahead of our + current token, we'll wait until the stream position advances *until* we hit the + timeout. + + This can happen if replication streams start lagging, and the client's + previous sync request was serviced by a worker ahead of ours. + """ + # We simulate a lagging stream by getting a stream ID from the ID gen + # and then waiting to mark it as "persisted". + receipt_id_gen = self.store.get_receipts_stream_id_gen() + ctx_mgr = receipt_id_gen.get_next() + receipt_stream_id = self.get_success(ctx_mgr.__aenter__()) + + # Create the new token based on the stream ID above. + current_receipt_token = MultiWriterStreamToken.from_generator(receipt_id_gen) + receipt_token = current_receipt_token.copy_and_advance( + MultiWriterStreamToken(stream=receipt_stream_id) + ) + token = StreamToken.START.copy_and_advance(StreamKeyType.RECEIPT, receipt_token) + + # Function under test + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop record 0 as the `start` time. + self.reactor.advance(Duration(seconds=0).as_secs()) + + # This should block waiting for the stream to update + # + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop iterate. + self.reactor.advance(Duration(seconds=5).as_secs()) + # It should still not be done yet (not enough time to hit the timeout) + self.assertFalse(wait_d.called) + # Advance time past the 10 second timeout (5 + 6 = 11 seconds) to make the + # `wait_for_stream_token(...)` sleep loop give up. + self.reactor.advance(Duration(seconds=6).as_secs()) + + # Make sure we gave up waiting and not caught-up (False) + wait_result = self.get_success(wait_d) + self.assertEqual(wait_result, False) From ff55aff5b292dc3e108b15bd742add730daba89b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2026 10:12:42 +0100 Subject: [PATCH 24/42] Fix up event-construction in tests ahead of the Rust event port (#19781) When we port the `Event` class to Rust, the constructor will check for the existence of required fields. To support that, we tidy up the test code where we construct fake events to add all the required fields. There should be no behavioural changes. Review commit-by-commit. --- changelog.d/19781.misc | 1 + tests/api/test_filtering.py | 86 +++++++------ tests/crypto/test_event_signing.py | 17 +-- tests/events/test_auto_accept_invites.py | 6 +- tests/events/test_utils.py | 104 +++++++++------- tests/federation/test_federation_base.py | 3 + tests/federation/test_federation_client.py | 3 + .../test_federation_out_of_band_membership.py | 30 ++--- tests/federation/test_federation_server.py | 16 +-- tests/federation/transport/test_client.py | 10 ++ tests/handlers/test_federation.py | 26 ++-- tests/handlers/test_federation_event.py | 26 ++-- tests/handlers/test_room_member.py | 10 +- tests/handlers/test_room_policy.py | 19 +-- tests/handlers/test_room_summary.py | 4 +- tests/handlers/test_sync.py | 6 +- tests/push/test_presentable_names.py | 7 +- tests/push/test_push_rule_evaluator.py | 22 +++- .../client/sliding_sync/test_sliding_sync.py | 5 +- tests/rest/client/test_third_party_rules.py | 1 + tests/state/test_v2.py | 25 ++-- tests/state/test_v21.py | 1 + .../databases/main/test_events_worker.py | 6 +- tests/storage/test_events.py | 22 ++-- tests/storage/test_msc4242_state_dag.py | 6 +- tests/storage/test_sliding_sync_tables.py | 5 +- tests/test_event_auth.py | 47 +++---- tests/test_state.py | 5 +- tests/test_utils/event_builders.py | 117 ++++++++++++++++++ 29 files changed, 403 insertions(+), 233 deletions(-) create mode 100644 changelog.d/19781.misc create mode 100644 tests/test_utils/event_builders.py diff --git a/changelog.d/19781.misc b/changelog.d/19781.misc new file mode 100644 index 0000000000..63c2482ded --- /dev/null +++ b/changelog.d/19781.misc @@ -0,0 +1 @@ +Fix up event-construction in tests ahead of the Rust event port. diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 7742a06b4c..8c17e98c0d 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -36,7 +36,7 @@ from synapse.util.clock import Clock from synapse.util.frozenutils import freeze from tests import unittest -from tests.events.test_utils import MockEvent +from tests.test_utils.event_builders import make_test_event user_id = UserID.from_string("@test_user:test") user2_id = UserID.from_string("@test_user2:test") @@ -134,18 +134,22 @@ class FilteringTestCase(unittest.HomeserverTestCase): def test_definition_types_works_with_literals(self) -> None: definition = {"types": ["m.room.message", "org.matrix.foo.bar"]} - event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.message", room_id="!foo:bar" + ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_types_works_with_wildcards(self) -> None: definition = {"types": ["m.*", "org.matrix.foo.bar"]} - event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.message", room_id="!foo:bar" + ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_types_works_with_unknowns(self) -> None: definition = {"types": ["m.room.message", "org.matrix.foo.bar"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="now.for.something.completely.different", room_id="!foo:bar", @@ -154,19 +158,23 @@ class FilteringTestCase(unittest.HomeserverTestCase): def test_definition_not_types_works_with_literals(self) -> None: definition = {"not_types": ["m.room.message", "org.matrix.foo.bar"]} - event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.message", room_id="!foo:bar" + ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_types_works_with_wildcards(self) -> None: definition = {"not_types": ["m.room.message", "org.matrix.*"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_types_works_with_unknowns(self) -> None: definition = {"not_types": ["m.*", "org.*"]} - event = MockEvent(sender="@foo:bar", type="com.nom.nom.nom", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="com.nom.nom.nom", room_id="!foo:bar" + ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_not_types_takes_priority_over_types(self) -> None: @@ -174,33 +182,35 @@ class FilteringTestCase(unittest.HomeserverTestCase): "not_types": ["m.*", "org.*"], "types": ["m.room.message", "m.room.topic"], } - event = MockEvent(sender="@foo:bar", type="m.room.topic", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.topic", room_id="!foo:bar" + ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_senders_works_with_literals(self) -> None: definition = {"senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@flibble:wibble", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_senders_works_with_unknowns(self) -> None: definition = {"senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@challenger:appears", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_senders_works_with_literals(self) -> None: definition = {"not_senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@flibble:wibble", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_senders_works_with_unknowns(self) -> None: definition = {"not_senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@challenger:appears", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertTrue(Filter(self.hs, definition)._check(event)) @@ -210,21 +220,21 @@ class FilteringTestCase(unittest.HomeserverTestCase): "not_senders": ["@misspiggy:muppets"], "senders": ["@kermit:muppets", "@misspiggy:muppets"], } - event = MockEvent( + event = make_test_event( sender="@misspiggy:muppets", type="m.room.topic", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_rooms_works_with_literals(self) -> None: definition = {"rooms": ["!secretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown" ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_rooms_works_with_unknowns(self) -> None: definition = {"rooms": ["!secretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!anothersecretbase:unknown", @@ -233,7 +243,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): def test_definition_not_rooms_works_with_literals(self) -> None: definition = {"not_rooms": ["!anothersecretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!anothersecretbase:unknown", @@ -242,7 +252,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): def test_definition_not_rooms_works_with_unknowns(self) -> None: definition = {"not_rooms": ["!secretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!anothersecretbase:unknown", @@ -254,7 +264,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): "not_rooms": ["!secretbase:unknown"], "rooms": ["!secretbase:unknown"], } - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown" ) self.assertFalse(Filter(self.hs, definition)._check(event)) @@ -268,7 +278,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@kermit:muppets", # yup type="m.room.message", # yup room_id="!stage:unknown", # yup @@ -284,7 +294,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@misspiggy:muppets", # nope type="m.room.message", # yup room_id="!stage:unknown", # yup @@ -300,7 +310,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@kermit:muppets", # yup type="m.room.message", # yup room_id="!piggyshouse:muppets", # nope @@ -316,7 +326,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@kermit:muppets", # yup type="muppets.misspiggy.kisses", # nope room_id="!stage:unknown", # yup @@ -325,7 +335,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): def test_filter_labels(self) -> None: definition = {"org.matrix.labels": ["#fun"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -334,7 +344,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): self.assertTrue(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -344,7 +354,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): self.assertFalse(Filter(self.hs, definition)._check(event)) # check it works with frozen dictionaries too - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -354,7 +364,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): def test_filter_not_labels(self) -> None: definition = {"org.matrix.not_labels": ["#fun"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -363,7 +373,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -375,7 +385,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): @unittest.override_config({"experimental_features": {"msc3874_enabled": True}}) def test_filter_rel_type(self) -> None: definition = {"org.matrix.msc3874.rel_types": ["m.thread"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -384,7 +394,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -393,7 +403,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -405,7 +415,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): @unittest.override_config({"experimental_features": {"msc3874_enabled": True}}) def test_filter_not_rel_type(self) -> None: definition = {"org.matrix.msc3874.not_rel_types": ["m.thread"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -414,7 +424,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -423,7 +433,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): self.assertTrue(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -494,7 +504,9 @@ class FilteringTestCase(unittest.HomeserverTestCase): user_id=user_id, user_filter=user_filter_json ) ) - event = MockEvent(sender="@foo:bar", type="m.room.topic", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.topic", room_id="!foo:bar" + ) events = [event] user_filter = self.get_success( @@ -511,7 +523,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): user_id=user_id, user_filter=user_filter_json ) ) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar" ) events = [event] @@ -542,14 +554,14 @@ class FilteringTestCase(unittest.HomeserverTestCase): def test_filter_relations(self) -> None: events = [ # An event without a relation. - MockEvent( + make_test_event( event_id="$no_relation", sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar", ), # An event with a relation. - MockEvent( + make_test_event( event_id="$with_relation", sender="@foo:bar", type="org.matrix.custom.event", diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 6ec0f64ffc..e4621b1c2b 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -34,10 +34,11 @@ from synapse.crypto.event_signing import ( event_needs_resigning, resign_event, ) -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.types import JsonDict from tests import unittest +from tests.test_utils.event_builders import make_test_event # Perform these tests using given secret key so we get entirely deterministic # signatures output that we can test against. @@ -70,7 +71,7 @@ class EventSigningTestCase(unittest.TestCase): RoomVersions.V1, event_dict, HOSTNAME, self.signing_key ) - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) @@ -103,7 +104,7 @@ class EventSigningTestCase(unittest.TestCase): RoomVersions.V1, event_dict, HOSTNAME, self.signing_key ) - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) @@ -141,7 +142,7 @@ class EventResigningTestCase(unittest.TestCase): add_hashes_and_signatures( RoomVersions.V1, event_dict, HOSTNAME, self.signing_key ) - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) self.assertIn(HOSTNAME, event.signatures) self.assertIn(KEY_NAME, event.signatures[HOSTNAME]) signature = event.signatures[HOSTNAME][KEY_NAME] @@ -171,7 +172,7 @@ class EventResigningTestCase(unittest.TestCase): "signatures": {}, "unsigned": {"age_ts": 1000000}, } - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) resigned_event = resign_event(event, HOSTNAME, signing_key_2) self.assertIn(HOSTNAME, resigned_event["signatures"]) self.assertIn(key_name_2, resigned_event["signatures"][HOSTNAME]) @@ -188,7 +189,7 @@ class EventResigningTestCase(unittest.TestCase): "unsigned": {"age_ts": 1000000}, } internal_metadata: JsonDict = {} - event_that_needs_resigning = make_event_from_dict( + event_that_needs_resigning = make_test_event( event_that_needs_resigning_dict, RoomVersions.V1, internal_metadata, @@ -207,7 +208,7 @@ class EventResigningTestCase(unittest.TestCase): events_that_dont_need_resigning: list[TestCase] = [ { "name": "sender domain isn't ours", - "event": make_event_from_dict( + "event": make_test_event( {**event_that_needs_resigning_dict, "sender": "@u:somewhereelse"}, RoomVersions.V1, internal_metadata, @@ -215,7 +216,7 @@ class EventResigningTestCase(unittest.TestCase): }, { "name": "already signed with this key", - "event": make_event_from_dict( + "event": make_test_event( { **event_that_needs_resigning_dict, "signatures": { diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index e0ebdf0bca..632d1dc4f6 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -33,7 +33,6 @@ from synapse.api.errors import SynapseError from synapse.config._base import RootConfig from synapse.config.auto_accept_invites import AutoAcceptInvitesConfig from synapse.events.auto_accept_invites import InviteAutoAccepter -from synapse.federation.federation_base import event_from_pdu_json from synapse.handlers.sync import JoinedSyncResult, SyncRequestKey from synapse.module_api import ModuleApi from synapse.rest import admin @@ -43,6 +42,7 @@ from synapse.types import StreamToken, UserID, UserInfo, create_requester from synapse.util.clock import Clock from tests.handlers.test_sync import generate_sync_config +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import ( FederatingHomeserverTestCase, HomeserverTestCase, @@ -182,7 +182,7 @@ class AutoAcceptInvitesTestCase(FederatingHomeserverTestCase): ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -308,7 +308,7 @@ class AutoAcceptInvitesTestCase(FederatingHomeserverTestCase): remote_server = "otherserver" remote_user = "@otheruser:" + remote_server - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index a402dd70d1..8f78ae4944 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -26,7 +26,7 @@ from parameterized import parameterized from synapse.api.constants import EventContentFields from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.events.utils import ( FilteredEvent, PowerLevelsContent, @@ -42,31 +42,16 @@ from synapse.events.utils import ( from synapse.types import JsonDict, create_requester from synapse.util.frozenutils import freeze +from tests.test_utils.event_builders import make_test_event from tests.unittest import HomeserverTestCase if TYPE_CHECKING: from synapse.server import HomeServer -def MockEvent(**kwargs: Any) -> EventBase: - if "event_id" not in kwargs: - kwargs["event_id"] = "fake_event_id" - if "type" not in kwargs: - kwargs["type"] = "fake_type" - if "content" not in kwargs: - kwargs["content"] = {} - - # Move internal metadata out so we can call make_event properly - internal_metadata = kwargs.get("internal_metadata") - if internal_metadata is not None: - kwargs.pop("internal_metadata") - - return make_event_from_dict(kwargs, internal_metadata_dict=internal_metadata) - - class TestMaybeUpsertEventField(stdlib_unittest.TestCase): def test_update_okay(self) -> None: - event = make_event_from_dict({"event_id": "$1234"}) + event = make_test_event({"event_id": "$1234"}) success = maybe_upsert_event_field( event, event.unsigned, "replaces_state", "value" ) @@ -74,7 +59,7 @@ class TestMaybeUpsertEventField(stdlib_unittest.TestCase): self.assertEqual(event.unsigned["replaces_state"], "value") def test_update_not_okay(self) -> None: - event = make_event_from_dict({"event_id": "$1234"}) + event = make_test_event({"event_id": "$1234"}) LARGE_STRING = "a" * 100_000 success = maybe_upsert_event_field( event, event.unsigned, "replaces_state", LARGE_STRING @@ -83,7 +68,7 @@ class TestMaybeUpsertEventField(stdlib_unittest.TestCase): self.assertNotIn("replaces_state", event.unsigned) def test_update_not_okay_leaves_original_value(self) -> None: - event = make_event_from_dict( + event = make_test_event( {"event_id": "$1234", "unsigned": {"replaces_state": "value"}} ) LARGE_STRING = "a" * 100_000 @@ -95,6 +80,20 @@ class TestMaybeUpsertEventField(stdlib_unittest.TestCase): class PruneEventTestCase(stdlib_unittest.TestCase): + # Fields that `make_test_event` fills in by default and that `prune_event` + # preserves as spec-required keep fields. Pruning tests only spell out the + # fields they care about; these are merged into the expected dict so each + # test stays focused on what it is actually checking. + _DEFAULT_KEPT_FIELDS: JsonDict = { + "sender": "@test:test", + "room_id": "!test:test", + "depth": 1, + "origin_server_ts": 1, + "hashes": {}, + "auth_events": [], + "prev_events": [], + } + def run_test(self, evdict: JsonDict, matchdict: JsonDict, **kwargs: Any) -> None: """ Asserts that a new event constructed with `evdict` will look like @@ -105,8 +104,9 @@ class PruneEventTestCase(stdlib_unittest.TestCase): matchdict: The expected resulting dictionary. kwargs: Additional keyword arguments used to create the event. """ + expected = {**self._DEFAULT_KEPT_FIELDS, **matchdict} self.assertEqual( - prune_event(make_event_from_dict(evdict, **kwargs)).get_dict(), matchdict + prune_event(make_test_event(evdict, **kwargs)).get_dict(), expected ) def test_minimal(self) -> None: @@ -123,9 +123,6 @@ class PruneEventTestCase(stdlib_unittest.TestCase): def test_basic_keys(self) -> None: """Ensure that the keys that should be untouched are kept.""" - # Note that some of the values below don't really make sense, but the - # pruning of events doesn't worry about the values of any fields (with - # the exception of the content field). self.run_test( { "event_id": "$3:domain", @@ -134,12 +131,12 @@ class PruneEventTestCase(stdlib_unittest.TestCase): "sender": "@2:domain", "state_key": "B", "content": {"other_key": "foo"}, - "hashes": "hashes", + "hashes": {"sha256": "abc"}, "signatures": {"domain": {"algo:1": "sigs"}}, "depth": 4, - "prev_events": "prev_events", + "prev_events": [], "prev_state": "prev_state", - "auth_events": "auth_events", + "auth_events": [], "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", @@ -152,11 +149,11 @@ class PruneEventTestCase(stdlib_unittest.TestCase): "room_id": "!1:domain", "sender": "@2:domain", "state_key": "B", - "hashes": "hashes", + "hashes": {"sha256": "abc"}, "depth": 4, - "prev_events": "prev_events", + "prev_events": [], "prev_state": "prev_state", - "auth_events": "auth_events", + "auth_events": [], "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", @@ -625,7 +622,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase): class CloneEventTestCase(stdlib_unittest.TestCase): def test_unsigned_is_copied(self) -> None: - original = make_event_from_dict( + original = make_test_event( { "type": "A", "event_id": "$test:domain", @@ -679,7 +676,8 @@ class SerializeEventTestCase(HomeserverTestCase): def test_event_fields_works_with_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent(sender="@alice:localhost", room_id="!foo:bar"), ["room_id"] + make_test_event(sender="@alice:localhost", room_id="!foo:bar"), + ["room_id"], ), {"room_id": "!foo:bar"}, ) @@ -687,7 +685,7 @@ class SerializeEventTestCase(HomeserverTestCase): def test_event_fields_works_with_nested_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"body": "A message"}, @@ -700,7 +698,7 @@ class SerializeEventTestCase(HomeserverTestCase): def test_event_fields_works_with_dot_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"key.with.dots": {}}, @@ -713,7 +711,7 @@ class SerializeEventTestCase(HomeserverTestCase): def test_event_fields_works_with_nested_dot_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={ @@ -729,7 +727,7 @@ class SerializeEventTestCase(HomeserverTestCase): def test_event_fields_nops_with_unknown_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"foo": "bar"}, @@ -742,7 +740,7 @@ class SerializeEventTestCase(HomeserverTestCase): def test_event_fields_nops_with_non_dict_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"foo": ["I", "am", "an", "array"]}, @@ -755,7 +753,7 @@ class SerializeEventTestCase(HomeserverTestCase): def test_event_fields_nops_with_array_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"foo": ["I", "am", "an", "array"]}, @@ -768,7 +766,7 @@ class SerializeEventTestCase(HomeserverTestCase): def test_event_fields_all_fields_if_empty(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", @@ -782,6 +780,9 @@ class SerializeEventTestCase(HomeserverTestCase): "room_id": "!foo:bar", "content": {"foo": "bar"}, "unsigned": {}, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) @@ -799,12 +800,12 @@ class SerializeEventTestCase(HomeserverTestCase): # Default behaviour should be *not* to include it self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", content={"foo": "bar"}, - internal_metadata={"soft_failed": True}, + internal_metadata_dict={"soft_failed": True}, ), [], ), @@ -814,18 +815,21 @@ class SerializeEventTestCase(HomeserverTestCase): "room_id": "!foo:bar", "content": {"foo": "bar"}, "unsigned": {}, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) # When asked though, we should set it self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", content={"foo": "bar"}, - internal_metadata={"soft_failed": True}, + internal_metadata_dict={"soft_failed": True}, ), [], True, @@ -836,16 +840,19 @@ class SerializeEventTestCase(HomeserverTestCase): "room_id": "!foo:bar", "content": {"foo": "bar"}, "unsigned": {"io.element.synapse.soft_failed": True}, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", content={"foo": "bar"}, - internal_metadata={ + internal_metadata_dict={ "soft_failed": True, "policy_server_spammy": True, }, @@ -862,6 +869,9 @@ class SerializeEventTestCase(HomeserverTestCase): "io.element.synapse.soft_failed": True, "io.element.synapse.policy_server_spammy": True, }, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) @@ -896,7 +906,7 @@ class SerializeEventTestCase(HomeserverTestCase): redaction_id = "$redaction_event_id" - event = MockEvent( + event = make_test_event( type="foo", event_id="test", room_id="!foo:bar", @@ -904,7 +914,7 @@ class SerializeEventTestCase(HomeserverTestCase): ) event.internal_metadata.redacted_by = redaction_id - redaction_event = MockEvent( + redaction_event = make_test_event( type="m.room.redaction", event_id=redaction_id, content={"redacts": "test"}, diff --git a/tests/federation/test_federation_base.py b/tests/federation/test_federation_base.py index 1bc1da1feb..8b8c65a0c5 100644 --- a/tests/federation/test_federation_base.py +++ b/tests/federation/test_federation_base.py @@ -36,6 +36,9 @@ class FederationBaseTestCase(unittest.HomeserverTestCase): "type": "m.room.message", "room_id": "!r:domain", "sender": f"@u:{self.hs.config.server.server_name}", + "auth_events": [], + "prev_events": [], + "depth": 1, "signatures": {}, "unsigned": {"age_ts": 1000000}, } diff --git a/tests/federation/test_federation_client.py b/tests/federation/test_federation_client.py index 0535aed107..656665cb42 100644 --- a/tests/federation/test_federation_client.py +++ b/tests/federation/test_federation_client.py @@ -73,6 +73,7 @@ class FederationClientTest(FederatingHomeserverTestCase): "content": {"creator": self.creator}, "prev_events": [], "auth_events": [], + "depth": 1, "origin_server_ts": 500, } ) @@ -85,6 +86,7 @@ class FederationClientTest(FederatingHomeserverTestCase): "content": {"membership": "join"}, "prev_events": [], "auth_events": [], + "depth": 2, "origin_server_ts": 600, } ) @@ -97,6 +99,7 @@ class FederationClientTest(FederatingHomeserverTestCase): "content": {}, "prev_events": [], "auth_events": [], + "depth": 3, "origin_server_ts": 700, } ) diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py index a1ab72b7a1..1707081863 100644 --- a/tests/federation/test_federation_out_of_band_membership.py +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -33,11 +33,8 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.api.room_versions import RoomVersion, RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.events.utils import strip_event -from synapse.federation.federation_base import ( - event_from_pdu_json, -) from synapse.federation.transport.client import SendJoinResponse from synapse.http.matrixfederationclient import ( ByteParser, @@ -53,6 +50,7 @@ from synapse.types.handlers.sliding_sync import ( from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event from tests.utils import test_timeout logger = logging.getLogger(__name__) @@ -80,9 +78,7 @@ def required_state_json_to_state_map(required_state: Any) -> StateMap[EventBase] "Each event in `required_state` should have a string `state_key`" ) - state_map[(event_type, event_state_key)] = make_event_from_dict( - state_event_dict - ) + state_map[(event_type, event_state_key)] = make_test_event(state_event_dict) else: # Yell because we're in a test and this is unexpected raise AssertionError("`required_state` should be a list of event dicts") @@ -195,7 +191,7 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): remote_room_id = f"!remote-room:{self.OTHER_SERVER_NAME}" room_version = RoomVersions.V10 - room_create_event = make_event_from_dict( + room_create_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, @@ -217,7 +213,7 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): room_version=room_version, ) - creator_membership_event = make_event_from_dict( + creator_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, @@ -235,7 +231,7 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): ) # From the remote homeserver, invite user1 on the local homserver - user1_invite_membership_event = make_event_from_dict( + user1_invite_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, @@ -297,7 +293,7 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): # Prevent tight-looping to allow the `test_timeout` to work time.sleep(0.1) - user1_join_membership_event_template = make_event_from_dict( + user1_join_membership_event_template = make_test_event( { "room_id": remote_room_id, "sender": local_user1_id, @@ -377,7 +373,7 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): and parser is not None ): # As the remote server, we need to sign the event before sending it back - user1_join_membership_event_signed = make_event_from_dict( + user1_join_membership_event_signed = make_test_event( self.add_hashes_and_signatures_from_other_server(data), room_version=room_version, ) @@ -404,12 +400,12 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): if path.startswith("/_matrix/federation/v1/send/") and data is not None: for pdu in data.get("pdus", []): - event = event_from_pdu_json(pdu, room_version) + event = make_test_pdu_event(pdu, room_version) collected_pdus_from_hs1_federation_send.add(event.event_id) # Just acknowledge everything hs1 is trying to send hs2 return { - event_from_pdu_json(pdu, room_version).event_id: {} + make_test_pdu_event(pdu, room_version).event_id: {} for pdu in data.get("pdus", []) } @@ -521,12 +517,12 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): ) -> JsonDict | T: if path.startswith("/_matrix/federation/v1/send/") and data is not None: for pdu in data.get("pdus", []): - event = event_from_pdu_json(pdu, room_version) + event = make_test_pdu_event(pdu, room_version) collected_pdus_from_hs1_federation_send.add(event.event_id) # Just acknowledge everything hs1 is trying to send hs2 return { - event_from_pdu_json(pdu, room_version).event_id: {} + make_test_pdu_event(pdu, room_version).event_id: {} for pdu in data.get("pdus", []) } @@ -538,7 +534,7 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): self.federation_http_client.put_json.side_effect = put_json # From the remote homeserver, invite user2 on the local homserver - user2_invite_membership_event = make_event_from_dict( + user2_invite_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 7b1a2a5adc..1bf0817bd2 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -31,8 +31,7 @@ from synapse.api.errors import FederationError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import EventBase, make_event_from_dict -from synapse.federation.federation_base import event_from_pdu_json +from synapse.events import EventBase from synapse.http.types import QueryParams from synapse.logging.context import LoggingContext from synapse.rest import admin @@ -43,6 +42,7 @@ from synapse.types import JsonDict from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event from tests.unittest import override_config logger = logging.getLogger(__name__) @@ -95,7 +95,7 @@ class FederationServerTests(unittest.FederatingHomeserverTestCase): def _create_acl_event(content: JsonDict) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": "!a:b", "event_id": "$a:b", @@ -147,7 +147,7 @@ class MessageAcceptTests(unittest.FederatingHomeserverTestCase): # Join a remote user to the room that will attempt to send bad events self.remote_bad_user_id = f"@baduser:{self.OTHER_SERVER_NAME}" - self.remote_bad_user_join_event = make_event_from_dict( + self.remote_bad_user_join_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": self.room_id, @@ -212,7 +212,7 @@ class MessageAcceptTests(unittest.FederatingHomeserverTestCase): ) # Now lie about an event's prev_events - lying_event = make_event_from_dict( + lying_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": self.room_id, @@ -732,7 +732,7 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): "auth_events": [], "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"}, } - filtered_event = event_from_pdu_json(event1, RoomVersions.V1) + filtered_event = make_test_pdu_event(event1, RoomVersions.V1) # Make sure unauthorized fields are stripped from unsigned self.assertNotIn("more warez", filtered_event.unsigned) @@ -754,7 +754,7 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): }, } - filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1, received_time=20) + filtered_event2 = make_test_pdu_event(event2, RoomVersions.V1, received_time=20) self.assertIn("age_ts", filtered_event2.unsigned) self.assertEqual(6, filtered_event2.unsigned["age_ts"]) self.assertNotIn("more warez", filtered_event2.unsigned) @@ -779,7 +779,7 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): "invite_room_state": [], }, } - filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1, received_time=20) + filtered_event3 = make_test_pdu_event(event3, RoomVersions.V1, received_time=20) self.assertIn("age_ts", filtered_event3.unsigned) # Invite_room_state field is only permitted in event type m.room.member self.assertNotIn("invite_room_state", filtered_event3.unsigned) diff --git a/tests/federation/transport/test_client.py b/tests/federation/transport/test_client.py index 9a6bbabd35..881baa02a9 100644 --- a/tests/federation/transport/test_client.py +++ b/tests/federation/transport/test_client.py @@ -36,7 +36,16 @@ class SendJoinParserTestCase(TestCase): def test_two_writes(self) -> None: """Test that the parser can sensibly deserialise an input given in two slices.""" parser = SendJoinParser(RoomVersions.V1, True) + common_event_fields = { + "sender": "@user:example.org", + "depth": 1, + "origin_server_ts": 1, + "hashes": {}, + "auth_events": [], + "prev_events": [], + } parent_event = { + **common_event_fields, "content": { "see_room_version_spec": "The event format changes depending on the room version." }, @@ -45,6 +54,7 @@ class SendJoinParserTestCase(TestCase): "type": "m.room.minimal_pdu", } state = { + **common_event_fields, "content": { "see_room_version_spec": "The event format changes depending on the room version." }, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 20ffed68f4..794c0a3185 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -35,8 +35,7 @@ from synapse.api.errors import ( SynapseError, ) from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, make_event_from_dict -from synapse.federation.federation_base import event_from_pdu_json +from synapse.events import EventBase from synapse.federation.federation_client import SendJoinResult from synapse.rest import admin from synapse.rest.client import login, room @@ -45,6 +44,7 @@ from synapse.storage.databases.main.events_worker import EventCacheEntry from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event logger = logging.getLogger(__name__) @@ -132,7 +132,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): ) # build and send an event which will be rejected - ev = event_from_pdu_json( + ev = make_test_pdu_event( { "type": EventTypes.Message, "content": {}, @@ -183,7 +183,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): ) # build and send an event which will be rejected - ev = event_from_pdu_json( + ev = make_test_pdu_event( { "type": "org.matrix.test", "state_key": "test_key", @@ -227,7 +227,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): room_version = self.get_success(self.store.get_room_version(room_id)) # Build an event to backfill - event = event_from_pdu_json( + event = make_test_pdu_event( { "type": EventTypes.Message, "content": {"body": "hello world", "msgtype": "m.text"}, @@ -324,7 +324,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): def create_invite() -> EventBase: room_id = self.helper.create_room_as(room_creator=user_id, tok=tok) room_version = self.get_success(self.store.get_room_version(room_id)) - return event_from_pdu_json( + return make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -386,7 +386,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): class EventFromPduTestCase(TestCase): def test_valid_json(self) -> None: """Valid JSON should be turned into an event.""" - ev = event_from_pdu_json( + ev = make_test_pdu_event( { "type": EventTypes.Message, "content": {"bool": True, "null": None, "int": 1, "str": "foobar"}, @@ -413,7 +413,7 @@ class EventFromPduTestCase(TestCase): float("nan"), ]: with self.assertRaises(SynapseError): - event_from_pdu_json( + make_test_pdu_event( { "type": EventTypes.Message, "content": {"foo": value}, @@ -430,7 +430,7 @@ class EventFromPduTestCase(TestCase): def test_invalid_nested(self) -> None: """List and dictionaries are recursively searched.""" with self.assertRaises(SynapseError): - event_from_pdu_json( + make_test_pdu_event( { "type": EventTypes.Message, "content": {"foo": [{"bar": 2**56}]}, @@ -457,7 +457,7 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase): room_id = "!room:example.com" - EVENT_CREATE = make_event_from_dict( + EVENT_CREATE = make_test_event( { "room_id": room_id, "type": "m.room.create", @@ -470,7 +470,7 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase): }, room_version=RoomVersions.V10, ) - EVENT_CREATOR_MEMBERSHIP = make_event_from_dict( + EVENT_CREATOR_MEMBERSHIP = make_test_event( { "room_id": room_id, "type": "m.room.member", @@ -484,7 +484,7 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase): }, room_version=RoomVersions.V10, ) - EVENT_INVITATION_MEMBERSHIP = make_event_from_dict( + EVENT_INVITATION_MEMBERSHIP = make_test_event( { "room_id": room_id, "type": "m.room.member", @@ -501,7 +501,7 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase): }, room_version=RoomVersions.V10, ) - membership_event = make_event_from_dict( + membership_event = make_test_event( { "room_id": room_id, "type": "m.room.member", diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 1aaa86e2e8..9da9f52fb6 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -28,7 +28,6 @@ from synapse.event_auth import ( check_state_dependent_auth_rules, check_state_independent_auth_rules, ) -from synapse.events import make_event_from_dict from synapse.events.snapshot import EventContext from synapse.federation.transport.client import StateRequestResponse from synapse.logging.context import LoggingContext @@ -42,6 +41,7 @@ from synapse.util.clock import Clock from tests import unittest from tests.test_utils import event_injection +from tests.test_utils.event_builders import make_test_event class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): @@ -127,7 +127,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # mock up a load of state events which we are missing state_events = [ - make_event_from_dict( + make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_state_type", @@ -154,7 +154,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # mock up a prev event. # Depending on the test, we either persist this upfront (as an outlier), # or let the server request it. - prev_event = make_event_from_dict( + prev_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -191,7 +191,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): self.mock_federation_transport_client.get_event.side_effect = get_event # mock up a regular event to pass into _process_pulled_event - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -301,7 +301,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): ) ) - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -421,7 +421,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): member_event.event_id, ] - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -524,7 +524,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # We purposely don't run `add_hashes_and_signatures_from_other_server` # over this because we want the signature check to fail. - pulled_event_without_signatures = make_event_from_dict( + pulled_event_without_signatures = make_test_event( { "type": "test_regular_type", "room_id": room_id, @@ -540,7 +540,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # Create a regular event that should pass except for the # `pulled_event_without_signatures` in the `prev_event`. - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -720,7 +720,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): ] # Create a regular event that should process - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -878,7 +878,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # accepted, but the local homeserver will reject. next_depth = 100 next_timestamp = other_member_event.origin_server_ts + 100 - rejected_power_levels_event = make_event_from_dict( + rejected_power_levels_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.power_levels", @@ -927,7 +927,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # Then we create a kick event for a local user that cites the rejected power # levels event in its auth events. The kick event will be rejected solely # because of the rejected auth event and would otherwise be accepted. - rejected_kick_event = make_event_from_dict( + rejected_kick_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.member", @@ -1042,7 +1042,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # Create a missing event, so that the local homeserver has to do a `/state` or # `/state_ids` request to pull state from the remote homeserver. - missing_event = make_event_from_dict( + missing_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.message", @@ -1067,7 +1067,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # The pulled event has two prev events, one of which is missing. We will make a # `/state` or `/state_ids` request to the remote homeserver to ask it for the # state before the missing prev event. - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.message", diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 3890abdbc8..b550c2420b 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -9,9 +9,6 @@ from synapse.api.constants import AccountDataTypes, EventTypes, Membership from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import FrozenEventV3 -from synapse.federation.federation_base import ( - event_from_pdu_json, -) from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer from synapse.types import UserID, create_requester @@ -19,6 +16,7 @@ from synapse.util.clock import Clock from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import make_request +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import ( FederatingHomeserverTestCase, HomeserverTestCase, @@ -549,7 +547,7 @@ class TestMSC4155InviteFiltering(FederatingHomeserverTestCase): ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -595,7 +593,7 @@ class TestMSC4155InviteFiltering(FederatingHomeserverTestCase): ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -710,7 +708,7 @@ class TestMSC4380InviteBlocking(FederatingHomeserverTestCase): ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index 9adf0e38c2..2d3e917aef 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -22,7 +22,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes from synapse.api.errors import HttpResponseException, SynapseError from synapse.crypto.event_signing import compute_event_signature -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.handlers.room_policy import POLICY_SERVER_KEY_ID from synapse.rest import admin from synapse.rest.client import filter, login, room, sync @@ -32,6 +32,7 @@ from synapse.util.clock import Clock from tests import unittest from tests.test_utils import event_injection +from tests.test_utils.event_builders import make_test_event class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): @@ -75,7 +76,7 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): self.signing_key = signedjson.key.generate_signing_key("policy_server") # Create some sample events - self.spammy_event = make_event_from_dict( + self.spammy_event = make_test_event( room_version=room_version, internal_metadata_dict={}, event_dict={ @@ -88,7 +89,7 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): }, }, ) - self.not_spammy_event = make_event_from_dict( + self.not_spammy_event = make_test_event( room_version=room_version, internal_metadata_dict={}, event_dict={ @@ -272,7 +273,7 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): def test_signed_event_is_not_spam(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -302,7 +303,7 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): def test_ask_policy_server_to_sign_event_ok(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -324,7 +325,7 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): def test_ask_policy_server_to_sign_event_refuses(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -353,7 +354,7 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): def test_ask_policy_server_to_sign_event_cannot_reach(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -379,7 +380,7 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event_with_wrong_key - unverified_event = make_event_from_dict( + unverified_event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -398,7 +399,7 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): ) self.assertEqual(len(unverified_event.signatures), 1) - verified_event = make_event_from_dict( + verified_event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index ee65cb1afb..49076e69d8 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -35,7 +35,6 @@ from synapse.api.constants import ( ) from synapse.api.errors import AuthError, NotFoundError, SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events import make_event_from_dict from synapse.federation.transport.client import TransportLayerClient from synapse.handlers.room_summary import _child_events_comparison_key, _RoomEntry from synapse.rest import admin @@ -45,6 +44,7 @@ from synapse.types import JsonDict, UserID, create_requester from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event from tests.unittest import override_config @@ -217,7 +217,7 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): # Poke an invite over federation into the database. fed_handler = self.hs.get_federation_handler() fed_hostname = UserID.from_string(from_user).domain - event = make_event_from_dict( + event = make_test_event( { "room_id": room_id, "event_id": "!abcd:" + fed_hostname, diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index e4a4d59aeb..d2b2523321 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -32,7 +32,6 @@ from synapse.api.filtering import FilterCollection, Filtering from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.federation.federation_base import event_from_pdu_json from synapse.handlers.sync import ( SyncConfig, SyncRequestKey, @@ -55,6 +54,7 @@ from synapse.util.duration import Duration import tests.unittest import tests.utils +from tests.test_utils.event_builders import make_test_pdu_event _request_key = 0 @@ -913,7 +913,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): prev_events = self.get_success(self.store.get_prev_events_for_room(room_id)) # create a call invite event - call_event = event_from_pdu_json( + call_event = make_test_pdu_event( { "type": EventTypes.CallInvite, "content": {}, @@ -961,7 +961,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): priv_prev_events = self.get_success( self.store.get_prev_events_for_room(private_room_id) ) - private_call_event = event_from_pdu_json( + private_call_event = make_test_pdu_event( { "type": EventTypes.CallInvite, "content": {}, diff --git a/tests/push/test_presentable_names.py b/tests/push/test_presentable_names.py index 2558f2c0b2..2d382c8fab 100644 --- a/tests/push/test_presentable_names.py +++ b/tests/push/test_presentable_names.py @@ -23,11 +23,12 @@ from typing import Iterable, cast from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, FrozenEvent +from synapse.events import EventBase from synapse.push.presentable_names import calculate_room_name from synapse.types import StateKey, StateMap from tests import unittest +from tests.test_utils.event_builders import make_test_event class MockDataStore: @@ -44,7 +45,7 @@ class MockDataStore: self._events = {} for i, (event_id, content) in enumerate(events): - self._events[event_id] = FrozenEvent( + self._events[event_id] = make_test_event( { "event_id": "$event_id", "type": event_id[0], @@ -59,7 +60,7 @@ class MockDataStore: async def get_event( self, event_id: str, allow_none: bool = False - ) -> FrozenEvent | None: + ) -> EventBase | None: assert allow_none, "Mock not configured for allow_none = False" # Decode the state key from the event ID. diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py index a786d74bf1..2e389710b9 100644 --- a/tests/push/test_push_rule_evaluator.py +++ b/tests/push/test_push_rule_evaluator.py @@ -27,7 +27,6 @@ import synapse.rest.admin from synapse.api.constants import EventTypes, HistoryVisibility, Membership from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService -from synapse.events import FrozenEvent, make_event_from_dict from synapse.push.bulk_push_rule_evaluator import _flatten_dict from synapse.push.httppusher import tweaks_for_actions from synapse.rest import admin @@ -40,6 +39,7 @@ from synapse.util.clock import Clock from synapse.util.frozenutils import freeze from tests import unittest +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event, inject_member_event @@ -81,7 +81,7 @@ class FlattenDictTestCase(unittest.TestCase): def test_event(self) -> None: """Events can also be flattened.""" - event = make_event_from_dict( + event = make_test_event( { "room_id": "!test:test", "type": "m.room.message", @@ -103,6 +103,10 @@ class FlattenDictTestCase(unittest.TestCase): "room_id": "!test:test", "sender": "@alice:test", "type": "m.room.message", + "depth": 1, + "origin_server_ts": 1, + "auth_events": [], + "prev_events": [], } self.assertEqual(expected, _flatten_dict(event)) @@ -121,24 +125,32 @@ class FlattenDictTestCase(unittest.TestCase): } # For a current room version, there's no special behavior. - event = make_event_from_dict(event_dict, room_version=RoomVersions.V8) + event = make_test_event(event_dict, room_version=RoomVersions.V8) expected = { "room_id": "!test:test", "sender": "@alice:test", "type": "m.room.message", "content.org\\.matrix\\.msc1767\\.markup": [], + "depth": 1, + "origin_server_ts": 1, + "auth_events": [], + "prev_events": [], } self.assertEqual(expected, _flatten_dict(event)) # For a room version with extensible events, they parse out the text/plain # to a content.body property. - event = make_event_from_dict(event_dict, room_version=RoomVersions.MSC1767v10) + event = make_test_event(event_dict, room_version=RoomVersions.MSC1767v10) expected = { "content.body": "hello world!", "room_id": "!test:test", "sender": "@alice:test", "type": "m.room.message", "content.org\\.matrix\\.msc1767\\.markup": [], + "depth": 1, + "origin_server_ts": 1, + "auth_events": [], + "prev_events": [], } self.assertEqual(expected, _flatten_dict(event)) @@ -152,7 +164,7 @@ class PushRuleEvaluatorTestCase(unittest.TestCase): msc4210: bool = False, msc4306: bool = False, ) -> PushRuleEvaluator: - event = FrozenEvent( + event = make_test_event( { "event_id": "$event_id", "type": "m.room.history_visibility", diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index ac8dfd37d8..ebf41cd87c 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -30,7 +30,7 @@ from synapse.api.constants import ( RoomTypes, ) from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, StrippedStateEvent, make_event_from_dict +from synapse.events import EventBase, StrippedStateEvent from synapse.events.snapshot import EventContext from synapse.handlers.sliding_sync import StateValues from synapse.rest.client import account_data, devices, login, receipts, room, sync @@ -47,6 +47,7 @@ from synapse.util.stringutils import random_string from tests import unittest from tests.server import FakeChannel, TimedOutException +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event logger = logging.getLogger(__name__) @@ -308,7 +309,7 @@ class SlidingSyncBase(unittest.HomeserverTestCase): "invite_room_state": serialized_stripped_state_events } - invite_event = make_event_from_dict( + invite_event = make_test_event( invite_event_dict, room_version=RoomVersions.V10, ) diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py index 1709b27f67..b4fa71ece8 100644 --- a/tests/rest/client/test_third_party_rules.py +++ b/tests/rest/client/test_third_party_rules.py @@ -532,6 +532,7 @@ class ThirdPartyRulesTestCase(unittest.FederatingHomeserverTestCase): "origin_server_ts": self.clock.time_msec(), "prev_events": [], "auth_events": [], + "hashes": {}, "signatures": {}, "unsigned": {}, } diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 85ce5bede2..8b3b919f44 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -33,7 +33,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersions from synapse.event_auth import auth_types_for_event -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.state.v2 import ( _get_auth_chain_difference, _get_power_level_for_sender, @@ -45,6 +45,7 @@ from synapse.types import EventID, StateMap from synapse.util.duration import Duration from tests import unittest +from tests.test_utils.event_builders import make_test_event ALICE = "@alice:example.com" BOB = "@bob:example.com" @@ -119,7 +120,7 @@ class FakeEvent: if self.state_key is not None: event_dict["state_key"] = self.state_key - return make_event_from_dict(event_dict) + return make_test_event(event_dict) # All graphs start with this set of events @@ -878,7 +879,7 @@ class AuthChainDifferenceTestCase(unittest.TestCase): on room version""" store = TestStateResolutionStore({}) for room_version in [RoomVersions.V10, RoomVersions.V11]: - create_event = make_event_from_dict( + create_event = make_test_event( { "room_id": ROOM_ID, "sender": ALICE, @@ -894,9 +895,9 @@ class AuthChainDifferenceTestCase(unittest.TestCase): else {} ), }, - room_version, + room_version=room_version, ) - member_event = make_event_from_dict( + member_event = make_test_event( { "room_id": ROOM_ID, "sender": ALICE, @@ -908,9 +909,9 @@ class AuthChainDifferenceTestCase(unittest.TestCase): "auth_events": [create_event.event_id], "prev_events": [create_event.event_id], }, - room_version, + room_version=room_version, ) - pl_event = make_event_from_dict( + pl_event = make_test_event( { "room_id": ROOM_ID, "sender": ALICE, @@ -926,7 +927,7 @@ class AuthChainDifferenceTestCase(unittest.TestCase): "auth_events": [create_event.event_id, member_event.event_id], "prev_events": [member_event.event_id], }, - room_version, + room_version=room_version, ) event_map = { @@ -940,7 +941,7 @@ class AuthChainDifferenceTestCase(unittest.TestCase): CHARLIE: 10, } for user_id, want_pl in want_pls.items(): - test_event = make_event_from_dict( + test_event = make_test_event( { "room_id": ROOM_ID, "sender": user_id, @@ -954,7 +955,7 @@ class AuthChainDifferenceTestCase(unittest.TestCase): ], "prev_events": [pl_event.event_id], }, - room_version, + room_version=room_version, ) event_map[test_event.event_id] = test_event got_pl = self.successResultOf( @@ -977,7 +978,7 @@ class AuthChainDifferenceTestCase(unittest.TestCase): CHARLIE: 0, } for user_id, want_pl in want_pls.items(): - test_event = make_event_from_dict( + test_event = make_test_event( { "room_id": ROOM_ID, "sender": user_id, @@ -991,7 +992,7 @@ class AuthChainDifferenceTestCase(unittest.TestCase): ], "prev_events": [pl_event.event_id], }, - room_version, + room_version=room_version, ) got_pl = self.successResultOf( defer.ensureDeferred( diff --git a/tests/state/test_v21.py b/tests/state/test_v21.py index 9f607176c9..6b4d2837ab 100644 --- a/tests/state/test_v21.py +++ b/tests/state/test_v21.py @@ -495,6 +495,7 @@ class StateResV21TestCase(unittest.HomeserverTestCase): "prev_events": prev_events, "auth_events": auth_events, "origin_server_ts": monotonic_timestamp(), + "hashes": {}, } if event_type != EventTypes.Create: if room_id is None: diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index c786271c09..dd647d29d6 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -28,7 +28,6 @@ from twisted.internet.defer import CancelledError, Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor from synapse.api.room_versions import EventFormatVersions, RoomVersions -from synapse.events import make_event_from_dict from synapse.logging.context import LoggingContext from synapse.rest import admin from synapse.rest.client import login, room @@ -42,6 +41,7 @@ from synapse.util.async_helpers import yieldable_gather_results from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event, inject_event @@ -377,7 +377,7 @@ class DatabaseOutageTestCase(unittest.HomeserverTestCase): "type": f"test {idx}", "room_id": self.room_id, } - event = make_event_from_dict(event_json, room_version=RoomVersions.V4) + event = make_test_event(event_json, room_version=RoomVersions.V4) event_id = event.event_id self.get_success( self.store.db_pool.simple_upsert( @@ -400,7 +400,7 @@ class DatabaseOutageTestCase(unittest.HomeserverTestCase): {"event_id": event_id}, { "room_id": self.room_id, - "json": json.dumps(event_json), + "json": json.dumps(event.get_dict()), "internal_metadata": "{}", "format_version": EventFormatVersions.ROOM_V4_PLUS, }, diff --git a/tests/storage/test_events.py b/tests/storage/test_events.py index 7d1c96f97f..c511ac9252 100644 --- a/tests/storage/test_events.py +++ b/tests/storage/test_events.py @@ -26,13 +26,13 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions from synapse.events import EventBase -from synapse.federation.federation_base import event_from_pdu_json from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.types import StateMap from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import HomeserverTestCase logger = logging.getLogger(__name__) @@ -142,7 +142,7 @@ class ExtremPruneTestCase(HomeserverTestCase): # Fudge a remote event and persist it. This will be the extremity before # the gap. - self.remote_event_1 = event_from_pdu_json( + self.remote_event_1 = make_test_pdu_event( { "type": EventTypes.Message, "state_key": "@user:other", @@ -198,7 +198,7 @@ class ExtremPruneTestCase(HomeserverTestCase): # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other", @@ -237,7 +237,7 @@ class ExtremPruneTestCase(HomeserverTestCase): ) # Fudge a second event which points to an event we don't have. - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Message, "state_key": "@user:other", @@ -286,7 +286,7 @@ class ExtremPruneTestCase(HomeserverTestCase): # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -322,7 +322,7 @@ class ExtremPruneTestCase(HomeserverTestCase): # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -367,7 +367,7 @@ class ExtremPruneTestCase(HomeserverTestCase): # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -415,7 +415,7 @@ class ExtremPruneTestCase(HomeserverTestCase): # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -455,7 +455,7 @@ class ExtremPruneTestCase(HomeserverTestCase): # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -514,7 +514,7 @@ class InvalideUsersInRoomCacheTestCase(HomeserverTestCase): # Fudge a join event for a remote user. remote_user = "@user:other" - remote_event_1 = event_from_pdu_json( + remote_event_1 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": remote_user, @@ -561,7 +561,7 @@ class InvalideUsersInRoomCacheTestCase(HomeserverTestCase): # Fudge a join event for a remote user. remote_user = "@user:other" - remote_event_1 = event_from_pdu_json( + remote_event_1 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": remote_user, diff --git a/tests/storage/test_msc4242_state_dag.py b/tests/storage/test_msc4242_state_dag.py index 6720c1d659..2150bc0996 100644 --- a/tests/storage/test_msc4242_state_dag.py +++ b/tests/storage/test_msc4242_state_dag.py @@ -19,15 +19,13 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events import ( - make_event_from_dict, -) from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.events.snapshot import EventContext from synapse.rest.client import room from synapse.server import HomeServer from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_event from tests.unittest import HomeserverTestCase, override_config @@ -156,7 +154,7 @@ class MSC4242EventPersistenceStateDagsStoreTestCase(HomeserverTestCase): prev_state_events: list[str], rejected: bool = False, ) -> tuple[MSC4242Event, EventContext]: - ev = make_event_from_dict( + ev = make_test_event( { "prev_state_events": prev_state_events, "content": { diff --git a/tests/storage/test_sliding_sync_tables.py b/tests/storage/test_sliding_sync_tables.py index 39226ff9be..67b8d1138f 100644 --- a/tests/storage/test_sliding_sync_tables.py +++ b/tests/storage/test_sliding_sync_tables.py @@ -27,7 +27,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventContentFields, EventTypes, Membership, RoomTypes from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, StrippedStateEvent, make_event_from_dict +from synapse.events import EventBase, StrippedStateEvent from synapse.events.snapshot import EventContext from synapse.rest import admin from synapse.rest.client import login, room, sync @@ -46,6 +46,7 @@ from synapse.types.storage import _BackgroundUpdates from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event logger = logging.getLogger(__name__) @@ -240,7 +241,7 @@ class SlidingSyncTablesTestCaseBase(SlidingSyncBase): "prev_events": [], } - kick_event = make_event_from_dict( + kick_event = make_test_event( kick_event_dict, room_version=RoomVersions.V10, ) diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 9258f0d4dc..4537186ee6 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -29,11 +29,12 @@ from synapse import event_auth from synapse.api.constants import EventContentFields, RejectedReason from synapse.api.errors import AuthError, SynapseError from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.events import EventBase, event_exists_in_state_dag, make_event_from_dict +from synapse.events import EventBase, event_exists_in_state_dag from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import JsonDict, get_domain_from_id from tests.test_utils import get_awaitable_result +from tests.test_utils.event_builders import make_test_event class _StubEventSourceStore: @@ -128,7 +129,7 @@ class EventAuthTestCase(unittest.TestCase): # we make both a good event and a bad event, to check that we are rejecting # the bad event for the reason we think we are. - good_event = make_event_from_dict( + good_event = make_test_event( { "room_id": TEST_ROOM_ID, "type": "m.room.create", @@ -143,7 +144,7 @@ class EventAuthTestCase(unittest.TestCase): }, room_version=RoomVersions.V9, ) - bad_event = make_event_from_dict( + bad_event = make_test_event( {**good_event.get_dict(), "prev_events": ["$fakeevent"]}, room_version=RoomVersions.V9, ) @@ -387,7 +388,7 @@ class EventAuthTestCase(unittest.TestCase): creator = "@creator:example.com" room_version = RoomVersions.MSC4242v12 - create_event = make_event_from_dict( + create_event = make_test_event( { "type": "m.room.create", "sender": creator, @@ -398,7 +399,7 @@ class EventAuthTestCase(unittest.TestCase): }, room_version, ) - create_event_2 = make_event_from_dict( + create_event_2 = make_test_event( { "type": "m.room.create", "sender": creator, @@ -411,7 +412,7 @@ class EventAuthTestCase(unittest.TestCase): ) room_id = create_event.room_id another_room_id = create_event_2.room_id - join_event = make_event_from_dict( + join_event = make_test_event( { "room_id": room_id, "type": "m.room.member", @@ -424,7 +425,7 @@ class EventAuthTestCase(unittest.TestCase): room_version, {"calculated_auth_event_ids": [create_event.event_id]}, ) - event_in_another_room = make_event_from_dict( + event_in_another_room = make_test_event( { "room_id": another_room_id, "type": "m.room.join_rules", @@ -437,7 +438,7 @@ class EventAuthTestCase(unittest.TestCase): room_version, {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, ) - msg_event = make_event_from_dict( + msg_event = make_test_event( { "room_id": room_id, "type": "m.room.message", @@ -449,7 +450,7 @@ class EventAuthTestCase(unittest.TestCase): room_version, {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, ) - rejected_event = make_event_from_dict( + rejected_event = make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -470,7 +471,7 @@ class EventAuthTestCase(unittest.TestCase): RejectingTestCase( name="create event has prev_state_events", events_in_store=[], - test_event=make_event_from_dict( + test_event=make_test_event( { "type": "m.room.create", "sender": creator, @@ -486,7 +487,7 @@ class EventAuthTestCase(unittest.TestCase): RejectingTestCase( name="prev_state_event belongs in a different room", events_in_store=[create_event, join_event, event_in_another_room], - test_event=make_event_from_dict( + test_event=make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -508,7 +509,7 @@ class EventAuthTestCase(unittest.TestCase): RejectingTestCase( name="prev_state_event is a message event", events_in_store=[create_event, join_event, msg_event], - test_event=make_event_from_dict( + test_event=make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -530,7 +531,7 @@ class EventAuthTestCase(unittest.TestCase): RejectingTestCase( name="prev_state_event was rejected", events_in_store=[create_event, join_event, rejected_event], - test_event=make_event_from_dict( + test_event=make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -892,7 +893,7 @@ class EventAuthTestCase(unittest.TestCase): def test_room_v10_rejects_string_power_levels(self) -> None: pl_event_content = {"users_default": "42"} - pl_event = make_event_from_dict( + pl_event = make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(RoomVersions.V10), @@ -906,7 +907,7 @@ class EventAuthTestCase(unittest.TestCase): ) pl_event2_content = {"events": {"m.room.name": "42", "m.room.power_levels": 42}} - pl_event2 = make_event_from_dict( + pl_event2 = make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(RoomVersions.V10), @@ -936,7 +937,7 @@ class EventAuthTestCase(unittest.TestCase): """ def create_event(pl_event_content: dict[str, Any]) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(RoomVersions.V10), @@ -1049,7 +1050,7 @@ class EventAuthTestCase(unittest.TestCase): "signatures": {"test.com": {"ed25519:0": "some9signature"}}, } base.update(ev) - event = make_event_from_dict(base, RoomVersions.V10) + event = make_test_event(base, RoomVersions.V10) got = event_exists_in_state_dag(event) self.assertEqual( got, should_exist, f"{ev} should_exist={should_exist} but got {got}" @@ -1068,7 +1069,7 @@ def _create_event( room_version: RoomVersion, user_id: str, ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1089,7 +1090,7 @@ def _member_event( sender: str | None = None, additional_content: dict | None = None, ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1122,7 +1123,7 @@ def _power_levels_event( sender: str, content: JsonDict, ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1145,7 +1146,7 @@ def _alias_event(room_version: RoomVersion, sender: str, **kwargs: Any) -> Event "content": {"aliases": []}, } data.update(**kwargs) - return make_event_from_dict(data, room_version=room_version) + return make_test_event(data, room_version=room_version) def _build_auth_dict_for_room_version( @@ -1164,7 +1165,7 @@ def _random_state_event( ) -> EventBase: if auth_events is None: auth_events = [] - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1181,7 +1182,7 @@ def _random_state_event( def _join_rules_event( room_version: RoomVersion, sender: str, join_rule: str ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), diff --git a/tests/test_state.py b/tests/test_state.py index 0ca88aef74..0a6720d6ef 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -32,7 +32,7 @@ from twisted.internet import defer from synapse.api.auth.internal import InternalAuth from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.state import StateHandler, StateResolutionHandler, _make_state_cache_entry from synapse.types import JsonDict, MutableStateMap, StateMap @@ -41,6 +41,7 @@ from synapse.util.macaroons import MacaroonGenerator from tests import unittest from tests.server import get_clock +from tests.test_utils.event_builders import make_test_event from tests.utils import default_config _next_event_id = 1000 @@ -82,7 +83,7 @@ def create_event( d.update(kwargs) - return make_event_from_dict(d) + return make_test_event(d) class _DummyStore: diff --git a/tests/test_utils/event_builders.py b/tests/test_utils/event_builders.py new file mode 100644 index 0000000000..a8eb586c1f --- /dev/null +++ b/tests/test_utils/event_builders.py @@ -0,0 +1,117 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import TypedDict + +from typing_extensions import NotRequired, Unpack + +from synapse.api.room_versions import ( + RoomVersion, + RoomVersions, +) +from synapse.events import EventBase, make_event_from_dict +from synapse.federation.federation_base import event_from_pdu_json +from synapse.types import JsonDict + + +def default_event_fields(room_version: RoomVersion) -> JsonDict: + """Return default values for every field required by `room_version`.""" + + # We need to include entries for every required field for the room version. + # Note that they don't necessarily have to be valid values, just enough to + # allow us to construct the event class. (Ideally we'd build a fully valid + # event, but this is fine for now.) + defaults: JsonDict = { + "type": "m.test", + "sender": "@test:test", + "content": {}, + "depth": 1, + "origin_server_ts": 1, + "hashes": {}, + "prev_events": [], + "room_id": "!test:test", + } + + # MSC4242 versions require prev_state_events but not auth_events. + if room_version.msc4242_state_dags: + defaults["prev_state_events"] = [] + else: + defaults["auth_events"] = [] + + if room_version == RoomVersions.V1: + # V1 requires an event_id field, but later versions don't. + defaults["event_id"] = "$test_event_id:matrix.org" + + return defaults + + +def make_test_event( + event_dict: JsonDict | None = None, + room_version: RoomVersion = RoomVersions.V1, + internal_metadata_dict: JsonDict | None = None, + rejected_reason: str | None = None, + **fields: Unpack["_EventFields"], +) -> EventBase: + """Build an `EventBase` with defaults for the strict-required fields. + + Pass an `event_dict` and/or `**fields` keyword arguments — both are + merged on top of the format-version defaults from + `default_event_fields`. Explicit values win over defaults, and + `**fields` wins over `event_dict` so call sites can override a + shared base dict with one-off tweaks. + """ + merged: JsonDict = { + **default_event_fields(room_version), + **(event_dict or {}), + **fields, + } + return make_event_from_dict( + merged, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, + ) + + +def make_test_pdu_event( + pdu: JsonDict, + room_version: RoomVersion, + received_time: int | None = None, +) -> EventBase: + """Wrapper around `event_from_pdu_json` for test PDU dicts. + + Federation-side test fixtures often omit fields the strict Rust ctor + requires (e.g. `hashes`, `auth_events`, `prev_events`, `depth`) + because those tests focus on transport/auth flow rather than event + well-formedness. This helper layers in the same format-version + defaults as `make_test_event` before delegating. + """ + pdu = {**default_event_fields(room_version), **pdu} + return event_from_pdu_json(pdu, room_version, received_time=received_time) + + +class _EventFields(TypedDict): + """Type for `kwargs` in `make_test_event`.""" + + event_id: NotRequired[str] + type: NotRequired[str] + sender: NotRequired[str] + content: NotRequired[JsonDict] + depth: NotRequired[int] + origin_server_ts: NotRequired[int] + hashes: NotRequired[dict[str, str]] + auth_events: NotRequired[list[str]] + prev_events: NotRequired[list[str]] + prev_state_events: NotRequired[list[str]] + room_id: NotRequired[str] From cf64199ea0ebdd7181d8145f5c5a9e904a073a73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 10:24:49 +0000 Subject: [PATCH 25/42] Bump urllib3 from 2.6.3 to 2.7.0 (#19771) Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index d66b175b0a..5c8057e675 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3586,14 +3586,14 @@ files = [ [[package]] name = "urllib3" -version = "2.6.3" +version = "2.7.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, - {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, + {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, + {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, ] [package.extras] From 5c8419eed7aba38aa0799792adaf915ad62c7fe6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 10:26:51 +0000 Subject: [PATCH 26/42] Bump authlib from 1.6.11 to 1.6.12 (#19776) Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5c8057e675..e56155bf70 100644 --- a/poetry.lock +++ b/poetry.lock @@ -26,15 +26,15 @@ files = [ [[package]] name = "authlib" -version = "1.6.11" +version = "1.6.12" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" groups = ["main"] markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ - {file = "authlib-1.6.11-py2.py3-none-any.whl", hash = "sha256:c8687a9a26451c51a34a06fa17bb97cb15bba46a6a626755e2d7f50da8bff3e3"}, - {file = "authlib-1.6.11.tar.gz", hash = "sha256:64db35b9b01aeccb4715a6c9a6613a06f2bd7be2ab9d2eb89edd1dfc7580a38f"}, + {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, + {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, ] [package.dependencies] From 9ce68a6a4a2913cf399d058ef119f65b88e93fa9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 10:29:39 +0000 Subject: [PATCH 27/42] Bump gitpython from 3.1.47 to 3.1.50 (#19767) Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index e56155bf70..78f18caadf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -582,14 +582,14 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.47" +version = "3.1.50" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.47-py3-none-any.whl", hash = "sha256:489f590edfd6d20571b2c0e72c6a6ac6915ee8b8cd04572330e3842207a78905"}, - {file = "gitpython-3.1.47.tar.gz", hash = "sha256:dba27f922bd2b42cb54c87a8ab3cb6beb6bf07f3d564e21ac848913a05a8a3cd"}, + {file = "gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9"}, + {file = "gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc"}, ] [package.dependencies] From 3f0f03d53653bc97a35e76baf8c98da533516d75 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 15 May 2026 11:36:47 +0100 Subject: [PATCH 28/42] Revert "Send a SSS response immediately if the config has changed and there are new results to sync (#19714)" (#19784) Reverts: #19714 Opens: #19783 Closes: https://github.com/element-hq/backend-internal/issues/242 Related: #18880 (the performance problem that is aggravated by #19714) This reverts commit 2691d0b8b143e01987504210e320198560371435. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19784.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 52 ++++---- synapse/handlers/sliding_sync/room_lists.py | 12 +- .../sliding_sync/test_room_subscriptions.py | 119 ------------------ 4 files changed, 29 insertions(+), 155 deletions(-) create mode 100644 changelog.d/19784.bugfix diff --git a/changelog.d/19784.bugfix b/changelog.d/19784.bugfix new file mode 100644 index 0000000000..c68524d57a --- /dev/null +++ b/changelog.d/19784.bugfix @@ -0,0 +1 @@ +Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' due to performance problems. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 2e26bfc43a..ddce04d5b3 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -184,38 +184,34 @@ class SlidingSyncHandler: timeout_ms -= after_wait_ts - before_wait_ts timeout_ms = max(timeout_ms, 0) - # Compute a response immediately. We always need to do this before - # waiting for new data (unlike in /v3/sync), as the request config might - # have changed (e.g. new room subscriptions, etc). - now_token = self.event_sources.get_current_token() - result = await self.current_sync_for_user( - sync_config, - from_token=from_token, - to_token=now_token, - ) - - # Return immediately if we have a result, the timeout is 0, or this is - # an initial sync. - if result or timeout_ms == 0 or from_token is None: - return result, did_wait - - # Otherwise, we wait for something to happen and report it to the user. - async def current_sync_callback( - before_token: StreamToken, after_token: StreamToken - ) -> SlidingSyncResult: - return await self.current_sync_for_user( + # We're going to respond immediately if the timeout is 0 or if this is an + # initial sync (without a `from_token`) so we can avoid calling + # `notifier.wait_for_events()`. + if timeout_ms == 0 or from_token is None: + now_token = self.event_sources.get_current_token() + result = await self.current_sync_for_user( sync_config, from_token=from_token, - to_token=after_token, + to_token=now_token, ) + else: + # Otherwise, we wait for something to happen and report it to the user. + async def current_sync_callback( + before_token: StreamToken, after_token: StreamToken + ) -> SlidingSyncResult: + return await self.current_sync_for_user( + sync_config, + from_token=from_token, + to_token=after_token, + ) - result = await self.notifier.wait_for_events( - sync_config.user.to_string(), - timeout_ms, - current_sync_callback, - from_token=now_token, - ) - did_wait = True + result = await self.notifier.wait_for_events( + sync_config.user.to_string(), + timeout_ms, + current_sync_callback, + from_token=from_token.stream_token, + ) + did_wait = True return result, did_wait diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 216ef3b071..8969d91583 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -852,15 +852,11 @@ class SlidingSyncRoomLists: previous_connection_state.room_configs.get(room_id) ) if prev_room_sync_config is not None: - # Always include rooms whose effective config has - # expanded. This covers timeline-limit increases and - # required-state additions introduced by room - # subscriptions overriding list-derived params. + # Always include rooms whose timeline limit has increased. + # (see the "XXX: Odd behavior" described below) if ( - prev_room_sync_config.combine_room_sync_config( - room_config - ) - != prev_room_sync_config + prev_room_sync_config.timeline_limit + < room_config.timeline_limit ): rooms_should_send.add(room_id) continue diff --git a/tests/rest/client/sliding_sync/test_room_subscriptions.py b/tests/rest/client/sliding_sync/test_room_subscriptions.py index d970af367d..811478f1ba 100644 --- a/tests/rest/client/sliding_sync/test_room_subscriptions.py +++ b/tests/rest/client/sliding_sync/test_room_subscriptions.py @@ -22,7 +22,6 @@ import synapse.rest.admin from synapse.api.constants import EventTypes, HistoryVisibility from synapse.rest.client import login, room, sync from synapse.server import HomeServer -from synapse.types import JsonDict from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -127,124 +126,6 @@ class SlidingSyncRoomSubscriptionsTestCase(SlidingSyncBase): response_body["rooms"][room_id1], ) - def test_room_subscription_required_state_expansion_returns_immediately( - self, - ) -> None: - """ - Test that adding a room subscription with stronger params than the list causes an - incremental long-poll to return immediately, even without new stream activity. - """ - user1_id = self.register_user("user1", "pass") - user1_tok = self.login(user1_id, "pass") - - room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) - - sync_body: JsonDict = { - "lists": { - "foo-list": { - "ranges": [[0, 0]], - "required_state": [], - "timeline_limit": 0, - } - }, - "conn_id": "conn_id", - } - _, from_token = self.do_sync(sync_body, tok=user1_tok) - - sync_body["room_subscriptions"] = { - room_id1: { - "required_state": [ - [EventTypes.Create, ""], - ], - "timeline_limit": 0, - } - } - - channel = self.make_request( - "POST", - self.sync_endpoint + f"?timeout=10000&pos={from_token}", - content=sync_body, - access_token=user1_tok, - await_result=False, - ) - channel.await_result(timeout_ms=3000) - self.assertEqual(channel.code, 200, channel.json_body) - - state_map = self.get_success( - self.storage_controllers.state.get_current_state(room_id1) - ) - - room_response = channel.json_body["rooms"][room_id1] - self.assertNotIn("initial", room_response) - self._assertRequiredStateIncludes( - room_response["required_state"], - { - state_map[(EventTypes.Create, "")], - }, - exact=True, - ) - - def test_room_subscription_required_state_change_returns_immediately(self) -> None: - """ - Test that expanding an existing room subscription's required state causes an - incremental long-poll to return immediately, even without new stream activity. - """ - user1_id = self.register_user("user1", "pass") - user1_tok = self.login(user1_id, "pass") - - room_id1 = self.helper.create_room_as( - user1_id, tok=user1_tok, extra_content={"name": "Foo"} - ) - - sync_body: JsonDict = { - "room_subscriptions": { - room_id1: { - "required_state": [ - [EventTypes.Create, ""], - ], - "timeline_limit": 0, - } - }, - "conn_id": "conn_id", - } - response_body, from_token = self.do_sync(sync_body, tok=user1_tok) - - state_map = self.get_success( - self.storage_controllers.state.get_current_state(room_id1) - ) - self._assertRequiredStateIncludes( - response_body["rooms"][room_id1]["required_state"], - { - state_map[(EventTypes.Create, "")], - }, - exact=True, - ) - - sync_body["room_subscriptions"][room_id1]["required_state"] = [ - [EventTypes.Create, ""], - [EventTypes.Name, ""], - ] - - channel = self.make_request( - "POST", - self.sync_endpoint + f"?timeout=10000&pos={from_token}", - content=sync_body, - access_token=user1_tok, - await_result=False, - ) - channel.await_result(timeout_ms=3000) - self.assertEqual(channel.code, 200, channel.json_body) - - room_response = channel.json_body["rooms"][room_id1] - self.assertNotIn("initial", room_response) - self._assertRequiredStateIncludes( - room_response["required_state"], - { - state_map[(EventTypes.Name, "")], - }, - exact=True, - ) - def test_room_subscriptions_with_leave_membership(self) -> None: """ Test `room_subscriptions` with a leave room should give us timeline and state From 71e1da976cfbfbdf4be2598a84d6a22671c1bf46 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 15 May 2026 11:36:47 +0100 Subject: [PATCH 29/42] Revert "Send a SSS response immediately if the config has changed and there are new results to sync (#19714)" (#19784) Reverts: #19714 Opens: #19783 Closes: https://github.com/element-hq/backend-internal/issues/242 Related: #18880 (the performance problem that is aggravated by #19714) This reverts commit 2691d0b8b143e01987504210e320198560371435. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19784.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 52 ++++---- synapse/handlers/sliding_sync/room_lists.py | 12 +- .../sliding_sync/test_room_subscriptions.py | 119 ------------------ 4 files changed, 29 insertions(+), 155 deletions(-) create mode 100644 changelog.d/19784.bugfix diff --git a/changelog.d/19784.bugfix b/changelog.d/19784.bugfix new file mode 100644 index 0000000000..c68524d57a --- /dev/null +++ b/changelog.d/19784.bugfix @@ -0,0 +1 @@ +Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' due to performance problems. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index a3443b300c..1cc587d4a7 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -167,38 +167,34 @@ class SlidingSyncHandler: timeout_ms -= after_wait_ts - before_wait_ts timeout_ms = max(timeout_ms, 0) - # Compute a response immediately. We always need to do this before - # waiting for new data (unlike in /v3/sync), as the request config might - # have changed (e.g. new room subscriptions, etc). - now_token = self.event_sources.get_current_token() - result = await self.current_sync_for_user( - sync_config, - from_token=from_token, - to_token=now_token, - ) - - # Return immediately if we have a result, the timeout is 0, or this is - # an initial sync. - if result or timeout_ms == 0 or from_token is None: - return result, did_wait - - # Otherwise, we wait for something to happen and report it to the user. - async def current_sync_callback( - before_token: StreamToken, after_token: StreamToken - ) -> SlidingSyncResult: - return await self.current_sync_for_user( + # We're going to respond immediately if the timeout is 0 or if this is an + # initial sync (without a `from_token`) so we can avoid calling + # `notifier.wait_for_events()`. + if timeout_ms == 0 or from_token is None: + now_token = self.event_sources.get_current_token() + result = await self.current_sync_for_user( sync_config, from_token=from_token, - to_token=after_token, + to_token=now_token, ) + else: + # Otherwise, we wait for something to happen and report it to the user. + async def current_sync_callback( + before_token: StreamToken, after_token: StreamToken + ) -> SlidingSyncResult: + return await self.current_sync_for_user( + sync_config, + from_token=from_token, + to_token=after_token, + ) - result = await self.notifier.wait_for_events( - sync_config.user.to_string(), - timeout_ms, - current_sync_callback, - from_token=now_token, - ) - did_wait = True + result = await self.notifier.wait_for_events( + sync_config.user.to_string(), + timeout_ms, + current_sync_callback, + from_token=from_token.stream_token, + ) + did_wait = True return result, did_wait diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 216ef3b071..8969d91583 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -852,15 +852,11 @@ class SlidingSyncRoomLists: previous_connection_state.room_configs.get(room_id) ) if prev_room_sync_config is not None: - # Always include rooms whose effective config has - # expanded. This covers timeline-limit increases and - # required-state additions introduced by room - # subscriptions overriding list-derived params. + # Always include rooms whose timeline limit has increased. + # (see the "XXX: Odd behavior" described below) if ( - prev_room_sync_config.combine_room_sync_config( - room_config - ) - != prev_room_sync_config + prev_room_sync_config.timeline_limit + < room_config.timeline_limit ): rooms_should_send.add(room_id) continue diff --git a/tests/rest/client/sliding_sync/test_room_subscriptions.py b/tests/rest/client/sliding_sync/test_room_subscriptions.py index d970af367d..811478f1ba 100644 --- a/tests/rest/client/sliding_sync/test_room_subscriptions.py +++ b/tests/rest/client/sliding_sync/test_room_subscriptions.py @@ -22,7 +22,6 @@ import synapse.rest.admin from synapse.api.constants import EventTypes, HistoryVisibility from synapse.rest.client import login, room, sync from synapse.server import HomeServer -from synapse.types import JsonDict from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -127,124 +126,6 @@ class SlidingSyncRoomSubscriptionsTestCase(SlidingSyncBase): response_body["rooms"][room_id1], ) - def test_room_subscription_required_state_expansion_returns_immediately( - self, - ) -> None: - """ - Test that adding a room subscription with stronger params than the list causes an - incremental long-poll to return immediately, even without new stream activity. - """ - user1_id = self.register_user("user1", "pass") - user1_tok = self.login(user1_id, "pass") - - room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) - - sync_body: JsonDict = { - "lists": { - "foo-list": { - "ranges": [[0, 0]], - "required_state": [], - "timeline_limit": 0, - } - }, - "conn_id": "conn_id", - } - _, from_token = self.do_sync(sync_body, tok=user1_tok) - - sync_body["room_subscriptions"] = { - room_id1: { - "required_state": [ - [EventTypes.Create, ""], - ], - "timeline_limit": 0, - } - } - - channel = self.make_request( - "POST", - self.sync_endpoint + f"?timeout=10000&pos={from_token}", - content=sync_body, - access_token=user1_tok, - await_result=False, - ) - channel.await_result(timeout_ms=3000) - self.assertEqual(channel.code, 200, channel.json_body) - - state_map = self.get_success( - self.storage_controllers.state.get_current_state(room_id1) - ) - - room_response = channel.json_body["rooms"][room_id1] - self.assertNotIn("initial", room_response) - self._assertRequiredStateIncludes( - room_response["required_state"], - { - state_map[(EventTypes.Create, "")], - }, - exact=True, - ) - - def test_room_subscription_required_state_change_returns_immediately(self) -> None: - """ - Test that expanding an existing room subscription's required state causes an - incremental long-poll to return immediately, even without new stream activity. - """ - user1_id = self.register_user("user1", "pass") - user1_tok = self.login(user1_id, "pass") - - room_id1 = self.helper.create_room_as( - user1_id, tok=user1_tok, extra_content={"name": "Foo"} - ) - - sync_body: JsonDict = { - "room_subscriptions": { - room_id1: { - "required_state": [ - [EventTypes.Create, ""], - ], - "timeline_limit": 0, - } - }, - "conn_id": "conn_id", - } - response_body, from_token = self.do_sync(sync_body, tok=user1_tok) - - state_map = self.get_success( - self.storage_controllers.state.get_current_state(room_id1) - ) - self._assertRequiredStateIncludes( - response_body["rooms"][room_id1]["required_state"], - { - state_map[(EventTypes.Create, "")], - }, - exact=True, - ) - - sync_body["room_subscriptions"][room_id1]["required_state"] = [ - [EventTypes.Create, ""], - [EventTypes.Name, ""], - ] - - channel = self.make_request( - "POST", - self.sync_endpoint + f"?timeout=10000&pos={from_token}", - content=sync_body, - access_token=user1_tok, - await_result=False, - ) - channel.await_result(timeout_ms=3000) - self.assertEqual(channel.code, 200, channel.json_body) - - room_response = channel.json_body["rooms"][room_id1] - self.assertNotIn("initial", room_response) - self._assertRequiredStateIncludes( - room_response["required_state"], - { - state_map[(EventTypes.Name, "")], - }, - exact=True, - ) - def test_room_subscriptions_with_leave_membership(self) -> None: """ Test `room_subscriptions` with a leave room should give us timeline and state From 0ff50720d899e6f867b5694c0f05be1561d650f6 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 15 May 2026 11:43:12 +0100 Subject: [PATCH 30/42] 1.153.0rc3 --- CHANGES.md | 7 +++++++ changelog.d/19784.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/19784.bugfix diff --git a/CHANGES.md b/CHANGES.md index f722a1e703..96a9fadbfb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.153.0rc3 (2026-05-15) + +## Bugfixes + +- Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' (introduced in 1.153.0rc1) due to performance problems. ([\#19784](https://github.com/element-hq/synapse/issues/19784)) + + # Synapse 1.153.0rc2 (2026-05-13) ## Bugfixes diff --git a/changelog.d/19784.bugfix b/changelog.d/19784.bugfix deleted file mode 100644 index c68524d57a..0000000000 --- a/changelog.d/19784.bugfix +++ /dev/null @@ -1 +0,0 @@ -Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' due to performance problems. diff --git a/debian/changelog b/debian/changelog index bbd164b2a8..81f080bead 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.153.0~rc3) stable; urgency=medium + + * New Synapse release 1.153.0rc3. + + -- Synapse Packaging team Fri, 15 May 2026 11:42:29 +0100 + matrix-synapse-py3 (1.153.0~rc2) stable; urgency=medium * New Synapse release 1.153.0rc2. diff --git a/pyproject.toml b/pyproject.toml index a7422e1987..fed1a23f4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.153.0rc2" +version = "1.153.0rc3" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 0c6e0f79e5fef944ceca5f74404ec4a44be310bd Mon Sep 17 00:00:00 2001 From: mhlas7 Date: Fri, 15 May 2026 08:29:53 -0700 Subject: [PATCH 31/42] doc: Enhance update_profile_information documentation with picture claim (#19508) Added details how synapse syncs the picture claim when update_profile_information setting is true. Addresses #17836 --------- Co-authored-by: Michael Hlas <3398654+mhlas7@users.noreply.github.com> --- changelog.d/19508.doc | 1 + docs/usage/configuration/config_documentation.md | 5 ++++- schema/synapse-config.schema.yaml | 14 +++++++++----- 3 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 changelog.d/19508.doc diff --git a/changelog.d/19508.doc b/changelog.d/19508.doc new file mode 100644 index 0000000000..2550116341 --- /dev/null +++ b/changelog.d/19508.doc @@ -0,0 +1 @@ +Added details about how Synapse syncs the picture claim when `update_profile_information` setting is true. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e58a2dcf10..d028d65fe3 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3788,7 +3788,10 @@ This setting has the following sub-options: Defaults to `null`. -* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Currently only syncing the displayname is supported. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Defaults to `false`. +* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Fields that will be synced: + * displayname + * picture - only if Synapse media repository is running in the main + process (i.e. not workerized) and media is stored locally Defaults to `false`. Example configuration: ```yaml diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 8888d2c673..8b8d57b9bf 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -4620,11 +4620,15 @@ properties: type: boolean description: >- Use this setting to keep a user's profile fields in sync with - information from the identity provider. Currently only syncing the - displayname is supported. Fields are checked on every SSO login, and - are updated if necessary. Note that enabling this option will override - user profile information, regardless of whether users have opted-out - of syncing that information when first signing in. + information from the identity provider. Fields are checked on every + SSO login, and are updated if necessary. Note that enabling this + option will override user profile information, regardless of whether + users have opted-out of syncing that information when first signing + in. + Fields that will be synced: + * displayname + * picture - only if Synapse media repository is running in the main + process (i.e. not workerized) and media is stored locally default: false examples: - client_whitelist: From 19f636244ce3a4fa6970f5abe970d105d963e0e6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 15 May 2026 11:49:11 -0500 Subject: [PATCH 32/42] Prefer close backfill points (absolute distance) (#19748) This isn't fixing any particular issue. It's just a follow-up I thought about after merging https://github.com/element-hq/synapse/pull/19611 since we're now also dealing with backfill points in the nearby range ahead of the `current_depth`. And it's possible that the previous sort could bias to all nearby backfill points ahead of the `current_depth` that don't extend into the visible window of events we're paginating through. --- changelog.d/19748.misc | 1 + synapse/handlers/federation.py | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 changelog.d/19748.misc diff --git a/changelog.d/19748.misc b/changelog.d/19748.misc new file mode 100644 index 0000000000..eedd4e92a2 --- /dev/null +++ b/changelog.d/19748.misc @@ -0,0 +1 @@ +Prefer close backfill points (absolute distance). diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 166a02d7c7..ba83d4fd26 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -275,11 +275,22 @@ class FederationHandler: ) ] - # we now have a list of potential places to backpaginate from. We prefer to - # start with the most recent (ie, max depth), so let's sort the list. + # we now have a list of potential places to backpaginate from. Figure out which + # ones we should prefer, so let's sort the list. sorted_backfill_points: list[_BackfillPoint] = sorted( backwards_extremities, - key=lambda e: -int(e.depth), + key=lambda e: ( + # Prefer backfill points that are closer to the `current_depth` + # (absolute distance) + abs(current_depth - e.depth), + # For the tie-break, we care about events that are actually in the past + # as they're more likely to reveal history that we can return (something + # absolutely in the past is better than something can potentially extend + # into the past). + # + # This sorts ascending so 0 sorts before 1 + 0 if current_depth >= e.depth else 1, + ), ) logger.debug( @@ -300,7 +311,7 @@ class FederationHandler: str(len(sorted_backfill_points)), ) - # If we have no backfill points lower than the `current_depth` then either we + # If we have no backfill points lower than the `nearby_depth` then either we # can a) bail or b) still attempt to backfill. We opt to try backfilling anyway # just in case we do get relevant events. This is good for eventual consistency # sake but we don't need to block the client for something that is just as From 8eb220a5e25c57da979d5b0bc8dbc7b5b657e501 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 15 May 2026 13:51:03 -0500 Subject: [PATCH 33/42] Replace `wait_for_quarantined_media_stream_id(...)` with standard `wait_for_stream_token(...)` (#19764) In order to be able to use `wait_for_stream_token(...)`, we have to add the `quarantined_media` stream to the `StreamToken`. Even though we don't care about `/sync`'ing `quarantined_media`, this aligns with the future where all endpoints should probably use `StreamToken`, see https://github.com/element-hq/synapse/issues/19647 Follow-up to https://github.com/element-hq/synapse/pull/19558 and https://github.com/element-hq/synapse/pull/19644 --- changelog.d/19764.misc | 1 + synapse/rest/admin/media.py | 23 ++++++-- synapse/storage/databases/main/room.py | 80 ++++---------------------- synapse/streams/events.py | 3 + synapse/types/__init__.py | 27 +++++++-- tests/rest/admin/test_room.py | 4 +- tests/rest/client/test_rooms.py | 4 +- 7 files changed, 59 insertions(+), 83 deletions(-) create mode 100644 changelog.d/19764.misc diff --git a/changelog.d/19764.misc b/changelog.d/19764.misc new file mode 100644 index 0000000000..8704e3eed6 --- /dev/null +++ b/changelog.d/19764.misc @@ -0,0 +1 @@ +Replace unique `quarantined_media` waiting patterns with standard `wait_for_stream_token(...)`. diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index 1633cca884..35454c1522 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -43,7 +43,13 @@ from synapse.rest.admin._base import ( from synapse.storage.databases.main.media_repository import ( MediaSortOrder, ) -from synapse.types import JsonDict, UserID +from synapse.types import ( + JsonDict, + MultiWriterStreamToken, + StreamKeyType, + StreamToken, + UserID, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -243,6 +249,7 @@ class ListQuarantineChanges(RestServlet): self.auth = hs.get_auth() self.server_name = hs.hostname self.replication = hs.get_replication_data_handler() + self.notifier = hs.get_notifier() async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) @@ -256,8 +263,7 @@ class ListQuarantineChanges(RestServlet): # The caller is trying to get future data, which we don't allow because # we know it's an invalid state that should never happen. We could # wait until we reach the token but we might as well not waste our - # resources on that which is why `wait_for_quarantined_media_stream_id(...)` - # has assertions around this. + # resources on that. raise SynapseError( HTTPStatus.BAD_REQUEST, "The `from` token is considered invalid because it includes stream positions " @@ -268,9 +274,16 @@ class ListQuarantineChanges(RestServlet): errcode=Codes.INVALID_PARAM, ) + # Create a `StreamToken` that's compatible with `wait_for_stream_token`. + # + # FIXME: Ideally, this endpoint would use a `StreamToken` to begin with + from_token = StreamToken.START.copy_and_replace( + StreamKeyType.QUARANTINED_MEDIA, MultiWriterStreamToken(stream=from_id) + ) + # We need to wait to ensure that our current worker is actually caught up with # the stream position, otherwise we might not return what we think we're returning. - if not await self.store.wait_for_quarantined_media_stream_id(from_id): + if not await self.notifier.wait_for_stream_token(from_token): raise SynapseError( HTTPStatus.INTERNAL_SERVER_ERROR, "Timed out while waiting for the worker serving this request to catch up to the given " @@ -280,7 +293,7 @@ class ListQuarantineChanges(RestServlet): errcode=Codes.UNKNOWN, ) - to_id = await self.store.get_current_quarantined_media_stream_id() + to_id = self.store.get_current_quarantined_media_stream_id() changes = await self.store.get_quarantined_media_changes( from_id=from_id, to_id=to_id, diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index a0c42082f0..95aa2cb7dc 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -62,12 +62,12 @@ from synapse.storage.types import Cursor from synapse.storage.util.id_generators import IdGenerator, MultiWriterIdGenerator from synapse.types import ( JsonDict, + MultiWriterStreamToken, RetentionPolicy, StrCollection, ThirdPartyInstanceID, ) from synapse.util.caches.descriptors import cached, cachedList -from synapse.util.duration import Duration from synapse.util.json import json_encoder from synapse.util.stringutils import MXC_REGEX @@ -1302,7 +1302,15 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): return local_media_ids - async def get_current_quarantined_media_stream_id(self) -> int: + def get_quarantined_media_stream_token(self) -> MultiWriterStreamToken: + return MultiWriterStreamToken.from_generator( + self._quarantined_media_changes_id_gen + ) + + def get_quarantined_media_stream_id_generator(self) -> MultiWriterIdGenerator: + return self._quarantined_media_changes_id_gen + + def get_current_quarantined_media_stream_id(self) -> int: """Gets the position of the quarantined media changes stream. Returns: @@ -1318,74 +1326,6 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): """ return await self._quarantined_media_changes_id_gen.get_max_allocated_token() - async def wait_for_quarantined_media_stream_id(self, target_id: int) -> bool: - """Waits until the quarantined media changes stream reaches the given stream ID. - - See https://github.com/element-hq/synapse/pull/19644 for more details. - - TODO: Replace function and call sites with https://github.com/element-hq/synapse/pull/19644 - - Args: - target_id: The stream ID to wait for. - - Returns: - True when caught up to the target stream ID. - False when timing out while waiting. - """ - # We ideally would use something like `wait_for_stream_position` in the meantime, - # but that short circuits if the instance name matches the current instance name. - # Doing so means that if *another* writer is actually leading the to_id, then we'll - # assume that we're caught up when we aren't. - # - # NOTE: Because this is implemented to wait for stream positions by integer ID, - # we're technically waiting for *all* workers to catch up rather than just waiting - # for *our* worker to catch up. This is okay for now because the quarantined media - # stream should be pretty fast to update, and if it's not then the only thing we're - # affecting is an admin API that probably has a tool automatically retrying requests - # anyway. https://github.com/element-hq/synapse/pull/19644 does the waiting properly - # so this should be replaced by that (or similar). - - # Get the minimum shared position/ID across all workers - current_id = self._quarantined_media_changes_id_gen.get_current_token() - if current_id >= target_id: - return True # nothing to wait for: we're already caught up. - - # "This should never happen". Tokens we hand out via the API should exist. If they - # don't, then we're in a bad state and need to explode. - max_persisted_position = ( - await self._quarantined_media_changes_id_gen.get_max_allocated_token() - ) - assert max_persisted_position >= target_id, ( - f"Unable to wait for invalid future token (token={target_id} has positions " - f"ahead of our max persisted position={max_persisted_position})" - ) - - # Start waiting until we've caught up to the `stream_token` - start = self.clock.time_msec() - logged = False - while True: - # Like above, get the minimum shared ID across all workers - current_id = self._quarantined_media_changes_id_gen.get_current_token() - if current_id >= target_id: - return True - - now = self.clock.time_msec() - - # Timed out - if now - start > 10_000: - return False - - if not logged: - logger.info( - "Waiting for current token to reach %s; currently at %s", - target_id, - current_id, - ) - logged = True - - # TODO: be better - await self.clock.sleep(Duration(milliseconds=500)) - async def get_quarantined_media_changes( self, *, from_id: int, to_id: int, limit: int ) -> list[QuarantinedMediaUpdate]: diff --git a/synapse/streams/events.py b/synapse/streams/events.py index f5677a2082..36490fcb35 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -85,6 +85,7 @@ class EventSources: ) thread_subscriptions_key = self.store.get_max_thread_subscriptions_stream_id() sticky_events_key = self.store.get_max_sticky_events_stream_id() + quarantined_media_key = self.store.get_quarantined_media_stream_token() token = StreamToken( room_key=self.sources.room.get_current_key(), @@ -100,6 +101,7 @@ class EventSources: un_partial_stated_rooms_key=un_partial_stated_rooms_key, thread_subscriptions_key=thread_subscriptions_key, sticky_events_key=sticky_events_key, + quarantined_media_key=quarantined_media_key, ) return token @@ -128,6 +130,7 @@ class EventSources: StreamKeyType.UN_PARTIAL_STATED_ROOMS: self.store.get_un_partial_stated_rooms_id_generator(), StreamKeyType.THREAD_SUBSCRIPTIONS: self.store.get_thread_subscriptions_stream_id_generator(), StreamKeyType.STICKY_EVENTS: self.store.get_sticky_events_stream_id_generator(), + StreamKeyType.QUARANTINED_MEDIA: self.store.get_quarantined_media_stream_id_generator(), } for _, key in StreamKeyType.__members__.items(): diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 8b005ef84d..8537a63bde 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -1060,6 +1060,7 @@ class StreamKeyType(Enum): UN_PARTIAL_STATED_ROOMS = "un_partial_stated_rooms_key" THREAD_SUBSCRIPTIONS = "thread_subscriptions_key" STICKY_EVENTS = "sticky_events_key" + QUARANTINED_MEDIA = "quarantined_media_key" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -1067,7 +1068,7 @@ class StreamToken: """A collection of keys joined together by underscores in the following order and which represent the position in their respective streams. - ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242` + ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242_4141_4343` 1. `room_key`: `s2633508` which is a `RoomStreamToken` - `RoomStreamToken`'s can also look like `t426-2633508` or `m56~2.58~3.59` - See the docstring for `RoomStreamToken` for more details. @@ -1082,12 +1083,13 @@ class StreamToken: 10. `un_partial_stated_rooms_key`: `379` 11. `thread_subscriptions_key`: 4242 12. `sticky_events_key`: 4141 + 13. `quarantined_media_key`: 4343 You can see how many of these keys correspond to the various fields in a "/sync" response: ```json { - "next_batch": "s12_4_0_1_1_1_1_4_1_1", + "next_batch": "s12_4_0_1_1_1_1_4_1_1_1_1_1", "presence": { "events": [] }, @@ -1099,7 +1101,7 @@ class StreamToken: "!QrZlfIDQLNLdZHqTnt:hs1": { "timeline": { "events": [], - "prev_batch": "s10_4_0_1_1_1_1_4_1_1", + "prev_batch": "s10_4_0_1_1_1_1_4_1_1_1_1_1", "limited": false }, "state": { @@ -1142,6 +1144,9 @@ class StreamToken: un_partial_stated_rooms_key: int thread_subscriptions_key: int sticky_events_key: int + quarantined_media_key: MultiWriterStreamToken = attr.ib( + validator=attr.validators.instance_of(MultiWriterStreamToken) + ) _SEPARATOR = "_" START: ClassVar["StreamToken"] @@ -1171,6 +1176,7 @@ class StreamToken: un_partial_stated_rooms_key, thread_subscriptions_key, sticky_events_key, + quarantined_media_key, ) = keys return cls( @@ -1188,6 +1194,9 @@ class StreamToken: un_partial_stated_rooms_key=int(un_partial_stated_rooms_key), thread_subscriptions_key=int(thread_subscriptions_key), sticky_events_key=int(sticky_events_key), + quarantined_media_key=await MultiWriterStreamToken.parse( + store, quarantined_media_key + ), ) except CancelledError: raise @@ -1212,6 +1221,7 @@ class StreamToken: str(self.un_partial_stated_rooms_key), str(self.thread_subscriptions_key), str(self.sticky_events_key), + await self.quarantined_media_key.to_string(store), ] ) @@ -1241,6 +1251,12 @@ class StreamToken: self.device_list_key.copy_and_advance(new_value), ) return new_token + elif key == StreamKeyType.QUARANTINED_MEDIA: + new_token = self.copy_and_replace( + StreamKeyType.QUARANTINED_MEDIA, + self.quarantined_media_key.copy_and_advance(new_value), + ) + return new_token new_token = self.copy_and_replace(key, new_value) new_id = new_token.get_field(key) @@ -1263,6 +1279,7 @@ class StreamToken: key: Literal[ StreamKeyType.RECEIPT, StreamKeyType.DEVICE_LIST, + StreamKeyType.QUARANTINED_MEDIA, ], ) -> MultiWriterStreamToken: ... @@ -1334,7 +1351,8 @@ class StreamToken: f"account_data: {self.account_data_key}, push_rules: {self.push_rules_key}, " f"to_device: {self.to_device_key}, device_list: {self.device_list_key}, " f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}," - f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key})" + f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key}" + f"quarantined_media: {self.quarantined_media_key})" ) @@ -1351,6 +1369,7 @@ StreamToken.START = StreamToken( un_partial_stated_rooms_key=0, thread_subscriptions_key=0, sticky_events_key=0, + quarantined_media_key=MultiWriterStreamToken(stream=0), ) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 507cf10c5d..c4e4170c6f 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2549,7 +2549,7 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase): def test_topo_token_is_accepted(self) -> None: """Test Topo Token is accepted.""" - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), @@ -2563,7 +2563,7 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase): def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: """Test that stream token is accepted for forward pagination.""" - token = "s0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 28872fa06c..10325c536a 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2248,7 +2248,7 @@ class RoomMessageListTestCase(RoomBase): self.room_id = self.helper.create_room_as(self.user_id) def test_topo_token_is_accepted(self) -> None: - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2259,7 +2259,7 @@ class RoomMessageListTestCase(RoomBase): self.assertTrue("end" in channel.json_body) def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: - token = "s0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) From d8b4ffdf2d987a04ab1f98d8e67388a36f79a017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan?= <166950915+gaetan-sbt@users.noreply.github.com> Date: Mon, 18 May 2026 10:27:10 +0100 Subject: [PATCH 34/42] Fix validation of frozen message event with mentions. (#19634) Fixes: #19689 # What This PR fixes a bug I found when I run synapse (from dockerhub) and register a `check_event_allowed` callback and my client makes use of the mentions field in messages (`cinny:latest`). The bug doesn't appear when the `check_event_allowed` callback is not loaded. After some digging I noticed that the current validation of the mentions doesn't work when an event has been frozen with `event.freeze()`. For the messages this seems to happen when a the `check_event_allowed` is registered (but not otherwise), see [where the event is frozen for check_event_allowed callback](https://github.com/element-hq/synapse/blob/b0fc0b7a612a42e6f15b87dee2a1db4c383645fb/synapse/module_api/callbacks/third_party_event_rules_callbacks.py#L289) and [where the validation function is called](https://github.com/element-hq/synapse/blob/b0fc0b7a612a42e6f15b87dee2a1db4c383645fb/synapse/handlers/message.py#L1404). To have a minimal reproduction example, the following scripts fails on `develop` but succeeds in this branch: ``` python from synapse.api.room_versions import RoomVersions from synapse.events import EventBase, make_event_from_dict from synapse.events.validator import EventValidator from tests.utils import default_config def make_message_event(content: dict) -> EventBase: return make_event_from_dict( { "room_id": "!room:test", "type": "m.room.message", "sender": "@alice:test", "content": content, "auth_events": [], "prev_events": [], "hashes": {"sha256": "aGVsbG8="}, "signatures": {}, "depth": 1, "origin_server_ts": 1000, }, room_version=RoomVersions.V9, ) event = make_message_event( { "msgtype": "m.text", "body": "@moderator:example.com hello", "m.mentions": {"user_ids": ["@moderator:jailbreak-challenge.aqtiveguard.com"]}, } ) EventValidator().validate_new(event, default_config) # Ok event.freeze() EventValidator().validate_new(event, default_config) # throws # pydantic_core._pydantic_core.ValidationError: 1 validation error for Mentions # Input should be a valid dictionary or instance of Mentions [type=model_type, input_value=immutabledict({'user_ids'...nge.aqtiveguard.com',)}), input_type=immutabledict] # For further information visit https://errors.pydantic.dev/2.12/v/model_type ``` # How I made the validation logic also validate the transformation performed by the freezing process, namely: - `immutabledict` validates as `dict`. (was already implemented for POWER_LEVELS) - `tuple` validates as array (added this to the validator in this PR). --------- Co-authored-by: Eric Eastwood Co-authored-by: Olivier 'reivilibre --- changelog.d/19634.bugfix | 1 + synapse/events/validator.py | 81 ++++++++++++++++++++++------------ tests/events/test_validator.py | 50 +++++++++++++++++++++ 3 files changed, 103 insertions(+), 29 deletions(-) create mode 100644 changelog.d/19634.bugfix create mode 100644 tests/events/test_validator.py diff --git a/changelog.d/19634.bugfix b/changelog.d/19634.bugfix new file mode 100644 index 0000000000..e8fcb43570 --- /dev/null +++ b/changelog.d/19634.bugfix @@ -0,0 +1 @@ +Fix bug where Synapse would return 400 (`M_BAD_JSON`) when sending a message with `mentions` field and Synapse module `check_event_allowed` callback registered (frozen event). Contributed by @gaetan-sbt. \ No newline at end of file diff --git a/synapse/events/validator.py b/synapse/events/validator.py index d1b5152d77..e8d6cc9710 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -22,7 +22,6 @@ import collections.abc from typing import cast import jsonschema -from pydantic import Field, StrictBool, StrictStr from synapse.api.constants import ( MAX_ALIAS_LENGTH, @@ -40,10 +39,8 @@ from synapse.events.utils import ( CANONICALJSON_MIN_INT, validate_canonicaljson, ) -from synapse.http.servlet import validate_json_object from synapse.storage.controllers.state import server_acl_evaluator_from_event from synapse.types import EventID, JsonDict, JsonMapping, RoomID, StrCollection, UserID -from synapse.types.rest import RequestBodyModel class EventValidator: @@ -116,29 +113,18 @@ class EventValidator: cls=POWER_LEVELS_VALIDATOR, ) except jsonschema.ValidationError as e: - if e.path: - # example: "users_default": '0' is not of type 'integer' - # cast safety: path entries can be integers, if we fail to validate - # items in an array. However, the POWER_LEVELS_SCHEMA doesn't expect - # to see any arrays. - message = ( - '"' + cast(str, e.path[-1]) + '": ' + e.message # noqa: B306 - ) - # jsonschema.ValidationError.message is a valid attribute - else: - # example: '0' is not of type 'integer' - message = e.message # noqa: B306 - # jsonschema.ValidationError.message is a valid attribute - - raise SynapseError( - code=400, - msg=message, - errcode=Codes.BAD_JSON, - ) + raise _validation_error_to_api_error(e) # If the event contains a mentions key, validate it. if EventContentFields.MENTIONS in event.content: - validate_json_object(event.content[EventContentFields.MENTIONS], Mentions) + try: + jsonschema.validate( + instance=event.content[EventContentFields.MENTIONS], + schema=MENTIONS_SCHEMA, + cls=MENTIONS_VALIDATOR, + ) + except jsonschema.ValidationError as e: + raise _validation_error_to_api_error(e) def _validate_retention(self, event: EventBase) -> None: """Checks that an event that defines the retention policy for a room respects the @@ -284,10 +270,16 @@ POWER_LEVELS_SCHEMA = { }, } - -class Mentions(RequestBodyModel): - user_ids: list[StrictStr] = Field(default_factory=list) - room: StrictBool = False +MENTIONS_SCHEMA = { + "type": "object", + "properties": { + "user_ids": { + "type": "array", + "items": {"type": "string"}, + }, + "room": {"type": "boolean"}, + }, +} # This could return something newer than Draft 7, but that's the current "latest" @@ -295,14 +287,45 @@ class Mentions(RequestBodyModel): def _create_validator(schema: JsonDict) -> type[jsonschema.Draft7Validator]: validator = jsonschema.validators.validator_for(schema) - # by default jsonschema does not consider a immutabledict to be an object so - # we need to use a custom type checker + # by default jsonschema does not consider a immutabledict to be an object, or + # a tuple to be an array (frozenutils freezes lists to tuples), so we need a + # custom type checker for both. # https://python-jsonschema.readthedocs.io/en/stable/validate/?highlight=object#validating-with-additional-types type_checker = validator.TYPE_CHECKER.redefine( "object", lambda checker, thing: isinstance(thing, collections.abc.Mapping) + ).redefine( + "array", + lambda checker, thing: isinstance(thing, collections.abc.Sequence), ) return jsonschema.validators.extend(validator, type_checker=type_checker) +def _validation_error_to_api_error(err: jsonschema.ValidationError) -> SynapseError: + """ + Converts a JSONSchema `ValidationError` to a `SynapseError` that can be thrown + to give a Matrix API-compatible 400 Bad Request response with `M_BAD_JSON` code + and a descriptive error message. + """ + if err.path: + # example: "users_default": '0' is not of type 'integer' + # cast safety: path entries can be integers, if we fail to validate + # items in an array. However, the POWER_LEVELS_SCHEMA doesn't expect + # to see any arrays. + message = '"' + cast(str, err.path[-1]) + '": ' + err.message + # jsonschema.ValidationError.message is a valid attribute + else: + # example: '0' is not of type 'integer' + message = err.message + # jsonschema.ValidationError.message is a valid attribute + + return SynapseError( + code=400, + msg=message, + errcode=Codes.BAD_JSON, + ) + + POWER_LEVELS_VALIDATOR = _create_validator(POWER_LEVELS_SCHEMA) + +MENTIONS_VALIDATOR = _create_validator(MENTIONS_SCHEMA) diff --git a/tests/events/test_validator.py b/tests/events/test_validator.py new file mode 100644 index 0000000000..3810fdb3da --- /dev/null +++ b/tests/events/test_validator.py @@ -0,0 +1,50 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +from synapse.api.room_versions import RoomVersions +from synapse.events import make_event_from_dict +from synapse.events.validator import EventValidator + +from tests.unittest import HomeserverTestCase + + +class EventValidatorTestCase(HomeserverTestCase): + def test_validate_new_with_mentions_succeeds_even_when_frozen(self) -> None: + """ + Test that `EventValidator.validate_new` accepts an event with valid `m.mentions` + content even when the event is frozen. + """ + event = make_event_from_dict( + { + "room_id": "!room:test", + "type": "m.room.message", + "sender": "@alice:example.com", + "content": { + "msgtype": "m.text", + "body": "@alice:example.com hello", + "m.mentions": {"user_ids": ["@alice:example.com"]}, + }, + "auth_events": [], + "prev_events": [], + "hashes": {"sha256": "aGVsbG8="}, + "signatures": {}, + "depth": 1, + "origin_server_ts": 1000, + }, + room_version=RoomVersions.V9, + ) + # Sanity check that the event is valid before freezing + EventValidator().validate_new(event, self.hs.config) + event.freeze() + # Event should still be valid after freezing + EventValidator().validate_new(event, self.hs.config) From 4d0e4ff9359ecf40997a5f03890cf2aece410319 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 18 May 2026 12:15:57 +0100 Subject: [PATCH 35/42] Fix `/sync` failing when MSC4354 Sticky Events are enabled and the sync request filters out Ephemeral Data Units (EDUs). (#19787) Fixes: #19779 Fixes: https://github.com/element-hq/synapse/issues/19618 See also: #19786 (which would have caught this, but currently has too many findings to enable) Fix UnboundLocalError when MSC4354 is enabled in sync and all EDUs are filtered out --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19787.bugfix | 1 + synapse/handlers/sync.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 changelog.d/19787.bugfix diff --git a/changelog.d/19787.bugfix b/changelog.d/19787.bugfix new file mode 100644 index 0000000000..26bd3e2252 --- /dev/null +++ b/changelog.d/19787.bugfix @@ -0,0 +1 @@ +Fix `/sync` failing when [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) are enabled and the sync request filters out Ephemeral Data Units (EDUs). \ No newline at end of file diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index bca51b64b7..9ecfe0da0f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2236,19 +2236,23 @@ class SyncHandler: if block_all_room_ephemeral: ephemeral_by_room: dict[str, list[JsonDict]] = {} else: - now_token, ephemeral_by_room = await self.ephemeral_by_room( + ( + sync_result_builder.now_token, + ephemeral_by_room, + ) = await self.ephemeral_by_room( sync_result_builder, now_token=sync_result_builder.now_token, since_token=sync_result_builder.since_token, ) - sync_result_builder.now_token = now_token sticky_by_room: dict[str, list[str]] = {} if self.hs_config.experimental.msc4354_enabled: - now_token, sticky_by_room = await self.sticky_events_by_room( - sync_result_builder, now_token, since_token + ( + sync_result_builder.now_token, + sticky_by_room, + ) = await self.sticky_events_by_room( + sync_result_builder, sync_result_builder.now_token, since_token ) - sync_result_builder.now_token = now_token # 2. We check up front if anything has changed, if it hasn't then there is # no point in going further. From ec4950b438f17efd7ceaf3fd1e8fa2b9c7d41ac4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 16:10:40 +0000 Subject: [PATCH 36/42] Bump types-jsonschema from 4.26.0.20260202 to 4.26.0.20260508 (#19683) Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 78f18caadf..ab790015bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3417,14 +3417,14 @@ types-webencodings = "*" [[package]] name = "types-jsonschema" -version = "4.26.0.20260202" +version = "4.26.0.20260508" description = "Typing stubs for jsonschema" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "types_jsonschema-4.26.0.20260202-py3-none-any.whl", hash = "sha256:41c95343abc4de9264e333a55e95dfb4d401e463856d0164eec9cb182e8746da"}, - {file = "types_jsonschema-4.26.0.20260202.tar.gz", hash = "sha256:29831baa4308865a9aec547a61797a06fc152b0dac8dddd531e002f32265cb07"}, + {file = "types_jsonschema-4.26.0.20260508-py3-none-any.whl", hash = "sha256:4ec1dea0a757c8c2e2aa7bc085612fb54e1ae9562428d5da6f26dd7a0f24dbc2"}, + {file = "types_jsonschema-4.26.0.20260508.tar.gz", hash = "sha256:ae0be85ac6ec0cb94a98f75f876b0620cf2afa3e37fdf8460203f4d05f745acb"}, ] [package.dependencies] From 7b1c4da5dfbe48fa5e0dabc3933aed54f40e1644 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 19 May 2026 14:13:03 +0100 Subject: [PATCH 37/42] 1.153.0 --- CHANGES.md | 5 +++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 96a9fadbfb..95ab39d0e3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +# Synapse 1.153.0 (2026-05-19) + +No significant changes since 1.153.0rc3. + + # Synapse 1.153.0rc3 (2026-05-15) ## Bugfixes diff --git a/debian/changelog b/debian/changelog index 81f080bead..b1a4e04bc3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.153.0) stable; urgency=medium + + * New Synapse release 1.153.0. + + -- Synapse Packaging team Tue, 19 May 2026 14:12:51 +0100 + matrix-synapse-py3 (1.153.0~rc3) stable; urgency=medium * New Synapse release 1.153.0rc3. diff --git a/pyproject.toml b/pyproject.toml index fed1a23f4a..66cd3e83b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.153.0rc3" +version = "1.153.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 87095ae500ef1371346f57ef6dfb29231e822cb9 Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Wed, 20 May 2026 12:06:16 +0200 Subject: [PATCH 38/42] fix: invalidate access token cache on device deletion (#19483) when an access token had a refresh token associated to it in the database, deleting this refresh token (for example when deleting the device using it) would cascade delete the access token, which wouldn't be returned by the sql query that was supposed to delete it on its own, and an empty array was passed to the cache invalidation function. --- changelog.d/19483.bugfix | 1 + .../storage/databases/main/registration.py | 19 ++-- tests/rest/client/test_auth.py | 90 ++++++++++++++++++- 3 files changed, 100 insertions(+), 10 deletions(-) create mode 100644 changelog.d/19483.bugfix diff --git a/changelog.d/19483.bugfix b/changelog.d/19483.bugfix new file mode 100644 index 0000000000..9e4fb20996 --- /dev/null +++ b/changelog.d/19483.bugfix @@ -0,0 +1 @@ +Fix access token cache not being invalidated for sessions using refresh tokens. Contributed by @FrenchGithubUser @ Famedly. diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 7b0de92435..2455057b02 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -2474,14 +2474,6 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): def user_delete_access_tokens_for_devices_txn( txn: LoggingTransaction, batch_device_ids: StrCollection ) -> list[tuple[str, int, str | None]]: - self.db_pool.simple_delete_many_txn( - txn, - table="refresh_tokens", - keyvalues={"user_id": user_id}, - column="device_id", - values=batch_device_ids, - ) - clause, args = make_in_list_sql_clause( txn.database_engine, "device_id", batch_device_ids ) @@ -2500,6 +2492,17 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): self.get_user_by_access_token, [(t[0],) for t in tokens_and_devices], ) + # Delete access tokens first, before refresh tokens. + # This ensures we can capture the deleted access tokens for cache invalidation + # before any CASCADE deletes occur. + self.db_pool.simple_delete_many_txn( + txn, + table="refresh_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + return tokens_and_devices results = [] diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py index c266937a65..4eb918d5c5 100644 --- a/tests/rest/client/test_auth.py +++ b/tests/rest/client/test_auth.py @@ -30,7 +30,14 @@ import synapse.rest.admin from synapse.api.constants import ApprovalNoticeMedium, LoginType from synapse.api.errors import Codes, SynapseError from synapse.handlers.ui_auth.checkers import UserInteractiveAuthChecker -from synapse.rest.client import account, auth, devices, login, logout, register +from synapse.rest.client import ( + account, + auth, + devices, + login, + logout, + register, +) from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.server import HomeServer from synapse.storage.database import LoggingTransaction @@ -765,10 +772,11 @@ class RefreshAuthTests(unittest.HomeserverTestCase): servlets = [ auth.register_servlets, account.register_servlets, + devices.register_servlets, login.register_servlets, logout.register_servlets, - synapse.rest.admin.register_servlets_for_client_rest_resource, register.register_servlets, + synapse.rest.admin.register_servlets_for_client_rest_resource, ] hijack_auth = False @@ -1309,6 +1317,84 @@ class RefreshAuthTests(unittest.HomeserverTestCase): self.assertEqual(_table_length("access_tokens"), 0) self.assertEqual(_table_length("refresh_tokens"), 0) + def test_token_invalid_after_refresh_token_issued_and_device_removed(self) -> None: + """ + Test that an access token is invalidated after the device (which had a + refresh token) is removed by another device. + The removal of a refresh token cascade deletes the associated access + token in the db, which can make cache invalidation fail, if not handled + properly. This test will catch such behavior if it ever happens again. + 1. User logs in with device1 + 2. User logs in with device2 and requests a refresh token + 3. Device2 calls /whoami (should work) + 4. Device1 removes device2 + 5. Device2 calls /whoami (should fail) + """ + # Login with device1 + device1_tok = self.login("test", self.user_pass, device_id="device1") + + # Login with device2 and request a refresh token + login_response = self.make_request( + "POST", + "/_matrix/client/r0/login", + { + "type": "m.login.password", + "user": "test", + "password": self.user_pass, + "device_id": "device2", + "refresh_token": True, + }, + ) + self.assertEqual(login_response.code, HTTPStatus.OK, login_response.result) + device2_tok = login_response.json_body["access_token"] + device2_id = login_response.json_body["device_id"] + self.assertEqual(device2_id, "device2") + + # Device2 calls /whoami (should work) + channel = self.make_request( + "GET", + "/_matrix/client/v3/account/whoami", + access_token=device2_tok, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + + # Device1 removes device2 + # First, attempt to delete device2 + delete_channel = self.make_request( + "DELETE", + f"devices/{device2_id}", + access_token=device1_tok, + ) + self.assertEqual( + delete_channel.code, HTTPStatus.UNAUTHORIZED, delete_channel.result + ) + session = delete_channel.json_body["session"] + + # Complete the UI auth flow + delete_channel = self.make_request( + "DELETE", + f"devices/{device2_id}", + content={ + "auth": { + "type": "m.login.password", + "user": "test", + "password": self.user_pass, + "session": session, + }, + }, + access_token=device1_tok, + ) + self.assertEqual(delete_channel.code, HTTPStatus.OK, delete_channel.result) + + # Device2 calls /whoami (should fail) + channel = self.make_request( + "GET", + "/_matrix/client/v3/account/whoami", + access_token=device2_tok, + ) + self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.result) + self.assertEqual(channel.json_body["errcode"], "M_UNKNOWN_TOKEN") + def oidc_config( id: str, with_localpart_template: bool, **kwargs: Any From be03be7b50ca914d73757a00d2819807187aa3a8 Mon Sep 17 00:00:00 2001 From: Oleg Girko Date: Wed, 20 May 2026 11:47:43 +0100 Subject: [PATCH 39/42] Partially revert "Bump attrs from 25.4.0 to 26.1.0 (#19684)" (#19789) Accidental bump broke build for Fedora and RHEL. This reverts commit 2e9d6f7f35df53f510b689c09d3a9ace82a0d840. As discussed in the [Synapse Package Maintainers](https://matrix.to/#/!rh9Uxk45AsPongyP3ypgpsCmuIufiggD6mDXFWh4_FM/$0mdulZEyJFdI6bwS8GFwYnFt-zmpyCyx2DwcA8JyuY8?via=jki.re&via=matrix.org&via=element.io) room (private) --- changelog.d/19789.bugfix | 1 + poetry.lock | 53 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- 3 files changed, 29 insertions(+), 27 deletions(-) create mode 100644 changelog.d/19789.bugfix diff --git a/changelog.d/19789.bugfix b/changelog.d/19789.bugfix new file mode 100644 index 0000000000..f6c325ec82 --- /dev/null +++ b/changelog.d/19789.bugfix @@ -0,0 +1 @@ +Fix packaging for Fedora and EPEL caused by unnecessary bumping `attrs` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. diff --git a/poetry.lock b/poetry.lock index ab790015bb..92c25d9308 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -531,7 +531,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +556,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +606,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -889,7 +889,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -930,7 +930,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1122,7 +1122,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1239,7 +1239,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1553,7 +1553,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1834,7 +1834,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -2032,7 +2032,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2050,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2066,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2348,7 +2348,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2480,7 +2480,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2505,7 +2505,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2533,7 +2533,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2938,7 +2938,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3138,7 +3138,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3154,7 +3154,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3220,6 +3220,7 @@ files = [ {file = "tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a"}, {file = "tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c"}, ] +markers = {main = "python_version < \"3.14\""} [[package]] name = "tornado" @@ -3228,7 +3229,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3360,7 +3361,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3621,7 +3622,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3642,7 +3643,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, @@ -3755,4 +3756,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "d97bee07fec0f4048d964aa7127a50813920bce77b00e5191aa1815f83922c85" +content-hash = "ef0540b89c417a69668f551688bd0974256ea7a580044f3954a76bdf0d8fe7c9" diff --git a/pyproject.toml b/pyproject.toml index 66cd3e83b5..e92e7b5c21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ dependencies = [ "prometheus-client>=0.6.0", # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936 - "attrs>=26.1.0,!=21.1.0", + "attrs>=19.2.0,!=21.1.0", "netaddr>=0.7.18", # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not # end up with a broken installation, with recent MarkupSafe but old Jinja, we From 966e193e4e1900bc28891ed697aee30ab5a72dbd Mon Sep 17 00:00:00 2001 From: Joe Groocock Date: Wed, 20 May 2026 12:10:54 +0100 Subject: [PATCH 40/42] Fix sending heroes in SSS when `m.room.name=""` (#19468) As per the spec, a room with m.room.name value that is absent, null or empty should be treated as if there is no m.room.name event at all: https://spec.matrix.org/v1.17/client-server-api/#mroomname This fetches the full m.room.name event and checks the content.name instead of only checking the existence of the m.room.name event. This results in correctly sending heroes for those rooms. Fixes: https://github.com/element-hq/synapse/issues/19447 Signed-off-by: Joe Groocock --- changelog.d/19468.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 51 +++++++------- .../client/sliding_sync/test_rooms_meta.py | 70 +++++++++++++++++++ 3 files changed, 98 insertions(+), 24 deletions(-) create mode 100644 changelog.d/19468.bugfix diff --git a/changelog.d/19468.bugfix b/changelog.d/19468.bugfix new file mode 100644 index 0000000000..003716d296 --- /dev/null +++ b/changelog.d/19468.bugfix @@ -0,0 +1 @@ +Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index ddce04d5b3..df1270f7f9 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -872,20 +872,10 @@ class SlidingSyncHandler: # For incremental syncs, we can do this first to determine if something relevant # has changed and strategically avoid fetching other costly things. room_state_delta_id_map: MutableStateMap[str] = {} - name_event_id: str | None = None membership_changed = False name_changed = False avatar_changed = False - if initial: - # Check whether the room has a name set - name_state_ids = await self.get_current_state_ids_at( - room_id=room_id, - room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, - state_filter=StateFilter.from_types([(EventTypes.Name, "")]), - to_token=to_token, - ) - name_event_id = name_state_ids.get((EventTypes.Name, "")) - else: + if not initial: assert from_bound is not None # TODO: Limit the number of state events we're about to send down @@ -933,6 +923,27 @@ class SlidingSyncHandler: ): avatar_changed = True + # If a room has an m.room.name event with an absent, null, or empty + # name field, it should be treated the same as a room with no + # m.room.name event. + # https://spec.matrix.org/v1.17/client-server-api/#mroomname + # + # TODO: Should we also check for `EventTypes.CanonicalAlias` + # (`m.room.canonical_alias`) as a fallback for the room name? see + # https://github.com/matrix-org/matrix-spec-proposals/pull/4186/changes#r2860107511 + room_name: str | None = None + if initial or name_changed: + # Check whether the room has a name set + name_states = await self.get_current_state_at( + room_id=room_id, + room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, + state_filter=StateFilter.from_types([(EventTypes.Name, "")]), + to_token=to_token, + ) + name_event = name_states.get((EventTypes.Name, "")) + if name_event is not None: + room_name = name_event.content.get("name") + # We only need the room summary for calculating heroes, however if we do # fetch it then we can use it to calculate `joined_count` and # `invited_count`. @@ -949,12 +960,13 @@ class SlidingSyncHandler: hero_user_ids: list[str] = [] # TODO: Should we also check for `EventTypes.CanonicalAlias` # (`m.room.canonical_alias`) as a fallback for the room name? see - # https://github.com/matrix-org/matrix-spec-proposals/pull/3575#discussion_r1671260153 + # https://github.com/matrix-org/matrix-spec-proposals/pull/4186/changes#r2860107511 # - # We need to fetch the `heroes` if the room name is not set. But we only need to - # get them on initial syncs (or the first time we send down the room) or if the + # We need to fetch the `heroes` if the room name is not set (taking + # care to treat an empty string as unset). But we only need to get them + # on initial syncs (or the first time we send down the room) or if the # membership has changed which may change the heroes. - if name_event_id is None and (initial or (not initial and membership_changed)): + if not room_name and (initial or (not initial and membership_changed)): # We need the room summary to extract the heroes from if room_membership_for_user_at_to_token.membership != Membership.JOIN: # TODO: Figure out how to get the membership summary for left/banned rooms @@ -1332,15 +1344,6 @@ class SlidingSyncHandler: if required_state_filter != StateFilter.none(): required_room_state = required_state_filter.filter_state(room_state) - # Find the room name and avatar from the state - room_name: str | None = None - # TODO: Should we also check for `EventTypes.CanonicalAlias` - # (`m.room.canonical_alias`) as a fallback for the room name? see - # https://github.com/matrix-org/matrix-spec-proposals/pull/3575#discussion_r1671260153 - name_event = room_state.get((EventTypes.Name, "")) - if name_event is not None: - room_name = name_event.content.get("name") - room_avatar: str | None = None avatar_event = room_state.get((EventTypes.RoomAvatar, "")) if avatar_event is not None: diff --git a/tests/rest/client/sliding_sync/test_rooms_meta.py b/tests/rest/client/sliding_sync/test_rooms_meta.py index 4ff07b4d26..b1b771ef84 100644 --- a/tests/rest/client/sliding_sync/test_rooms_meta.py +++ b/tests/rest/client/sliding_sync/test_rooms_meta.py @@ -631,6 +631,76 @@ class SlidingSyncRoomsMetaTestCase(SlidingSyncBase): # We didn't request any state so we shouldn't see any `required_state` self.assertIsNone(response_body["rooms"][room_id1].get("required_state")) + def test_rooms_meta_heroes_empty_room_name(self) -> None: + """ + Test that the `rooms` `heroes` are included when the room name is an + empty string (i.e. unset as per the spec) + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + user3_id = self.register_user("user3", "pass") + _user3_tok = self.login(user3_id, "pass") + + room_id = self.helper.create_room_as( + user2_id, + tok=user2_tok, + extra_content={ + # https://spec.matrix.org/v1.17/client-server-api/#mroomname + # > If a room has an m.room.name event with an absent, null, or + # > empty name field, it should be treated the same as a room + # > with no m.room.name event. + "name": "", + }, + ) + self.helper.join(room_id, user1_id, tok=user1_tok) + # User3 is invited + self.helper.invite(room_id, src=user2_id, targ=user3_id, tok=user2_tok) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 1, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Room has an empty name so we should see `heroes` populated + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + self.assertIsNone(response_body["rooms"][room_id].get("name")) + self.assertCountEqual( + [ + hero["user_id"] + for hero in response_body["rooms"][room_id].get("heroes", []) + ], + # Heroes shouldn't include the user themselves (we shouldn't see user1) + [user2_id, user3_id], + ) + self.assertEqual( + response_body["rooms"][room_id]["joined_count"], + 2, + ) + self.assertEqual( + response_body["rooms"][room_id]["invited_count"], + 1, + ) + + # We didn't request any state so we shouldn't see any `required_state` + self.assertIsNone(response_body["rooms"][room_id].get("required_state")) + + # Send a message to make the room come down sync + self.helper.send(room_id, "message in room", tok=user2_tok) + + # Incremental sync + incremental_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + self.assertNotIn("name", incremental_body["rooms"][room_id]) + self.assertNotIn("heroes", incremental_body["rooms"][room_id]) + def test_rooms_meta_heroes_when_banned(self) -> None: """ Test that the `rooms` `heroes` are included in the response when the room From 57299fd5bdb1a563c41b03da1e75b8a55413701c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 13:32:08 +0000 Subject: [PATCH 41/42] Bump lxml from 6.0.2 to 6.1.0 (#19716) Signed-off-by: dependabot[bot] --- poetry.lock | 327 +++++++++++++++++++++++++--------------------------- 1 file changed, 160 insertions(+), 167 deletions(-) diff --git a/poetry.lock b/poetry.lock index 92c25d9308..6093b13c02 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -531,7 +531,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +556,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +606,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -889,7 +889,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -930,7 +930,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1122,7 +1122,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1234,153 +1234,147 @@ files = [ [[package]] name = "lxml" -version = "6.0.2" +version = "6.1.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ - {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, - {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f952dacaa552f3bb8834908dddd500ba7d508e6ea6eb8c52eb2d28f48ca06a31"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71695772df6acea9f3c0e59e44ba8ac50c4f125217e84aab21074a1a55e7e5c9"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f68764f35fd78d7c4cc4ef209a184c38b65440378013d24b8aecd327c3e0c8"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:058027e261afed589eddcfe530fcc6f3402d7fd7e89bfd0532df82ebc1563dba"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8ffaeec5dfea5881d4c9d8913a32d10cfe3923495386106e4a24d45300ef79c"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:f2e3b1a6bb38de0bc713edd4d612969dd250ca8b724be8d460001a387507021c"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d6690ec5ec1cce0385cb20896b16be35247ac8c2046e493d03232f1c2414d321"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2a50c3c1d11cad0ebebbac357a97b26aa79d2bcaf46f256551152aa85d3a4d1"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:3efe1b21c7801ffa29a1112fab3b0f643628c30472d507f39544fd48e9549e34"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:59c45e125140b2c4b33920d21d83681940ca29f0b83f8629ea1a2196dc8cfe6a"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:452b899faa64f1805943ec1c0c9ebeaece01a1af83e130b69cdefeda180bb42c"}, - {file = "lxml-6.0.2-cp310-cp310-win32.whl", hash = "sha256:1e786a464c191ca43b133906c6903a7e4d56bef376b75d97ccbb8ec5cf1f0a4b"}, - {file = "lxml-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:dacf3c64ef3f7440e3167aa4b49aa9e0fb99e0aa4f9ff03795640bf94531bcb0"}, - {file = "lxml-6.0.2-cp310-cp310-win_arm64.whl", hash = "sha256:45f93e6f75123f88d7f0cfd90f2d05f441b808562bf0bc01070a00f53f5028b5"}, - {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607"}, - {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7"}, - {file = "lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46"}, - {file = "lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078"}, - {file = "lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285"}, - {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456"}, - {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322"}, - {file = "lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849"}, - {file = "lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f"}, - {file = "lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6"}, - {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77"}, - {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314"}, - {file = "lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2"}, - {file = "lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7"}, - {file = "lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf"}, - {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe"}, - {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c"}, - {file = "lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b"}, - {file = "lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed"}, - {file = "lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8"}, - {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d"}, - {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f"}, - {file = "lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312"}, - {file = "lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca"}, - {file = "lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c"}, - {file = "lxml-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a656ca105115f6b766bba324f23a67914d9c728dafec57638e2b92a9dcd76c62"}, - {file = "lxml-6.0.2-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c54d83a2188a10ebdba573f16bd97135d06c9ef60c3dc495315c7a28c80a263f"}, - {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:1ea99340b3c729beea786f78c38f60f4795622f36e305d9c9be402201efdc3b7"}, - {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:af85529ae8d2a453feee4c780d9406a5e3b17cee0dd75c18bd31adcd584debc3"}, - {file = "lxml-6.0.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:fe659f6b5d10fb5a17f00a50eb903eb277a71ee35df4615db573c069bcf967ac"}, - {file = "lxml-6.0.2-cp38-cp38-win32.whl", hash = "sha256:5921d924aa5468c939d95c9814fa9f9b5935a6ff4e679e26aaf2951f74043512"}, - {file = "lxml-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:0aa7070978f893954008ab73bb9e3c24a7c56c054e00566a21b553dc18105fca"}, - {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2c8458c2cdd29589a8367c09c8f030f1d202be673f0ca224ec18590b3b9fb694"}, - {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fee0851639d06276e6b387f1c190eb9d7f06f7f53514e966b26bae46481ec90"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b2142a376b40b6736dfc214fd2902409e9e3857eff554fed2d3c60f097e62a62"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a6b5b39cc7e2998f968f05309e666103b53e2edd01df8dc51b90d734c0825444"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4aec24d6b72ee457ec665344a29acb2d35937d5192faebe429ea02633151aad"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:b42f4d86b451c2f9d06ffb4f8bbc776e04df3ba070b9fe2657804b1b40277c48"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cdaefac66e8b8f30e37a9b4768a391e1f8a16a7526d5bc77a7928408ef68e93"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:b738f7e648735714bbb82bdfd030203360cfeab7f6e8a34772b3c8c8b820568c"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daf42de090d59db025af61ce6bdb2521f0f102ea0e6ea310f13c17610a97da4c"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:66328dabea70b5ba7e53d94aa774b733cf66686535f3bc9250a7aab53a91caaf"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:e237b807d68a61fc3b1e845407e27e5eb8ef69bc93fe8505337c1acb4ee300b6"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:ac02dc29fd397608f8eb15ac1610ae2f2f0154b03f631e6d724d9e2ad4ee2c84"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:817ef43a0c0b4a77bd166dc9a09a555394105ff3374777ad41f453526e37f9cb"}, - {file = "lxml-6.0.2-cp39-cp39-win32.whl", hash = "sha256:bc532422ff26b304cfb62b328826bd995c96154ffd2bac4544f37dbb95ecaa8f"}, - {file = "lxml-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:995e783eb0374c120f528f807443ad5a83a656a8624c467ea73781fc5f8a8304"}, - {file = "lxml-6.0.2-cp39-cp39-win_arm64.whl", hash = "sha256:08b9d5e803c2e4725ae9e8559ee880e5328ed61aa0935244e0515d7d9dbec0aa"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e748d4cf8fef2526bb2a589a417eba0c8674e29ffcb570ce2ceca44f1e567bf6"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4ddb1049fa0579d0cbd00503ad8c58b9ab34d1254c77bc6a5576d96ec7853dba"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb233f9c95f83707dae461b12b720c1af9c28c2d19208e1be03387222151daf5"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc456d04db0515ce3320d714a1eac7a97774ff0849e7718b492d957da4631dd4"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2613e67de13d619fd283d58bda40bff0ee07739f624ffee8b13b631abf33083d"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:24a8e756c982c001ca8d59e87c80c4d9dcd4d9b44a4cbeb8d9be4482c514d41d"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e"}, - {file = "lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62"}, + {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, + {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1bc4cc83fb7f66ffb16f74d6dd0162e144333fc36ebcce32246f80c8735b2551"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:20cf4d0651987c906a2f5cba4e3a8d6ba4bfdf973cfe2a96c0d6053888ea2ecd"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffb34ea45a82dd637c2c97ae1bbb920850c1e59bcae79ce1c15af531d83e7215"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1d9b99e5b2597e4f5aed2484fef835256fa1b68a19e4265c97628ef4bf8bcf4"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_28_i686.whl", hash = "sha256:d43aa26dcda363f21e79afa0668f5029ed7394b3bb8c92a6927a3d34e8b610ea"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:6262b87f9e5c1e5fe501d6c153247289af42eb44ad7660b9b3de17baaf92d6f6"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d1392c569c032f78a11a25d1de1c43fff13294c793b39e19d84fade3045cbbc3"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:045e387d1f4f42a418380930fa3f45c73c9b392faf67e495e58902e68e8f44a7"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:9f93d5b8b07f73e8c77e3c6556a3db269918390c804b5e5fcdd4858232cc8f16"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:de550d129f18d8ab819651ffe4f38b1b713c7e116707de3c0c6400d0ef34fbc1"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c08da09dc003c9e8c70e06b53a11db6fb3b250c21c4236b03c7d7b443c318e7a"}, + {file = "lxml-6.1.0-cp310-cp310-win32.whl", hash = "sha256:37448bf9c7d7adfc5254763901e2bbd6bb876228dfc1fc7f66e58c06368a7544"}, + {file = "lxml-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:2593a0a6621545b9095b71ad74ed4226eba438a7d9fc3712a99bdb15508cf93a"}, + {file = "lxml-6.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:e80807d72f96b96ad5588cb85c75616e4f2795a7737d4630784c51497beb7776"}, + {file = "lxml-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cec05be8c876f92a5aa07b01d60bbb4d11cfbdd654cad0561c0d7b5c043a61b9"}, + {file = "lxml-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9c03e048b6ce8e77b09c734e931584894ecd58d08296804ca2d0b184c933ce50"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:942454ff253da14218f972b23dc72fa4edf6c943f37edd19cd697618b626fac5"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d036ee7b99d5148072ac7c9b847193decdfeac633db350363f7bce4fff108f0e"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ae5d8d5427f3cc317e7950f2da7ad276df0cfa37b8de2f5658959e618ea8512"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:363e47283bde87051b821826e71dde47f107e08614e1aa312ba0c5711e77738c"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:f504d861d9f2a8f94020130adac88d66de93841707a23a86244263d1e54682f5"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:23a5dc68e08ed13331d61815c08f260f46b4a60fdd1640bbeb82cf89a9d90289"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f15401d8d3dbf239e23c818afc10c7207f7b95f9a307e092122b6f86dd43209a"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fcf3da95e93349e0647d48d4b36a12783105bcc74cb0c416952f9988410846a3"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:0d082495c5fcf426e425a6e28daaba1fcb6d8f854a4ff01effb1f1f381203eb9"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e3c4f84b24a1fcba435157d111c4b755099c6ff00a3daee1ad281817de75ed11"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:976a6b39b1b13e8c354ad8d3f261f3a4ac6609518af91bdb5094760a08f132c4"}, + {file = "lxml-6.1.0-cp311-cp311-win32.whl", hash = "sha256:857efde87d365706590847b916baff69c0bc9252dc5af030e378c9800c0b10e3"}, + {file = "lxml-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:183bfb45a493081943be7ea2b5adfc2b611e1cf377cefa8b8a8be404f45ef9a7"}, + {file = "lxml-6.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:19f4164243fc206d12ed3d866e80e74f5bc3627966520da1a5f97e42c32a3f39"}, + {file = "lxml-6.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d2f17a16cd8751e8eb233a7e41aecdf8e511712e00088bf9be455f604cd0d28d"}, + {file = "lxml-6.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f0cea5b1d3e6e77d71bd2b9972eb2446221a69dc52bb0b9c3c6f6e5700592d93"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc46da94826188ed45cb53bd8e3fc076ae22675aea2087843d4735627f867c6d"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9147d8e386ec3b82c3b15d88927f734f565b0aaadef7def562b853adca45784a"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5715e0e28736a070f3f34a7ccc09e2fdcba0e3060abbcf61a1a5718ff6d6b105"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4937460dc5df0cdd2f06a86c285c28afda06aefa3af949f9477d3e8df430c485"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc783ee3147e60a25aa0445ea82b3e8aabb83b240f2b95d32cb75587ff781814"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:40d9189f80075f2e1f88db21ef815a2b17b28adf8e50aaf5c789bfe737027f32"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:05b9b8787e35bec69e68daf4952b2e6dfcfb0db7ecf1a06f8cdfbbac4eb71aad"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0f0f08beb0182e3e9a86fae124b3c47a7b41b7b69b225e1377db983802404e54"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73becf6d8c81d4c76b1014dbd3584cb26d904492dcf73ca85dc8bff08dcd6d2d"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1ae225f66e5938f4fa29d37e009a3bb3b13032ac57eb4eb42afa44f6e4054e69"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:690022c7fae793b0489aa68a658822cea83e0d5933781811cabbf5ea3bcfe73d"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:63aeafc26aac0be8aff14af7871249e87ea1319be92090bfd632ec68e03b16a5"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:264c605ab9c0e4aa1a679636f4582c4d3313700009fac3ec9c3412ed0d8f3e1d"}, + {file = "lxml-6.1.0-cp312-cp312-win32.whl", hash = "sha256:56971379bc5ee8037c5a0f09fa88f66cdb7d37c3e38af3e45cf539f41131ac1f"}, + {file = "lxml-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bba078de0031c219e5dd06cf3e6bf8fb8e6e64a77819b358f53bb132e3e03366"}, + {file = "lxml-6.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3592631e652afa34999a088f98ba7dfc7d6aff0d535c410bea77a71743f3819"}, + {file = "lxml-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a0092f2b107b69601adf562a57c956fbb596e05e3e6651cabd3054113b007e45"}, + {file = "lxml-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fc7140d7a7386e6b545d41b7358f4d02b656d4053f5fa6859f92f4b9c2572c4d"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:419c58fc92cc3a2c3fa5f78c63dbf5da70c1fa9c1b25f25727ecee89a96c7de2"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:37fabd1452852636cf38ecdcc9dd5ca4bba7a35d6c53fa09725deeb894a87491"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2853c8b2170cc6cd54a6b4d50d2c1a8a7aeca201f23804b4898525c7a152cfc"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e369cbd690e788c8d15e56222d91a09c6a417f49cbc543040cba0fe2e25a79e"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e69aa6805905807186eb00e66c6d97a935c928275182eb02ee40ba00da9623b2"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:4bd1bdb8a9e0e2dd229de19b5f8aebac80e916921b4b2c6ef8a52bc131d0c1f9"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:cbd7b79cdcb4986ad78a2662625882747f09db5e4cd7b2ae178a88c9c51b3dfe"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:43e4d297f11080ec9d64a4b1ad7ac02b4484c9f0e2179d9c4ef78e886e747b88"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cc16682cc987a3da00aa56a3aa3075b08edb10d9b1e476938cfdbee8f3b67181"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:d6d8efe71429635f0559579092bb5e60560d7b9115ee38c4adbea35632e7fa24"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e39ab3a28af7784e206d8606ec0e4bcad0190f63a492bca95e94e5a4aef7f6e"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9eb667bf50856c4a58145f8ca2d5e5be160191e79eb9e30855a476191b3c3495"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7f4a77d6f7edf9230cee3e1f7f6764722a41604ee5681844f18db9a81ea0ec33"}, + {file = "lxml-6.1.0-cp313-cp313-win32.whl", hash = "sha256:28902146ffbe5222df411c5d19e5352490122e14447e98cd118907ee3fd6ee62"}, + {file = "lxml-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:4a1503c56e4e2b38dc76f2f2da7bae69670c0f1933e27cfa34b2fa5876410b16"}, + {file = "lxml-6.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:e0af85773850417d994d019741239b901b22c6680206f46a34766926e466141d"}, + {file = "lxml-6.1.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ab863fd37458fed6456525f297d21239d987800c46e67da5ef04fc6b3dd93ac8"}, + {file = "lxml-6.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6fd8b1df8254ff4fd93fd31da1fc15770bde23ac045be9bb1f87425702f61cc9"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:47024feaae386a92a146af0d2aeed65229bf6fff738e6a11dda6b0015fb8fd03"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3f00972f84450204cd5d93a5395965e348956aaceaadec693a22ec743f8ae3eb"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97faa0860e13b05b15a51fb4986421ef7a30f0b3334061c416e0981e9450ca4c"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:972a6451204798675407beaad97b868d0c733d9a74dafefc63120b81b8c2de28"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fe022f20bc4569ec66b63b3fb275a3d628d9d32da6326b2982584104db6d3086"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:75c4c7c619a744f972f4451bf5adf6d0fb00992a1ffc9fd78e13b0bc817cc99f"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3648f20d25102a22b6061c688beb3a805099ea4beb0a01ce62975d926944d292"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:77b9f99b17cbf14026d1e618035077060fc7195dd940d025149f3e2e830fbfcb"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32662519149fd7a9db354175aa5e417d83485a8039b8aaa62f873ceee7ea4cad"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:73d658216fc173cf2c939e90e07b941c5e12736b0bf6a99e7af95459cfe8eabb"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ac4db068889f8772a4a698c5980ec302771bb545e10c4b095d4c8be26749616f"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:45e9dfbd1b661eb64ba0d4dbe762bd210c42d86dd1e5bd2bdf89d634231beb43"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:89e8d73d09ac696a5ba42ec69787913d53284f12092f651506779314f10ba585"}, + {file = "lxml-6.1.0-cp314-cp314-win32.whl", hash = "sha256:ebe33f4ec1b2de38ceb225a1749a2965855bffeef435ba93cd2d5d540783bf2f"}, + {file = "lxml-6.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:398443df51c538bd578529aa7e5f7afc6c292644174b47961f3bf87fe5741120"}, + {file = "lxml-6.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:8c8984e1d8c4b3949e419158fda14d921ff703a9ed8a47236c6eb7a2b6cb4946"}, + {file = "lxml-6.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1081dd10bc6fa437db2500e13993abf7cc30716d0a2f40e65abb935f02ec559c"}, + {file = "lxml-6.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dabecc48db5f42ba348d1f5d5afdc54c6c4cc758e676926c7cd327045749517d"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e3dd5fe19c9e0ac818a9c7f132a5e43c1339ec1cbbfecb1a938bd3a47875b7c9"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e7b0a4ca6dcc007a4cef00a761bba2dea959de4bd2df98f926b33c92ca5dfb9"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d27bbe326c6b539c64b42638b18bc6003a8d88f76213a97ac9ed4f885efeab7"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4e425db0c5445ef0ad56b0eec54f89b88b2d884656e536a90b2f52aecb4ca86"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b89b098105b8599dc57adac95d1813409ac476d3c948a498775d3d0c6124bfb"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:c4a699432846df86cc3de502ee85f445ebad748a1c6021d445f3e514d2cd4b1c"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:30e7b2ed63b6c8e97cca8af048589a788ab5c9c905f36d9cf1c2bb549f450d2f"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:022981127642fe19866d2907d76241bb07ed21749601f727d5d5dd1ce5d1b773"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:23cad0cc86046d4222f7f418910e46b89971c5a45d3c8abfad0f64b7b05e4a9b"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:21c3302068f50d1e8728c67c87ba92aa87043abee517aa2576cca1855326b405"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:be10838781cb3be19251e276910cd508fe127e27c3242e50521521a0f3781690"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2173a7bffe97667bbf0767f8a99e587740a8c56fdf3befac4b09cb29a80276fd"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c6854e9cf99c84beb004eecd7d3a3868ef1109bf2b1df92d7bc11e96a36c2180"}, + {file = "lxml-6.1.0-cp314-cp314t-win32.whl", hash = "sha256:00750d63ef0031a05331b9223463b1c7c02b9004cef2346a5b2877f0f9494dd2"}, + {file = "lxml-6.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:80410c3a7e3c617af04de17caa9f9f20adaa817093293d69eae7d7d0522836f5"}, + {file = "lxml-6.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:26dd9f57ee3bd41e7d35b4c98a2ffd89ed11591649f421f0ec19f67d50ec67ac"}, + {file = "lxml-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b6c2f225662bc5ad416bdd06f72ca301b31b39ce4261f0e0097017fc2891b940"}, + {file = "lxml-6.1.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a86f06f059e22a0d574990ee2df24ede03f7f3c68c1336293eee9536c4c776cd"}, + {file = "lxml-6.1.0-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:468479e52ecf3ec23799c863336d02c05fc2f7ffd1a1424eeeb9a28d4eb69d13"}, + {file = "lxml-6.1.0-cp38-cp38-manylinux_2_28_i686.whl", hash = "sha256:a02ca8fe48815bddcfca3248efe54451abb9dbf2f7d1c5744c8aa4142d476919"}, + {file = "lxml-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:bb40648d96157f9081886defe13eac99253e663be969ff938a9289eff6e47b72"}, + {file = "lxml-6.1.0-cp38-cp38-win32.whl", hash = "sha256:1dd6a1c3ad4cb674f44525d9957f3e9c209bb6dd9213245195167a281fcc2bdc"}, + {file = "lxml-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:4e2c54d6b47361d0f1d3bc8d4e082ad87201e56ccdcca4d3b9ee3644ff595ec8"}, + {file = "lxml-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:920354904d1cb86577d4b3cfe2830c2dbe81d6f4449e57ada428f1609b5985f7"}, + {file = "lxml-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c871299c595ee004d186f61840f0bfc4941aa3f17c8ba4a565ead7e4f4f820ee"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d0d799ff958655781296ec870d5e2448e75150da2b3d07f13ff5b0c2c35beefd"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ba11752e346bd804ea312ec2eea2532dfa8b8d3261d81a32ef9e6ab16256280"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26c5272c6a4bf4cf32d3f5a7890c942b0e04438691157d341616d02cca74d4bd"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c53fa3a5a52122d590e847a57ccf955557b9634a7f99ff5a35131321b0a85317"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_28_i686.whl", hash = "sha256:76b958b4ea3104483c20f74866d55aa056546e15ebe83dd7aecd63698f43b755"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:8c11b984b5ce6add4dccc7144c7be5d364d298f15b0c6a57da1991baedc750ce"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d3829a6e6fd550a219564912d4002c537f65da4c6ae4e093cc34462f4fa027ad"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:52b0ac6903cf74ebf997eb8c682d2fbac7d1ab7e4c552413eec55868a9b73f39"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:29f5c00cb7d752bce2c70ebd2d31b0a42f9499ffdd3ecb2f31a5b73ee43031ad"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:c748ebcb6877de89f48ab90ca96642ac458fff5dec291a2b9337cd4d0934e383"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:08950a23f296b3f83521577274e3d3b0f3d739bf2e68d01a752e4288bc50d286"}, + {file = "lxml-6.1.0-cp39-cp39-win32.whl", hash = "sha256:11a873c77a181b4fef9c2e357d08ed399542c2af1390101da66720a19c7c9618"}, + {file = "lxml-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:81ff55c70b67d19d52b6fd118a114c0a4c97d799cd3089ff9bd9e2ff4b414ee2"}, + {file = "lxml-6.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:481d6e2104285d9add34f41b42b247b76b61c5b5c26c303c2e9707bbf8bd9a64"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:546b66c0dd1bb8d9fa89d7123e5fa19a8aff3a1f2141eb22df96112afb17b842"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5cfa1a34df366d9dc0d5eaf420f4cf2bb1e1bebe1066d1c2fc28c179f8a4004c"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db88156fcf544cdbf0d95588051515cfdfd4c876fc66444eb98bceb5d6db76de"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07f98f5496f96bf724b1e3c933c107f0cbf2745db18c03d2e13a291c3afd2635"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4642e04449a1e164b5ff71ffd901ddb772dfabf5c9adf1b7be5dffe1212bc037"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7da13bb6fbadfafb474e0226a30570a3445cfd47c86296f2446dafbd77079ace"}, + {file = "lxml-6.1.0.tar.gz", hash = "sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13"}, ] [package.extras] @@ -1553,7 +1547,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1834,7 +1828,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -2032,7 +2026,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2044,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2060,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2348,7 +2342,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2480,7 +2474,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2505,7 +2499,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2533,7 +2527,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2938,7 +2932,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3138,7 +3132,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3154,7 +3148,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3220,7 +3214,6 @@ files = [ {file = "tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a"}, {file = "tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c"}, ] -markers = {main = "python_version < \"3.14\""} [[package]] name = "tornado" @@ -3229,7 +3222,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3361,7 +3354,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3622,7 +3615,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3643,7 +3636,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, From 4655b435ee6e83518cd9ea6941d403be1cdf3a63 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 20 May 2026 16:58:39 +0100 Subject: [PATCH 42/42] Follow #19468 (Fix sending heroes in Sliding Sync) with small tweaks (#19791) Follows: #19468 The main change is from this comment https://github.com/element-hq/synapse/pull/19468#discussion_r2810364196 I am pretty sure it's safe and was tempted to add it to that PR, but for easier bisection and reversion in case it goes wrong, thought a separate commit would be the best. The other drive-by change is a boolean logic simplification Simplify condition (boolean equivalence) Don't fetch name state from `meta_room_state` since it's no longer used there --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19791.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19791.bugfix diff --git a/changelog.d/19791.bugfix b/changelog.d/19791.bugfix new file mode 100644 index 0000000000..003716d296 --- /dev/null +++ b/changelog.d/19791.bugfix @@ -0,0 +1 @@ +Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index df1270f7f9..4d6287b147 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -966,7 +966,7 @@ class SlidingSyncHandler: # care to treat an empty string as unset). But we only need to get them # on initial syncs (or the first time we send down the room) or if the # membership has changed which may change the heroes. - if not room_name and (initial or (not initial and membership_changed)): + if not room_name and (initial or membership_changed): # We need the room summary to extract the heroes from if room_membership_for_user_at_to_token.membership != Membership.JOIN: # TODO: Figure out how to get the membership summary for left/banned rooms @@ -1185,8 +1185,6 @@ class SlidingSyncHandler: (EventTypes.Member, hero_user_id) for hero_user_id in hero_user_ids ] meta_room_state = list(hero_room_state) - if initial or name_changed: - meta_room_state.append((EventTypes.Name, "")) if initial or avatar_changed: meta_room_state.append((EventTypes.RoomAvatar, ""))