Add MSC4354 history visibility exemption over federation

rust/src/events/filter.rs

@@ -28,6 +28,7 @@ pub fn event_visible_to_server_py(
     sender: String,
     target_server_name: String,
     history_visibility: String,
+    sticky: bool,
     erased_senders: HashMap<String, bool>,
     partial_state_invisible: bool,
     memberships: Vec<(String, String)>, // (state_key, membership)
@@ -36,6 +37,7 @@ pub fn event_visible_to_server_py(
         sender,
         target_server_name,
         history_visibility,
+        sticky,
         erased_senders,
         partial_state_invisible,
         memberships,
@@ -44,19 +46,23 @@ pub fn event_visible_to_server_py(
 }
 
 /// Return whether the target server is allowed to see the event.
+/// The target server is assumed to currently be in the room.
 ///
 /// For a fully stated room, the target server is allowed to see an event E if:
 ///   - the state at E has world readable or shared history vis, OR
-///   - the state at E says that the target server is in the room.
+///   - the state at E says that the target server is in the room, OR
+///   - E is an unexpired sticky event.
 ///
 /// For a partially stated room, the target server is allowed to see E if:
 ///   - E was created by this homeserver, AND:
 ///       - the partial state at E has world readable or shared history vis, OR
-///       - the partial state at E says that the target server is in the room.
+///       - the partial state at E says that the target server is in the room, OR
+///       - E is an unexpired sticky event.
 pub fn event_visible_to_server(
     sender: String,
     target_server_name: String,
     history_visibility: String,
+    sticky: bool,
     erased_senders: HashMap<String, bool>,
     partial_state_invisible: bool,
     memberships: Vec<(String, String)>, // (state_key, membership)
@@ -71,6 +77,10 @@ pub fn event_visible_to_server(
         return Ok(false);
     }
 
+    if sticky {
+        return Ok(true);
+    }
+
     if history_visibility != HISTORY_VISIBILITY_INVITED
         && history_visibility != HISTORY_VISIBILITY_JOINED
     {

synapse/federation/sender/per_destination_queue.py

@@ -717,6 +717,7 @@ class PerDestinationQueue:
                     # kick should send it out if its online).
                     new_pdus = await filter_events_for_server(
                         self._storage_controllers,
+                        self._clock,
                         self._destination,
                         self.server_name,
                         new_pdus,
@@ -1111,6 +1112,8 @@ class StickyEventBacklogTracker:
 
         self._own_server_name = hs.hostname
 
+        self._clock = hs.get_clock()
+
         self._storage_controllers = hs.get_storage_controllers()
 
         self._store = hs.get_datastores().main
@@ -1174,6 +1177,7 @@ class StickyEventBacklogTracker:
         # Filter the sticky events
         sticky_events = await filter_events_for_server(
             self._storage_controllers,
+            self._clock,
             target_server_name=self._destination,
             local_server_name=self._own_server_name,
             events=sticky_events,

synapse/handlers/federation.py

@@ -468,6 +468,7 @@ class FederationHandler:
             # positives from users having been erased.
             filtered_extremities = await filter_events_for_server(
                 self._storage_controllers,
+                self.clock,
                 self.server_name,
                 self.server_name,
                 events_to_check,
@@ -1869,6 +1870,7 @@ class FederationHandler:
 
         events = await filter_events_for_server(
             self._storage_controllers,
+            self.clock,
             origin,
             self.server_name,
             events,
@@ -1904,6 +1906,7 @@ class FederationHandler:
 
         events = await filter_events_for_server(
             self._storage_controllers,
+            self.clock,
             origin,
             self.server_name,
             [event],
@@ -1938,6 +1941,7 @@ class FederationHandler:
 
         missing_events = await filter_events_for_server(
             self._storage_controllers,
+            self.clock,
             origin,
             self.server_name,
             missing_events,

synapse/synapse_rust/events.pyi

@@ -143,6 +143,7 @@ def event_visible_to_server(
     sender: str,
     target_server_name: str,
     history_visibility: str,
+    sticky: bool,
     erased_senders: Mapping[str, bool],
     partial_state_invisible: bool,
     memberships: list[tuple[str, str]],
@@ -153,6 +154,9 @@ def event_visible_to_server(
         sender: The sender of the event.
         target_server_name: The server we want to send the event to.
         history_visibility: The history_visibility value at the event.
+        sticky: Whether the event is an unexpired MSC4354 sticky event that is still sticky.
+            If True, the history visibility check is skipped.
+            Erasure and partial state checks still apply.
         erased_senders: A mapping of users and whether they have requested erasure. If a
             user is not in the map, it is treated as though they haven't requested erasure.
         partial_state_invisible: Whether the event should be treated as invisible due to

synapse/visibility.py

@@ -456,8 +456,8 @@ def _check_client_allowed_to_see_event(
         # > — https://github.com/matrix-org/matrix-spec-proposals/blob/3635b765267452eb452cd2ced963aee52d1fd185/proposals/4354-sticky-events.md#L101-L102
         #
         # > [^hisvis]: This ensures that newly joined servers can see sticky events sent from before they were joined to the room, regardless
-        # > of the history visibility setting. This matches the behaviour of state events.
-        # > — https://github.com/matrix-org/matrix-spec-proposals/blob/3635b765267452eb452cd2ced963aee52d1fd185/proposals/4354-sticky-events.md#L501-L502
+        # > of the history visibility setting.
+        # > — https://github.com/matrix-org/matrix-spec-proposals/blob/eec798a3ea2f37fb3ea3fb9fcd85954b7486fc0f/proposals/4354-sticky-events.md#L638
         sticky_until_ts = event.locally_sticky_until_ts()
         if (
             sticky_until_ts is not None
@@ -690,6 +690,7 @@ def get_effective_room_visibility_from_state(state: StateMap[EventBase]) -> str:
 
 async def filter_events_for_server(
     storage: StorageControllers,
+    clock: Clock,
     target_server_name: str,
     local_server_name: str,
     events: Sequence[EventBase],
@@ -703,6 +704,7 @@ async def filter_events_for_server(
 
     Args:
         storage
+        clock
         target_server_name
         local_server_name
         events
@@ -759,12 +761,32 @@ async def filter_events_for_server(
         target_server_name,
     )
 
+    now_ms = clock.time_msec()
+
     to_return = []
     for e in events:
+        # For sticky events, apply an exemption from the membership test part of history visibility rules,
+        # as long as the target server is currently joined to the room.
+        # Essentially, this alters `invited`/`joined` history visibility rules to `shared` for sticky events.
+        #
+        # > History visibility **checks** MUST NOT be applied to sticky events. This applies to all endpoints where the sticky events could be returned.
+        # > Any joined user or server is authorised to see sticky events for the duration they remain sticky.[^hisvis]
+        # > — https://github.com/matrix-org/matrix-spec-proposals/blob/3635b765267452eb452cd2ced963aee52d1fd185/proposals/4354-sticky-events.md#L101-L102
+        #
+        # > [^hisvis]: This ensures that newly joined servers can see sticky events sent from before they were joined to the room, regardless
+        # > of the history visibility setting.
+        # > — https://github.com/matrix-org/matrix-spec-proposals/blob/eec798a3ea2f37fb3ea3fb9fcd85954b7486fc0f/proposals/4354-sticky-events.md#L638
+        sticky_until_ts = e.locally_sticky_until_ts()
+        is_sticky = (
+            storage.main.hs.config.experimental.msc4354_enabled
+            and sticky_until_ts is not None
+            and now_ms < sticky_until_ts
+        )
         if event_visible_to_server(
             sender=e.sender,
             target_server_name=target_server_name,
             history_visibility=event_to_history_vis[e.event_id],
+            sticky=is_sticky,
             erased_senders=erased_senders,
             partial_state_invisible=e.event_id in partial_state_invisible_event_ids,
             memberships=list(event_to_memberships.get(e.event_id, {}).values()),

tests/test_visibility.py

@@ -18,19 +18,21 @@
 #
 #
 import logging
+import sqlite3
 from unittest.mock import patch
 
 from twisted.test.proto_helpers import MemoryReactor
 
-from synapse.api.constants import AccountDataTypes
+from synapse.api.constants import AccountDataTypes, EventTypes
 from synapse.api.room_versions import RoomVersions
 from synapse.events import EventBase, make_event_from_dict
 from synapse.events.snapshot import EventContext
 from synapse.rest import admin
 from synapse.rest.client import login, room
 from synapse.server import HomeServer
-from synapse.types import create_requester
+from synapse.types import JsonDict, create_requester
 from synapse.util.clock import Clock
+from synapse.util.duration import Duration
 from synapse.visibility import (
     filter_and_transform_events_for_client,
     filter_events_for_server,
@@ -39,7 +41,7 @@ from synapse.visibility import (
 from tests import unittest
 from tests.test_utils.event_injection import inject_event, inject_member_event
 from tests.unittest import HomeserverTestCase
-from tests.utils import create_room
+from tests.utils import USE_POSTGRES_FOR_TESTS, create_room
 
 logger = logging.getLogger(__name__)
 
@@ -97,6 +99,7 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
         filtered = self.get_success(
             filter_events_for_server(
                 self._storage_controllers,
+                self.clock,
                 "test_server",
                 "hs",
                 events_to_filter,
@@ -136,6 +139,7 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
             self.get_success(
                 filter_events_for_server(
                     self._storage_controllers,
+                    self.clock,
                     "remote_hs",
                     "hs",
                     [outlier],
@@ -155,6 +159,7 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
         filtered = self.get_success(
             filter_events_for_server(
                 self._storage_controllers,
+                self.clock,
                 "remote_hs",
                 "local_hs",
                 [outlier, evt],
@@ -173,6 +178,7 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
         filtered = self.get_success(
             filter_events_for_server(
                 self._storage_controllers,
+                self.clock,
                 "other_server",
                 "local_hs",
                 [outlier, evt],
@@ -229,6 +235,7 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
         filtered = self.get_success(
             filter_events_for_server(
                 self._storage_controllers,
+                self.clock,
                 "test_server",
                 "local_hs",
                 events_to_filter,
@@ -277,6 +284,95 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
         return event
 
 
+class FilterEventsForServerStickyEventsTestCase(HomeserverTestCase):
+    """
+    Tests that MSC4354 Sticky Events are exempt from history visibility checks
+    when filtering events for remote servers.
+    """
+
+    if not USE_POSTGRES_FOR_TESTS and sqlite3.sqlite_version_info < (3, 40, 0):
+        # We need the JSON functionality in SQLite
+        skip = f"SQLite version is too old to support sticky events: {sqlite3.sqlite_version_info} (See https://github.com/element-hq/synapse/issues/19428)"
+
+    servlets = [
+        admin.register_servlets,
+        login.register_servlets,
+        room.register_servlets,
+    ]
+
+    def default_config(self) -> JsonDict:
+        config = super().default_config()
+        config["experimental_features"] = {"msc4354_enabled": True}
+        return config
+
+    def test_joined_history_visibility_sticky_events_visible_to_newly_joined_server(
+        self,
+    ) -> None:
+        """
+        Tests that, in a joined history visibility room, a newly-joined server
+        can see sticky events from before it joined, but not regular events.
+        Once the sticky events expire, the exemption no longer applies.
+        """
+        self.register_user("resident", "password")
+        tok = self.login("resident", "password")
+        room_id = self.helper.create_room_as("resident", tok=tok)
+        self.helper.send_state(
+            room_id,
+            "m.room.history_visibility",
+            body={"history_visibility": "joined"},
+            tok=tok,
+        )
+
+        # Send 3 timeline events: regular, sticky, regular
+        # (We have this 'sandwich' to prove that history visibility filtering is working)
+        regular_1_event_id = self.helper.send(room_id, body="regular 1", tok=tok)[
+            "event_id"
+        ]
+        sticky_event_id = self.helper.send_sticky_event(
+            room_id,
+            type=EventTypes.Message,
+            content={"body": "sticky", "msgtype": "m.text"},
+            duration=Duration(hours=1),
+            tok=tok,
+        )["event_id"]
+        regular_2_event_id = self.helper.send(room_id, body="regular 2", tok=tok)[
+            "event_id"
+        ]
+
+        # Now a user on a remote server joins
+        self.get_success(
+            inject_member_event(self.hs, room_id, "@joiner:remote", "join")
+        )
+
+        events = self.get_success(
+            self.hs.get_datastores().main.get_events_as_list(
+                [regular_1_event_id, sticky_event_id, regular_2_event_id]
+            )
+        )
+
+        def filter_for_remote() -> list[str]:
+            filtered = self.get_success(
+                filter_events_for_server(
+                    self.hs.get_storage_controllers(),
+                    self.clock,
+                    "remote",
+                    "test",
+                    events,
+                    redact=False,
+                    filter_out_erased_senders=True,
+                    filter_out_remote_partial_state_events=True,
+                )
+            )
+            return [e.event_id for e in filtered]
+
+        # The remote server can only see the sticky event
+        self.assertEqual(filter_for_remote(), [sticky_event_id])
+
+        # Once the sticky event expires, the remote server can't see it anymore
+        self.reactor.advance(Duration(hours=1).as_secs())
+        self.assertEqual(filter_for_remote(), [])
+
+
 class FilterEventsForServerAdminsTestCase(HomeserverTestCase):
     servlets = [
         admin.register_servlets,