diff --git a/changelog.d/20440.bugfix b/changelog.d/20440.bugfix
new file mode 100644
index 0000000000..77f6ad3e63
--- /dev/null
+++ b/changelog.d/20440.bugfix
@@ -0,0 +1 @@
+Return the event with an empty context, rather than a 500, when `GET /_matrix/client/v3/rooms/{roomId}/context/{eventId}` is called on an out-of-band membership event such as a remote invite or a knock.
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index e5de95a972..b2664a4f62 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -120,8 +120,8 @@ class EventContext:
     events_after: list[FilteredEvent]
     state: list[EventBase]
     aggregations: dict[str, BundledAggregations]
-    start: str
-    end: str
+    start: str | None
+    end: str | None
 
 
 class RoomCreationHandler:
@@ -2069,6 +2069,30 @@ class RoomContextHandler:
         if not filtered:
             raise AuthError(403, "You don't have permission to access that event.")
 
+        if (
+            event.internal_metadata.outlier
+            and event.type == EventTypes.Member
+            and event.state_key == user.to_string()
+        ):
+            # An out-of-band membership (a remote invite, a knock, or a leave that
+            # rejects either) is an outlier: it has no state group, so
+            # get_state_for_events below raises for it. Its stream position points into
+            # a room whose history we do not hold, so the surrounding events and
+            # pagination tokens derived from it would describe nothing. Return the
+            # event alone.
+            #
+            # Any other outlier still raises. Only the admin API can reach one, since
+            # the visibility filter drops them for everyone else.
+            return EventContext(
+                events_before=[],
+                event=filtered[0],
+                events_after=[],
+                state=[],
+                aggregations={},
+                start=None,
+                end=None,
+            )
+
         results = await self.store.get_events_around(
             room_id, event_id, before_limit, after_limit, event_filter
         )
diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py
index e47b6e9efe..da36c94a51 100644
--- a/synapse/rest/admin/rooms.py
+++ b/synapse/rest/admin/rooms.py
@@ -901,7 +901,7 @@ class RoomEventContextServlet(RestServlet):
             )
 
         time_now = self.clock.time_msec()
-        results = {
+        results: JsonDict = {
             "events_before": await self._event_serializer.serialize_events(
                 event_context.events_before,
                 time_now,
@@ -921,10 +921,15 @@ class RoomEventContextServlet(RestServlet):
                 [FilteredEvent.state(e) for e in event_context.state],
                 time_now,
             ),
-            "start": event_context.start,
-            "end": event_context.end,
         }
 
+        # Omitted when the handler has no token to give, as in
+        # RoomEventContextServlet.
+        if event_context.start is not None:
+            results["start"] = event_context.start
+        if event_context.end is not None:
+            results["end"] = event_context.end
+
         return HTTPStatus.OK, results
 
 
diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py
index b5dcc8902e..5af2a37257 100644
--- a/synapse/rest/client/room.py
+++ b/synapse/rest/client/room.py
@@ -1280,7 +1280,7 @@ class RoomEventContextServlet(RestServlet):
         serializer_options = await self._event_serializer.create_config(
             requester=requester
         )
-        results = {
+        results: JsonDict = {
             "events_before": await self._event_serializer.serialize_events(
                 event_context.events_before,
                 time_now,
@@ -1304,10 +1304,16 @@ class RoomEventContextServlet(RestServlet):
                 time_now,
                 config=serializer_options,
             ),
-            "start": event_context.start,
-            "end": event_context.end,
         }
 
+        # An out-of-band membership event cannot be placed in the room's stream,
+        # so the handler has no tokens to give for it. Every other response
+        # carries both.
+        if event_context.start is not None:
+            results["start"] = event_context.start
+        if event_context.end is not None:
+            results["end"] = event_context.end
+
         return 200, results