From 0e0bdd2973a7febd3545ea7b604ca24701bb80f9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 9 Sep 2026 00:07:10 +0100 Subject: [PATCH] Speed up by recursive relations by 10-100x by joining events inside the CTE (#20182) This solves the root cause of a user opening the thread panel in Element Web DoSing synapse with recursive relation requests, starving out delayed events and causing MatrixRTC calls to drop - see https://github.com/matrix-org/matrix-js-sdk/pull/5519 for papering over it clientside. Fixes https://github.com/element-hq/synapse/issues/18788 Claude rationale: Postgres cannot estimate the size of a recursive CTE. When it guesses large it stops probing events by event_id and instead hashes every event in the room, so a single recursive `/relations` request in a busy room takes seconds and a Threads-panel fan-out of 30 of them can pin a client-reader's DB pool for a minute. Joining events per recursion step keeps the lookups as index probes regardless of the estimate. The recursion also moves from `UNION` to `UNION ALL`. An event carries a single `m.relates_to` and is stored as exactly one `event_relations` row (unique index on `event_id`), so the relation graph is a tree: every node is reached along one path and `UNION` never had duplicates to remove. `UNION ALL` drops the sort-and-dedupe pass over the working table on every iteration, which matters more now that each row also carries the joined events columns. A cycle from bogus events is still terminated by the depth bound, as before, and `UNION` gave no protection there anyway since such rows differ in depth. Measured on Postgres 16 against a synthetic corpus modelled on a large homeserver: 4 rooms x 300k events; one 40-reply thread rooted 280k events back in the timeline, with 2 reactions per reply; every 5th event elsewhere a reaction, plus 200 popular roots with 2000 reactions each so that the `relates_to_id statistics` are skewed the way they are in production. Default limit (6 rows), warm cache, JIT off: ``` before after single request 77 ms 1.4 ms 20 concurrent requests 1.21 s 0.14 s ``` Before: Hash Join with a Hash over all 300k events of the room (4 batches). After: Nested Loop with an Index Scan on `events_event_id_key` per row. (found/solved by fable) --------- Co-authored-by: Eric Eastwood --- changelog.d/20182.bugfix | 1 + synapse/storage/databases/main/relations.py | 25 ++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20182.bugfix diff --git a/changelog.d/20182.bugfix b/changelog.d/20182.bugfix new file mode 100644 index 0000000000..637c3efb96 --- /dev/null +++ b/changelog.d/20182.bugfix @@ -0,0 +1 @@ +Fix slow recursive `/relations` requests in large rooms by joining events inside the recursive query. diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 9d9c37e2a4..58f9321628 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -237,19 +237,28 @@ class RelationsWorkerStore(SQLBaseStore): # If no recursion is needed then the event_relations table is queried # for direct children of the requested event. if recurse: + # The events table is joined inside the recursion rather than + # after it: Postgres cannot accurately estimate the size of a recursive CTE, + # and when the guess is large it joins the CTE against a scan of + # *every* event in the room, which takes seconds in busy rooms. + # Joining per step keeps every events lookup an index probe. sql = """ WITH RECURSIVE related_events AS ( - SELECT event_id, relation_type, relates_to_id, 0 AS depth - FROM event_relations - WHERE relates_to_id = ? - UNION SELECT e.event_id, e.relation_type, e.relates_to_id, depth + 1 - FROM event_relations e - INNER JOIN related_events r ON r.event_id = e.relates_to_id - WHERE depth <= 3 + SELECT er.event_id, er.relation_type, ev.room_id, ev.sender, + ev.topological_ordering, ev.stream_ordering, ev.type, 0 AS depth + FROM event_relations er + INNER JOIN events ev ON ev.event_id = er.event_id + WHERE er.relates_to_id = ? + UNION ALL + SELECT er.event_id, er.relation_type, ev.room_id, ev.sender, + ev.topological_ordering, ev.stream_ordering, ev.type, r.depth + 1 + FROM related_events r + INNER JOIN event_relations er ON er.relates_to_id = r.event_id + INNER JOIN events ev ON ev.event_id = er.event_id + WHERE r.depth <= 3 ) SELECT event_id, relation_type, sender, topological_ordering, stream_ordering FROM related_events - INNER JOIN events USING (event_id) WHERE %s ORDER BY topological_ordering %s, stream_ordering %s LIMIT ?;