From abf5a155ba6d5baafa9c7a8297d967e6b32d842f Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 9 May 2026 22:32:52 +0100 Subject: [PATCH 01/81] feat: Add backfill_missing_events helper --- .../fetch_and_handle_outliers.rs | 531 +++++++++++------- .../rooms/event_handler/parse_incoming_pdu.rs | 5 +- 2 files changed, 335 insertions(+), 201 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index e465131fb..0576d994a 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -1,233 +1,364 @@ use std::{ - collections::{BTreeMap, HashSet, VecDeque, hash_map}, + cmp::min, + collections::{BTreeMap, HashMap, HashSet, VecDeque, hash_map}, time::Instant, }; use conduwuit::{ - Event, PduEvent, debug, debug_warn, implement, matrix::event::gen_event_id_canonical_json, - trace, utils::continue_exponential_backoff_secs, warn, + Err, Event, PduEvent, debug, debug_info, debug_warn, err, + matrix::event::gen_event_id_canonical_json, trace, utils::continue_exponential_backoff_secs, + warn, }; +use futures::StreamExt; use ruma::{ - CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, - api::federation::event::get_event, + CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, RoomId, ServerName, + api::federation::event::{get_event, get_missing_events}, }; use super::get_room_version_rules; -/// Find the event and auth it. Once the event is validated (steps 1 - 8) -/// it is appended to the outliers Tree. -/// -/// Returns pdu and if we fetched it over federation the raw json. -/// -/// a. Look in the main timeline (pduid_pdu tree) -/// b. Look at outlier pdu tree -/// c. Ask origin server over federation -/// d. TODO: Ask other servers over federation? -#[implement(super::Service)] -pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>( - &self, - origin: &'a ServerName, - events: Events, - create_event: &'a Pdu, - room_id: &'a RoomId, -) -> Vec<(PduEvent, Option>)> -where - Pdu: Event + Send + Sync, - Events: Iterator + Clone + Send, -{ - let back_off = |id| match self - .services - .globals - .bad_event_ratelimiter - .write() - .entry(id) - { - | hash_map::Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - | hash_map::Entry::Occupied(mut e) => { - *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); - }, - }; - - let mut events_with_auth_events = Vec::with_capacity(events.clone().count()); - trace!("Fetching {} outlier pdus", events.clone().count()); - - for id in events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { - trace!("Found {id} in main timeline or outlier tree"); - events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![])); - continue; +impl super::Service { + /// Uses `/_matrix/federation/v1/get_missing_events` to fill gaps in the + /// DAG. + /// + /// When this function is called, the "earliest events" (current forward + /// extremities) will be collected, and the function will loop with an + /// exponentially incrementing limit (up to 100 per request) until it has + /// filled the gap, i.e. when the remote says there's no more events. + /// + /// This function does not persist the events. The caller is responsible for + /// passing them through handle_incoming_pdu. + pub(super) async fn backfill_missing_events( + &self, + room_id: OwnedRoomId, + latest_events: Vec, + via: OwnedServerName, + ) -> conduwuit::Result> { + if latest_events.is_empty() { + return Ok(HashMap::new()); } + let earliest_events = self + .services + .state + .get_forward_extremities(&room_id) + .collect::>() + .await; + // TODO: min_depth is probably necessary to avoid fetching the entire room + // history if there are very long gaps + let mut latest_events = latest_events; + let mut loop_count: u64 = 3; + // Start with a base number of 3 so that we fetch 10, 16, 25, 36, etc + // instead of 1, 2, 4, 9, so on. + let mut backfilled_events = HashMap::with_capacity(10); - // c. Ask origin server over federation - // We also handle its auth chain here so we don't get a stack overflow in - // handle_outlier_pdu. - let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into(); - let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); - - let mut events_all = HashSet::with_capacity(todo_auth_events.len()); - while let Some(next_id) = todo_auth_events.pop_front() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .get(&*next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 60 * 2; - const MAX_DURATION: u64 = 60 * 60 * 8; - if continue_exponential_backoff_secs( - MIN_DURATION, - MAX_DURATION, - time.elapsed(), - *tries, - ) { - debug_warn!( - tried = ?*tries, - elapsed = ?time.elapsed(), - "Backing off from {next_id}", - ); - continue; - } + while !latest_events.is_empty() { + let mut request = get_missing_events::v1::Request::new( + room_id.clone(), + earliest_events.clone(), + latest_events.clone(), + ); + request.limit = min(loop_count.saturating_pow(2), 100) + .try_into() + .expect("limit cannot be greater than 100, which fits into UInt"); + if backfilled_events.len() > 1000 { + warn!( + "Received {} missing events, refusing to fetch more (infinite loop?)", + backfilled_events.len() + ); + break; } - if events_all.contains(&next_id) { - continue; - } - - if self.services.timeline.pdu_exists(&next_id).await { - trace!("Found {next_id} in db"); - continue; - } - - debug!("Fetching {next_id} over federation from {origin}."); - match self + debug_info!( + backfilled=%backfilled_events.len(), + %loop_count, + "Asking {via} for up to {} missing events", + request.limit + ); + trace!( + ?latest_events, + ?earliest_events, + %via, + limit=%request.limit, + "Requesting missing events" + ); + let response: get_missing_events::v1::Response = self .services .sending - .send_federation_request( - origin, - get_event::v1::Request::new((*next_id).to_owned()), - ) - .await - { - | Ok(res) => { - debug!("Got {next_id} over federation from {origin}"); - let Ok(room_version_rules) = get_room_version_rules(create_event) else { - back_off((*next_id).to_owned()); - continue; - }; + .send_federation_request(&via, request) + .await?; + loop_count = loop_count.saturating_add(1); - let Ok((calculated_event_id, value)) = - gen_event_id_canonical_json(&res.pdu, &room_version_rules) - else { - back_off((*next_id).to_owned()); - continue; - }; - - if calculated_event_id != *next_id { - warn!( - "Server didn't return event id we requested: requested: {next_id}, \ - we got {calculated_event_id}. Event: {:?}", - &res.pdu - ); - } - - if let Some(auth_events) = value - .get("auth_events") - .and_then(CanonicalJsonValue::as_array) - { - for auth_event in auth_events { - match serde_json::from_value::( - auth_event.clone().into(), - ) { - | Ok(auth_event) => { - trace!( - "Found auth event id {auth_event} for event {next_id}" - ); - todo_auth_events.push_back(auth_event); - }, - | _ => { - warn!("Auth event id is not valid"); - }, - } - } - } else { - warn!("Auth event list invalid"); - } - - events_in_reverse_order.push((next_id.clone(), value)); - events_all.insert(next_id); - }, - | Err(e) => { - warn!("Failed to fetch auth event {next_id} from {origin}: {e}"); - back_off((*next_id).to_owned()); - }, + // Some buggy servers (including old continuwuity) may return the same events + // multiple times, which can cause this to be an infinite loop. + // In order to break this loop, if we see no new events from this response (i.e. + // all events in the response are already in backfilled_events), we stop, + // with a warning. + let mut unseen: usize = 0; + let chunk_len = response.events.len(); + if response.events.is_empty() { + debug_info!("No more missing events found"); + break; } + for event in response.events { + let (incoming_room_id, event_id, pdu_json) = + self.parse_incoming_pdu(&event).await.map_err(|e| { + err!(BadServerResponse("{via} returned an invalid event: {e:?}")) + })?; + if incoming_room_id != room_id { + return Err!(BadServerResponse( + "{via} returned {event_id} in missing events which belongs to \ + {incoming_room_id}, not {room_id}" + )); + } + if backfilled_events.contains_key(&event_id) { + debug_warn!(%via, %event_id, "Remote retransmitted event"); + continue; + } + // TODO: Should this be scoped to the GME session? We might end up incorrectly + // assuming we've caught up if we do this + if self.services.timeline.pdu_exists(&event_id).await { + debug!(%via, %event_id, "Already seen event in database"); + continue; + } + unseen = unseen.saturating_add(1); + let parsed = PduEvent::from_id_val(&event_id, pdu_json) + .map_err(|e| err!(BadServerResponse("Unable to parse {event_id}: {e}")))?; + backfilled_events.insert(event_id, parsed.clone()); + for prev_event_id in parsed.prev_events() { + if backfilled_events.contains_key(prev_event_id) + || self.services.timeline.pdu_exists(prev_event_id).await + { + continue; + } + latest_events.push(prev_event_id.to_owned()); + } + } + latest_events.retain(|e| !backfilled_events.contains_key(e)); + debug!( + chunk=%chunk_len, + remaining=%latest_events.len(), + "Got missing events" + ); } - events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order)); + Ok(backfilled_events) } - let mut pdus = Vec::with_capacity(events_with_auth_events.len()); - for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Some(local_pdu) = local_pdu { - trace!("Found {id} in main timeline or outlier tree"); - pdus.push((local_pdu.clone(), None)); - } + /// Find the event and auth it. Once the event is validated (steps 1 - 8) + /// it is appended to the outliers Tree. + /// + /// Returns pdu and if we fetched it over federation the raw json. + /// + /// a. Look in the main timeline (pduid_pdu tree) + /// b. Look at outlier pdu tree + /// c. Ask origin server over federation + /// d. TODO: Ask other servers over federation? + pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>( + &self, + origin: &'a ServerName, + events: Events, + create_event: &'a Pdu, + room_id: &'a RoomId, + ) -> Vec<(PduEvent, Option>)> + where + Pdu: Event + Send + Sync, + Events: Iterator + Clone + Send, + { + let back_off = |id| match self + .services + .globals + .bad_event_ratelimiter + .write() + .entry(id) + { + | hash_map::Entry::Vacant(e) => { + e.insert((Instant::now(), 1)); + }, + | hash_map::Entry::Occupied(mut e) => { + *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); + }, + }; - for (next_id, value) in events_in_reverse_order.into_iter().rev() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .get(&*next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs( - MIN_DURATION, - MAX_DURATION, - time.elapsed(), - *tries, - ) { - debug!("Backing off from {next_id}"); + let mut events_with_auth_events = Vec::with_capacity(events.clone().count()); + trace!("Fetching {} outlier pdus", events.clone().count()); + + for id in events { + // a. Look in the main timeline (pduid_pdu tree) + // b. Look at outlier pdu tree + // (get_pdu_json checks both) + if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { + trace!("Found {id} in main timeline or outlier tree"); + events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![])); + continue; + } + + // c. Ask origin server over federation + // We also handle its auth chain here so we don't get a stack overflow in + // handle_outlier_pdu. + let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into(); + let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); + + let mut events_all = HashSet::with_capacity(todo_auth_events.len()); + while let Some(next_id) = todo_auth_events.pop_front() { + if let Some((time, tries)) = self + .services + .globals + .bad_event_ratelimiter + .read() + .get(&*next_id) + { + // Exponential backoff + const MIN_DURATION: u64 = 60 * 2; + const MAX_DURATION: u64 = 60 * 60 * 8; + if continue_exponential_backoff_secs( + MIN_DURATION, + MAX_DURATION, + time.elapsed(), + *tries, + ) { + debug_warn!( + tried = ?*tries, + elapsed = ?time.elapsed(), + "Backing off from {next_id}", + ); + continue; + } + } + + if events_all.contains(&next_id) { continue; } + + if self.services.timeline.pdu_exists(&next_id).await { + trace!("Found {next_id} in db"); + continue; + } + + debug!("Fetching {next_id} over federation from {origin}."); + match self + .services + .sending + .send_federation_request( + origin, + get_event::v1::Request::new((*next_id).to_owned()), + ) + .await + { + | Ok(res) => { + debug!("Got {next_id} over federation from {origin}"); + let Ok(room_version_rules) = get_room_version_rules(create_event) else { + back_off((*next_id).to_owned()); + continue; + }; + + let Ok((calculated_event_id, value)) = + gen_event_id_canonical_json(&res.pdu, &room_version_rules) + else { + back_off((*next_id).to_owned()); + continue; + }; + + if calculated_event_id != *next_id { + warn!( + "Server didn't return event id we requested: requested: \ + {next_id}, we got {calculated_event_id}. Event: {:?}", + &res.pdu + ); + } + + if let Some(auth_events) = value + .get("auth_events") + .and_then(CanonicalJsonValue::as_array) + { + for auth_event in auth_events { + match serde_json::from_value::( + auth_event.clone().into(), + ) { + | Ok(auth_event) => { + trace!( + "Found auth event id {auth_event} for event \ + {next_id}" + ); + todo_auth_events.push_back(auth_event); + }, + | _ => { + warn!("Auth event id is not valid"); + }, + } + } + } else { + warn!("Auth event list invalid"); + } + + events_in_reverse_order.push((next_id.clone(), value)); + events_all.insert(next_id); + }, + | Err(e) => { + warn!("Failed to fetch auth event {next_id} from {origin}: {e}"); + back_off((*next_id).to_owned()); + }, + } } - trace!("Handling outlier {next_id}"); - match Box::pin(self.handle_outlier_pdu( - origin, - create_event, - &next_id, - room_id, - value.clone(), - true, - )) - .await - { - | Ok((pdu, json)) => - if next_id == *id { - trace!("Handled outlier {next_id} (original request)"); - pdus.push((pdu, Some(json))); + events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order)); + } + + let mut pdus = Vec::with_capacity(events_with_auth_events.len()); + for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { + // a. Look in the main timeline (pduid_pdu tree) + // b. Look at outlier pdu tree + // (get_pdu_json checks both) + if let Some(local_pdu) = local_pdu { + trace!("Found {id} in main timeline or outlier tree"); + pdus.push((local_pdu.clone(), None)); + } + + for (next_id, value) in events_in_reverse_order.into_iter().rev() { + if let Some((time, tries)) = self + .services + .globals + .bad_event_ratelimiter + .read() + .get(&*next_id) + { + // Exponential backoff + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs( + MIN_DURATION, + MAX_DURATION, + time.elapsed(), + *tries, + ) { + debug!("Backing off from {next_id}"); + continue; + } + } + + trace!("Handling outlier {next_id}"); + match Box::pin(self.handle_outlier_pdu( + origin, + create_event, + &next_id, + room_id, + value.clone(), + true, + )) + .await + { + | Ok((pdu, json)) => + if next_id == *id { + trace!("Handled outlier {next_id} (original request)"); + pdus.push((pdu, Some(json))); + }, + | Err(e) => { + warn!("Authentication of event {next_id} failed: {e:?}"); + back_off(next_id); }, - | Err(e) => { - warn!("Authentication of event {next_id} failed: {e:?}"); - back_off(next_id); - }, + } } } + trace!("Fetched and handled {} outlier pdus", pdus.len()); + pdus } - trace!("Fetched and handled {} outlier pdus", pdus.len()); - pdus } diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index b4af6b4a8..34a33d83d 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -56,7 +56,10 @@ fn extract_room_id(event_type: &str, pdu: &CanonicalJsonObject) -> Result Result> { +pub(super) fn expect_event_id_array( + value: &CanonicalJsonObject, + field: &str, +) -> Result> { value .get(field) .ok_or_else(|| err!(Request(BadJson("missing field `{field}` on PDU"))))? From ba9dc27773789e5b1777d1db8e395a7ac98832d1 Mon Sep 17 00:00:00 2001 From: timedout Date: Sun, 10 May 2026 01:36:15 +0100 Subject: [PATCH 02/81] feat: Better prev event fetching fix: Don't panic in debug mode when making an empty notary query --- src/api/server/send.rs | 1 + .../fetch_and_handle_outliers.rs | 158 +++++++++--- src/service/rooms/event_handler/fetch_prev.rs | 228 +++++++++--------- .../rooms/event_handler/fetch_state.rs | 2 - .../event_handler/handle_incoming_pdu.rs | 67 +---- .../rooms/event_handler/handle_outlier_pdu.rs | 85 ++++--- .../rooms/event_handler/handle_prev_pdu.rs | 89 ------- src/service/rooms/event_handler/mod.rs | 3 +- .../event_handler/upgrade_outlier_pdu.rs | 2 + src/service/server_keys/request.rs | 5 +- 10 files changed, 307 insertions(+), 333 deletions(-) delete mode 100644 src/service/rooms/event_handler/handle_prev_pdu.rs diff --git a/src/api/server/send.rs b/src/api/server/send.rs index b766a2c69..6f6d35d10 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -381,6 +381,7 @@ async fn handle_room( .rooms .event_handler .handle_incoming_pdu(origin, room_id, &event_id, value, true) + .boxed() .await .map(|_| ()); results.push((event_id, result)); diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 0576d994a..6f1262ffd 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -6,17 +6,87 @@ use conduwuit::{ Err, Event, PduEvent, debug, debug_info, debug_warn, err, - matrix::event::gen_event_id_canonical_json, trace, utils::continue_exponential_backoff_secs, - warn, + matrix::event::gen_event_id_canonical_json, state_res::lexicographical_topological_sort, + trace, utils::continue_exponential_backoff_secs, warn, }; use futures::StreamExt; use ruma::{ - CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, RoomId, ServerName, + CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, + OwnedRoomId, OwnedServerName, RoomId, ServerName, UInt, api::federation::event::{get_event, get_missing_events}, + int, }; use super::get_room_version_rules; +/// Attempts to build a localised directed acyclic graph out of the given PDUs, +/// returning them in a topologically sorted order. +/// +/// This is used to attempt to process PDUs in an order that respects their +/// dependencies, however it is ultimately the sender's responsibility to send +/// them in a processable order, so this is just a best effort attempt. It does +/// not account for power levels or other tie breaks. +pub async fn build_local_dag( + pdu_map: &HashMap, +) -> conduwuit::Result> { + debug_assert!(pdu_map.len() >= 2, "needless call to build_local_dag with less than 2 PDUs"); + let mut dag: HashMap> = + HashMap::with_capacity(pdu_map.len()); + let mut id_origin_ts: HashMap = HashMap::with_capacity(pdu_map.len()); + + for (event_id, value) in pdu_map { + // We already checked that these properties are correct in parse_incoming_pdu, + // so it's safe to unwrap here. + // We also filter to remove any prev_events that are not in this pdu_map, as we + // need to have at least one event with zero out degrees for the lexico-topo + // sort below. If there are multiple events with omitted prevs, they will be + // ordered by timestamp, then event ID. At that point though, it's unlikely to + // matter. + let prev_events = value + .get("prev_events") + .unwrap() + .as_array() + .unwrap() + .iter() + .map(|v| EventId::parse(v.as_str().unwrap()).unwrap()) + .filter(|id| pdu_map.contains_key(id)) + .collect(); + + dag.insert(event_id.clone(), prev_events); + let origin_server_ts = value + .get("origin_server_ts") + .and_then(CanonicalJsonValue::as_integer) + .unwrap_or_default(); + id_origin_ts.insert(event_id.clone(), origin_server_ts); + } + + debug!(count = dag.len(), "Sorting incoming events with partial graph"); + lexicographical_topological_sort(&dag, &async |node_id| { + // Note: we don't bother fetching power levels because that would massively slow + // this function down. This is a best-effort attempt to order events correctly + // for processing, however ultimately that should be the sender's job. + let ts = id_origin_ts + .get(&node_id) + .copied() + .unwrap_or_else(|| int!(0)) + .to_string() + .parse::() + .ok() + .and_then(UInt::new) + .unwrap_or_default(); + Ok((int!(0), MilliSecondsSinceUnixEpoch(ts))) + }) + .await + .inspect(|sorted| { + debug_assert_eq!( + sorted.len(), + pdu_map.len(), + "Sorted graph was not the same size as the input graph" + ); + }) + .map_err(|e| err!("failed to resolve local graph: {e}")) +} + impl super::Service { /// Uses `/_matrix/federation/v1/get_missing_events` to fill gaps in the /// DAG. @@ -31,13 +101,13 @@ impl super::Service { pub(super) async fn backfill_missing_events( &self, room_id: OwnedRoomId, - latest_events: Vec, + head: Vec, via: OwnedServerName, ) -> conduwuit::Result> { - if latest_events.is_empty() { + if head.is_empty() { return Ok(HashMap::new()); } - let earliest_events = self + let tail = self .services .state .get_forward_extremities(&room_id) @@ -45,40 +115,31 @@ pub(super) async fn backfill_missing_events( .await; // TODO: min_depth is probably necessary to avoid fetching the entire room // history if there are very long gaps - let mut latest_events = latest_events; + let mut latest_events: HashSet = HashSet::from_iter(head.clone()); let mut loop_count: u64 = 3; // Start with a base number of 3 so that we fetch 10, 16, 25, 36, etc // instead of 1, 2, 4, 9, so on. let mut backfilled_events = HashMap::with_capacity(10); while !latest_events.is_empty() { - let mut request = get_missing_events::v1::Request::new( - room_id.clone(), - earliest_events.clone(), - latest_events.clone(), - ); - request.limit = min(loop_count.saturating_pow(2), 100) + let todo: Vec = latest_events.clone().into_iter().collect(); + let mut request = + get_missing_events::v1::Request::new(room_id.clone(), tail.clone(), todo.clone()); + let limit = min(loop_count.saturating_pow(2), 100); + request.limit = limit .try_into() .expect("limit cannot be greater than 100, which fits into UInt"); - if backfilled_events.len() > 1000 { - warn!( - "Received {} missing events, refusing to fetch more (infinite loop?)", - backfilled_events.len() - ); - break; - } debug_info!( backfilled=%backfilled_events.len(), %loop_count, - "Asking {via} for up to {} missing events", - request.limit + "Asking {via} for up to {limit} missing events", ); trace!( ?latest_events, - ?earliest_events, + ?tail, %via, - limit=%request.limit, + %limit, "Requesting missing events" ); let response: get_missing_events::v1::Response = self @@ -100,47 +161,76 @@ pub(super) async fn backfill_missing_events( break; } for event in response.events { + trace!("Parsing incoming event from backfill"); let (incoming_room_id, event_id, pdu_json) = self.parse_incoming_pdu(&event).await.map_err(|e| { err!(BadServerResponse("{via} returned an invalid event: {e:?}")) })?; + trace!(%incoming_room_id, %event_id, "Parsed incoming event from backfill"); if incoming_room_id != room_id { return Err!(BadServerResponse( "{via} returned {event_id} in missing events which belongs to \ {incoming_room_id}, not {room_id}" )); } + latest_events.remove(&event_id); + if head.contains(&event_id) || tail.contains(&event_id) { + debug!("Skipping known event {event_id}"); + continue; + } if backfilled_events.contains_key(&event_id) { debug_warn!(%via, %event_id, "Remote retransmitted event"); continue; } // TODO: Should this be scoped to the GME session? We might end up incorrectly // assuming we've caught up if we do this - if self.services.timeline.pdu_exists(&event_id).await { + if let Ok(pdu) = self.services.timeline.get_pdu(&event_id).await { debug!(%via, %event_id, "Already seen event in database"); - continue; + backfilled_events.insert(event_id.clone(), pdu); + } else { + unseen = unseen.saturating_add(1); } - unseen = unseen.saturating_add(1); let parsed = PduEvent::from_id_val(&event_id, pdu_json) .map_err(|e| err!(BadServerResponse("Unable to parse {event_id}: {e}")))?; - backfilled_events.insert(event_id, parsed.clone()); for prev_event_id in parsed.prev_events() { - if backfilled_events.contains_key(prev_event_id) - || self.services.timeline.pdu_exists(prev_event_id).await + // Verify that we have all of this event's prev_events. If we don't, add it to + // the work queue + if !(backfilled_events.contains_key(prev_event_id) + || self.services.timeline.pdu_exists(prev_event_id).await) { - continue; + latest_events.insert(prev_event_id.to_owned()); + break; } - latest_events.push(prev_event_id.to_owned()); + continue; + } + backfilled_events.insert(event_id.clone(), parsed); + } + for event_id in todo { + latest_events.remove(&event_id); + if let hash_map::Entry::Vacant(e) = backfilled_events.entry(event_id.clone()) { + let evt = self.services.timeline.get_pdu(&event_id).await?; + e.insert(evt); } } - latest_events.retain(|e| !backfilled_events.contains_key(e)); debug!( - chunk=%chunk_len, + count=%chunk_len, + new=%unseen, remaining=%latest_events.len(), "Got missing events" ); + if unseen == 0 { + debug_warn!("Didn't see any new events, breaking cycle"); + break; + } else if chunk_len < usize::try_from(limit)? { + debug!( + "Got less than the limit number of events, assuming there's no more to fetch" + ); + break; + } } + debug_info!("Successfully fetched {} missing events from {via}", backfilled_events.len()); + trace!("Missing_events: {backfilled_events:?}"); Ok(backfilled_events) } diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index efc7a434d..270b3d0d7 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,128 +1,118 @@ -use std::{ - collections::{BTreeMap, HashMap, HashSet, VecDeque}, - iter::once, -}; +use std::collections::{HashMap, HashSet, VecDeque}; -use conduwuit::{ - Event, PduEvent, Result, debug_warn, err, implement, - state_res::{self}, -}; -use futures::{FutureExt, future}; -use ruma::{ - CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName, - int, uint, -}; +use conduwuit::{Event, PduEvent, debug, debug_info, error, trace}; +use ruma::{OwnedEventId, RoomId, ServerName}; -use super::check_room_id; +use crate::rooms::event_handler::build_local_dag; -#[implement(super::Service)] -#[tracing::instrument( - level = "debug", - skip_all, - fields(%origin), -)] -#[allow(clippy::type_complexity)] -pub(super) async fn fetch_prev<'a, Pdu, Events>( - &self, - origin: &ServerName, - create_event: &Pdu, - room_id: &RoomId, - first_ts_in_room: MilliSecondsSinceUnixEpoch, - initial_set: Events, -) -> Result<( - Vec, - HashMap)>, -)> -where - Pdu: Event + Send + Sync, - Events: Iterator + Clone + Send, -{ - let num_ids = initial_set.clone().count(); - let mut eventid_info = HashMap::new(); - let mut graph: HashMap = HashMap::with_capacity(num_ids); - let mut todo_outlier_stack: VecDeque = - initial_set.map(ToOwned::to_owned).collect(); +impl super::Service { + pub(super) async fn fetch_prevs( + &self, + room_id: &RoomId, + create_event: &PduEvent, + incoming_pdu: &PduEvent, + origin: &ServerName, + ) -> conduwuit::Result<()> { + let mut queue: VecDeque = VecDeque::new(); + queue.push_back(incoming_pdu.event_id().to_owned()); - let mut amount = 0; - - while let Some(prev_event_id) = todo_outlier_stack.pop_front() { - self.services.server.check_running()?; - - match self - .fetch_and_handle_outliers( - origin, - once(prev_event_id.as_ref()), - create_event, - room_id, - ) - .boxed() - .await - .pop() - { - | Some((pdu, mut json_opt)) => { - check_room_id(room_id, &pdu)?; - - let limit = self.services.server.config.max_fetch_prev_events; - if amount > limit { - debug_warn!("Max prev event limit reached! Limit: {limit}"); - graph.insert(prev_event_id.clone(), HashSet::new()); - continue; - } - - if json_opt.is_none() { - json_opt = self - .services - .outlier - .get_outlier_pdu_json(&prev_event_id) - .await - .ok(); - } - - if let Some(json) = json_opt { - if pdu.origin_server_ts() > first_ts_in_room { - amount = amount.saturating_add(1); - for prev_prev in pdu.prev_events() { - if !graph.contains_key(prev_prev) { - todo_outlier_stack.push_back(prev_prev.to_owned()); - } - } - - graph.insert( - prev_event_id.clone(), - pdu.prev_events().map(ToOwned::to_owned).collect(), - ); - } else { - // Time based check failed - graph.insert(prev_event_id.clone(), HashSet::new()); - } - - eventid_info.insert(prev_event_id.clone(), (pdu, json)); + while let Some(event_id) = queue.pop_front() { + debug!(event_id=%incoming_pdu.event_id, "Fetching any missing prev_events"); + let mut missing_prev_events: HashSet = + incoming_pdu.prev_events().map(ToOwned::to_owned).collect(); + for pid in missing_prev_events.clone() { + if self.services.timeline.pdu_exists(&pid).await { + trace!("Found prev event {pid} for outlier event {event_id} locally"); + missing_prev_events.remove(&pid); } else { - // Get json failed, so this was not fetched over federation - graph.insert(prev_event_id.clone(), HashSet::new()); + debug_info!( + "Could not find prev event {pid} for outlier event {event_id} locally, \ + will fetch over federation" + ); } - }, - | _ => { - // Fetch and handle failed - graph.insert(prev_event_id.clone(), HashSet::new()); - }, + } + if !missing_prev_events.is_empty() { + debug_info!( + "Fetching {} missing prev events for outlier event {event_id}", + missing_prev_events.len() + ); + let backfilled = self + .backfill_missing_events( + room_id.to_owned(), + vec![event_id.clone()], + origin.to_owned(), + ) + .await?; + debug_info!("Fetched {} missing events for {event_id}", backfilled.len()); + let mapped = backfilled + .iter() + .map(|(eid, evt)| { + let mut obj = evt.to_canonical_object(); + obj.remove("event_id"); // event_id is inserted by backfill_missing_events + (eid.clone(), obj) + }) + .collect::>(); + let local_dag = if mapped.len() == 1 { + mapped.keys().map(ToOwned::to_owned).collect() + } else { + build_local_dag(&mapped).await? + }; + debug_info!("Preparing to handle {} missing events", backfilled.len()); + for prev_event_id in local_dag { + let obj = mapped + .get(&prev_event_id) + .expect("We should have this event in memory"); + debug_info!("Handling prev event {prev_event_id}"); + match self + .handle_outlier_pdu( + origin, + create_event, + &prev_event_id, + room_id, + obj.clone(), + false, + ) + .await + { + | Ok(_) => { + debug!("Successfully handled {prev_event_id} as an outlier"); + missing_prev_events.remove(&prev_event_id); + }, + | Err(e) => + error!(error=?e, %prev_event_id, %event_id, "Failed to handle prev event"), + } + debug_info!("Finished handling prev"); + } + } + let outlier = self.services.timeline.get_pdu(&event_id).await; + if missing_prev_events.is_empty() + && let Ok(pdu) = outlier + { + // promote any prevs first + for prev_event_id in pdu.prev_events() { + debug_info!("Promoting prev event {prev_event_id} to timeline"); + let prev_pdu = self.services.timeline.get_pdu(&event_id).await?; + let val = prev_pdu.to_canonical_object(); + self.upgrade_outlier_to_timeline_pdu( + prev_pdu, + val, + create_event, + origin, + room_id, + ) + .await?; + debug_info!("Finished prev promoting {prev_event_id} to timeline"); + } + debug_info!("Promoting event {event_id} to timeline"); + let val = pdu.to_canonical_object(); + self.upgrade_outlier_to_timeline_pdu(pdu, val, create_event, origin, room_id) + .await?; + debug_info!("Finished promoting {event_id} to timeline"); + } else { + debug!(?missing_prev_events, ok=%outlier.is_ok(), "Not promoting {event_id}"); + } } + + Ok(()) } - - let event_fetch = |event_id| { - let origin_server_ts = eventid_info - .get(&event_id) - .map_or_else(|| uint!(0), |info| info.0.origin_server_ts().get()); - - // This return value is the key used for sorting events, - // events are then sorted by power level, time, - // and lexically by event_id. - future::ok((int!(0), MilliSecondsSinceUnixEpoch(origin_server_ts))) - }; - - let sorted = state_res::lexicographical_topological_sort(&graph, &event_fetch) - .await - .map_err(|e| err!(Database(error!("Error sorting prev events: {e}"))))?; - - Ok((sorted, eventid_info)) } diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 790b4ec9e..d6c20518c 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,7 +1,6 @@ use std::collections::{HashMap, hash_map}; use conduwuit::{Err, Event, Result, debug, debug_warn, err, implement}; -use futures::FutureExt; use ruma::{ EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids, events::StateEventType, @@ -42,7 +41,6 @@ pub(super) async fn fetch_state( let state_ids = res.pdu_ids.iter().map(AsRef::as_ref); let state_vec = self .fetch_and_handle_outliers(origin, state_ids, create_event, room_id) - .boxed() .await; let mut state: HashMap = HashMap::with_capacity(state_vec.len()); diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 204dfa715..5033ab625 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -1,14 +1,11 @@ -use std::{ - collections::{BTreeMap, hash_map}, - time::Instant, -}; +use std::{collections::BTreeMap, time::Instant}; use conduwuit::{ Err, Event, PduEvent, Result, debug::INFO_SPAN_LEVEL, debug_error, debug_info, defer, err, - implement, info, trace, utils::stream::IterStream, warn, + implement, info, trace, warn, }; use futures::{ - FutureExt, TryFutureExt, TryStreamExt, + FutureExt, future::{OptionFuture, try_join4}, }; use ruma::{ @@ -236,63 +233,21 @@ pub async fn handle_incoming_pdu<'a>( } // Skip old events - let first_ts_in_room = self - .services - .timeline - .first_pdu_in_room(room_id) - .await? - .origin_server_ts(); + // let first_ts_in_room = self + // .services + // .timeline + // .first_pdu_in_room(room_id) + // .await? + // .origin_server_ts(); // 9. Fetch any missing prev events doing all checks listed here starting at 1. // These are timeline events - let (sorted_prev_events, mut eventid_info) = self - .fetch_prev(origin, create_event, room_id, first_ts_in_room, incoming_pdu.prev_events()) - .await?; + debug!("Handling previous events"); - debug!( - events = ?sorted_prev_events, - "Handling previous events" - ); - - sorted_prev_events - .iter() - .try_stream() - .map_ok(AsRef::as_ref) - .try_for_each(|prev_id| { - self.handle_prev_pdu( - origin, - event_id, - room_id, - eventid_info.remove(prev_id), - create_event, - first_ts_in_room, - prev_id, - ) - .inspect_err(move |e| { - warn!("Prev {prev_id} failed: {e}"); - match self - .services - .globals - .bad_event_ratelimiter - .write() - .entry(prev_id.into()) - { - | hash_map::Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - | hash_map::Entry::Occupied(mut e) => { - let tries = e.get().1.saturating_add(1); - *e.get_mut() = (Instant::now(), tries); - }, - } - }) - .map(|_| self.services.server.check_running()) - }) - .boxed() + self.fetch_prevs(room_id, create_event, &incoming_pdu, origin) .await?; // Done with prev events, now handling the incoming event self.upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) - .boxed() .await } diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index dc08ded5e..a8bba57fc 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap, hash_map}; +use std::collections::{BTreeMap, HashMap, HashSet, hash_map}; use conduwuit::{ Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, implement, state_res, @@ -10,7 +10,7 @@ events::StateEventType, }; -use super::{check_room_id, get_room_version_rules}; +use super::{build_local_dag, check_room_id, get_room_version_rules}; use crate::rooms::timeline::pdu_fits; #[implement(super::Service)] @@ -22,7 +22,7 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( event_id: &'a EventId, room_id: &'a RoomId, mut value: CanonicalJsonObject, - auth_events_known: bool, + _auth_events_known: bool, ) -> Result<(PduEvent, BTreeMap)> where Pdu: Event + Send + Sync, @@ -107,44 +107,71 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( } // Fetch any missing ones & reject invalid ones - let missing_auth_events = if auth_events_known { - pdu_event - .auth_events() - .filter(|id| !auth_events.contains_key(*id)) - .collect::>() - } else { - pdu_event.auth_events().collect::>() - }; - if !missing_auth_events.is_empty() || !auth_events_known { + let mut missing_auth_events: HashSet = pdu_event + .auth_events() + .filter(|id| !auth_events.contains_key(*id)) + .map(ToOwned::to_owned) + .collect(); + + if !missing_auth_events.is_empty() { debug_info!( "Fetching {} missing auth events for outlier event {event_id}", missing_auth_events.len() ); - for (pdu, _) in self - .fetch_and_handle_outliers( - origin, - missing_auth_events.iter().copied(), - create_event, - room_id, + let backfilled = self + .backfill_missing_events( + room_id.to_owned(), + vec![event_id.to_owned()], + origin.to_owned(), ) - .await - { - auth_events.insert(pdu.event_id().to_owned(), pdu); + .await?; + debug_info!("Fetched {} missing auth events for {event_id}", backfilled.len()); + let mapped = backfilled + .iter() + .map(|(eid, evt)| { + let mut obj = evt.to_canonical_object(); + obj.remove("event_id"); // event_id is inserted by backfill_missing_events + (eid.clone(), obj) + }) + .collect::>(); + let local_dag = if mapped.len() == 1 { + mapped.keys().map(ToOwned::to_owned).collect() + } else { + build_local_dag(&mapped).await? + }; + debug_info!("Preparing to handle {} missing auth events", backfilled.len()); + for prev_event_id in local_dag { + let obj = mapped + .get(&prev_event_id) + .expect("We should have this event in memory"); + debug_info!("Handling prev {prev_event_id}"); + let (prev, _) = Box::pin(self.handle_outlier_pdu( + origin, + create_event, + &prev_event_id, + room_id, + obj.clone(), + false, + )) + .await?; + if missing_auth_events.contains(&*prev_event_id) { + missing_auth_events.remove(&prev_event_id); + auth_events.insert(prev_event_id, prev); + } + debug_info!("Finished handling prev auth event"); } } else { debug!("No missing auth events for outlier event {event_id}"); } - // reject if we are still missing some - let still_missing = pdu_event - .auth_events() - .filter(|id| !auth_events.contains_key(*id)) - .collect::>(); - if !still_missing.is_empty() { + // reject if we are still missing some auth events. + // If we're still missing prev events, we will fetch them individually later, + // but there's no reason for us to be missing auth events now we've gapfilled + // the DAG. + if !missing_auth_events.is_empty() { // Don't reject: this could be a temporary condition - // TODO: use get_missing_events? return Err!(Request(InvalidParam( "Could not fetch all auth events for outlier event {event_id}, still missing: \ - {still_missing:?}" + {missing_auth_events:?}" ))); } diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs deleted file mode 100644 index cb4978d94..000000000 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ /dev/null @@ -1,89 +0,0 @@ -use std::{collections::BTreeMap, time::Instant}; - -use conduwuit::{ - Err, Event, PduEvent, Result, debug::INFO_SPAN_LEVEL, defer, implement, - utils::continue_exponential_backoff_secs, -}; -use ruma::{CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, RoomId, ServerName}; -use tracing::debug; - -#[implement(super::Service)] -#[allow(clippy::type_complexity)] -#[allow(clippy::too_many_arguments)] -#[tracing::instrument( - name = "prev", - level = INFO_SPAN_LEVEL, - skip_all, - fields(%prev_id), -)] -pub(super) async fn handle_prev_pdu<'a, Pdu>( - &self, - origin: &'a ServerName, - event_id: &'a EventId, - room_id: &'a RoomId, - eventid_info: Option<(PduEvent, BTreeMap)>, - create_event: &'a Pdu, - first_ts_in_room: MilliSecondsSinceUnixEpoch, - prev_id: &'a EventId, -) -> Result -where - Pdu: Event + Send + Sync, -{ - // Check for disabled again because it might have changed - if self.services.metadata.is_disabled(room_id).await { - return Err!(Request(Forbidden(debug_warn!( - "Federaton of room {room_id} is currently disabled on this server. Request by \ - origin {origin} and event ID {event_id}" - )))); - } - - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .get(prev_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { - debug!( - ?tries, - duration = ?time.elapsed(), - "Backing off from prev_event" - ); - return Ok(()); - } - } - - let Some((pdu, json)) = eventid_info else { - return Ok(()); - }; - - // Skip old events - if pdu.origin_server_ts() < first_ts_in_room { - return Ok(()); - } - - let start_time = Instant::now(); - self.federation_handletime - .write() - .insert(room_id.into(), ((*prev_id).to_owned(), start_time)); - - defer! {{ - self.federation_handletime - .write() - .remove(room_id); - }}; - - self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id) - .await?; - - debug!( - elapsed = ?start_time.elapsed(), - "Handled prev_event", - ); - - Ok(()) -} diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 6197bc87e..f51d701f0 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -4,7 +4,6 @@ mod fetch_state; mod handle_incoming_pdu; mod handle_outlier_pdu; -mod handle_prev_pdu; mod parse_incoming_pdu; mod policy_server; mod resolve_state; @@ -15,6 +14,7 @@ use async_trait::async_trait; use conduwuit::{Err, Event, PduEvent, Result, Server, SyncRwLock, utils::MutexMap}; +pub use fetch_and_handle_outliers::build_local_dag; use ruma::{ OwnedEventId, OwnedRoomId, RoomId, events::room::create::RoomCreateEventContent, room_version_rules::RoomVersionRules, @@ -22,7 +22,6 @@ use tokio::sync::Notify; use crate::{Dep, globals, rooms, sending, server_keys}; - pub struct Service { pub mutex_federation: RoomMutexMap, pub federation_handletime: SyncRwLock, diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 0a711b468..db947a803 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -41,6 +41,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( .get_pdu_id(incoming_pdu.event_id()) .await { + trace!(event_id=%incoming_pdu.event_id(), "Skipping upgrade of already upgraded PDU"); return Ok(Some(pduid)); } @@ -81,6 +82,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( }; if state_at_incoming_event.is_none() { + trace!("Could not calculate incoming state, asking remote {origin} for it"); state_at_incoming_event = self .fetch_state(origin, create_event, room_id, incoming_pdu.event_id()) .await?; diff --git a/src/service/server_keys/request.rs b/src/service/server_keys/request.rs index c8823fdc7..422bdc599 100644 --- a/src/service/server_keys/request.rs +++ b/src/service/server_keys/request.rs @@ -34,8 +34,9 @@ pub(super) async fn batch_notary_request<'a, S, K>( batch }); - - debug_assert!(!server_keys.is_empty(), "empty batch request to notary"); + if server_keys.is_empty() { + return Ok(vec![]); + } let mut results = Vec::new(); while let Some(batch) = server_keys From 0d5aa7ede11ab37f55f4236bd963e6aae6f4bb64 Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 26 May 2026 01:00:12 +0100 Subject: [PATCH 03/81] fix: This is some bullshit I tell you --- src/admin/debug/commands.rs | 31 +++- src/admin/debug/mod.rs | 6 + src/api/server/send.rs | 1 - .../fetch_and_handle_outliers.rs | 72 +++----- src/service/rooms/event_handler/fetch_prev.rs | 173 ++++++++---------- .../rooms/event_handler/handle_outlier_pdu.rs | 11 +- .../rooms/event_handler/state_at_incoming.rs | 4 +- src/service/rooms/timeline/mod.rs | 4 + 8 files changed, 154 insertions(+), 148 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index f9a678f19..8194c726b 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -22,7 +22,7 @@ use lettre::message::Mailbox; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, - OwnedRoomOrAliasId, OwnedServerName, RoomId, RoomVersionId, UInt, + OwnedRoomOrAliasId, OwnedServerName, RoomId, RoomVersionId, ServerName, UInt, api::federation::event::get_room_state, events::AnyStateEvent, serde::Raw, }; use service::rooms::{ @@ -1138,3 +1138,32 @@ pub(super) async fn send_test_email(&self) -> Result { Ok(()) } + +#[admin_command] +pub(super) async fn walk_missing_events( + &self, + latest: OwnedEventId, + earliest: OwnedEventId, + via: String, +) -> Result { + let earliest_pdu = self.services.rooms.timeline.get_pdu(&earliest).await?; + let events = self + .services + .rooms + .event_handler + .backfill_missing_events( + earliest_pdu.room_id_or_hash(), + HashSet::from_iter(vec![latest]), + vec![earliest], + ServerName::parse(via)?, + ) + .await?; + self.write_str(&format!("Found {} events:\n\n```\n", events.len())) + .await?; + for (event_id, event) in events { + self.write_str(&format!("{event_id}: {:?}\n\n", serde_json::to_string(&event))) + .await?; + } + self.write_str("\n```").await?; + Ok(()) +} diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index a9478ce82..608d8c5aa 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -32,6 +32,12 @@ pub enum DebugCommand { event_id: OwnedEventId, }, + WalkMissingEvents { + latest: OwnedEventId, + earliest: OwnedEventId, + via: String, + }, + /// Parse and print a PDU from a JSON /// /// The PDU event is only checked for validity and is not added to the diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 6f6d35d10..fd0ba490d 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -47,7 +47,6 @@ FederationTxnState, TransactionError, TxnKey, WrappedTransactionResponse, }; use tokio::sync::watch::{Receiver, Sender}; -use tracing::instrument; use crate::Ruma; diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 6f1262ffd..1797ebce8 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -1,5 +1,4 @@ use std::{ - cmp::min, collections::{BTreeMap, HashMap, HashSet, VecDeque, hash_map}, time::Instant, }; @@ -9,7 +8,6 @@ matrix::event::gen_event_id_canonical_json, state_res::lexicographical_topological_sort, trace, utils::continue_exponential_backoff_secs, warn, }; -use futures::StreamExt; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedServerName, RoomId, ServerName, UInt, @@ -98,37 +96,33 @@ impl super::Service { /// /// This function does not persist the events. The caller is responsible for /// passing them through handle_incoming_pdu. - pub(super) async fn backfill_missing_events( + pub async fn backfill_missing_events( &self, room_id: OwnedRoomId, - head: Vec, + head: HashSet, + tail: Vec, via: OwnedServerName, ) -> conduwuit::Result> { if head.is_empty() { return Ok(HashMap::new()); } - let tail = self - .services - .state - .get_forward_extremities(&room_id) - .collect::>() - .await; // TODO: min_depth is probably necessary to avoid fetching the entire room // history if there are very long gaps - let mut latest_events: HashSet = HashSet::from_iter(head.clone()); + let mut latest_events = head.clone(); let mut loop_count: u64 = 3; - // Start with a base number of 3 so that we fetch 10, 16, 25, 36, etc - // instead of 1, 2, 4, 9, so on. + // Start with 3 so that we fetch 9, 16, 25, 36, 49, 64, 81, 100 events. + // This gives steady growth to the server's typical limit of 100. It's unlikely + // we'll end up close to that. let mut backfilled_events = HashMap::with_capacity(10); while !latest_events.is_empty() { + // TODO: holy clone() + let frontier_before = latest_events.clone(); let todo: Vec = latest_events.clone().into_iter().collect(); let mut request = get_missing_events::v1::Request::new(room_id.clone(), tail.clone(), todo.clone()); - let limit = min(loop_count.saturating_pow(2), 100); - request.limit = limit - .try_into() - .expect("limit cannot be greater than 100, which fits into UInt"); + let limit = loop_count.saturating_pow(2).min(100); + request.limit = limit.try_into().expect("limit must fit into UInt"); debug_info!( backfilled=%backfilled_events.len(), @@ -148,6 +142,7 @@ pub(super) async fn backfill_missing_events( .send_federation_request(&via, request) .await?; loop_count = loop_count.saturating_add(1); + trace!(?response, "get_missing_events response"); // Some buggy servers (including old continuwuity) may return the same events // multiple times, which can cause this to be an infinite loop. @@ -160,6 +155,7 @@ pub(super) async fn backfill_missing_events( debug_info!("No more missing events found"); break; } + for event in response.events { trace!("Parsing incoming event from backfill"); let (incoming_room_id, event_id, pdu_json) = @@ -178,52 +174,41 @@ pub(super) async fn backfill_missing_events( debug!("Skipping known event {event_id}"); continue; } - if backfilled_events.contains_key(&event_id) { + let retransmitted = backfilled_events.contains_key(&event_id); + if retransmitted { debug_warn!(%via, %event_id, "Remote retransmitted event"); - continue; - } - // TODO: Should this be scoped to the GME session? We might end up incorrectly - // assuming we've caught up if we do this - if let Ok(pdu) = self.services.timeline.get_pdu(&event_id).await { - debug!(%via, %event_id, "Already seen event in database"); - backfilled_events.insert(event_id.clone(), pdu); } else { - unseen = unseen.saturating_add(1); + if let Ok(pdu) = self.services.timeline.get_pdu(&event_id).await { + debug!(%via, %event_id, "Already seen event in database"); + backfilled_events.insert(event_id.clone(), pdu); + } else { + unseen = unseen.saturating_add(1); + } } let parsed = PduEvent::from_id_val(&event_id, pdu_json) .map_err(|e| err!(BadServerResponse("Unable to parse {event_id}: {e}")))?; for prev_event_id in parsed.prev_events() { - // Verify that we have all of this event's prev_events. If we don't, add it to - // the work queue if !(backfilled_events.contains_key(prev_event_id) || self.services.timeline.pdu_exists(prev_event_id).await) { latest_events.insert(prev_event_id.to_owned()); - break; } - continue; } - backfilled_events.insert(event_id.clone(), parsed); - } - for event_id in todo { - latest_events.remove(&event_id); - if let hash_map::Entry::Vacant(e) = backfilled_events.entry(event_id.clone()) { - let evt = self.services.timeline.get_pdu(&event_id).await?; - e.insert(evt); + if !retransmitted { + backfilled_events.insert(event_id.clone(), parsed); } } + latest_events.retain(|event_id| !backfilled_events.contains_key(event_id)); debug!( count=%chunk_len, new=%unseen, remaining=%latest_events.len(), "Got missing events" ); - if unseen == 0 { - debug_warn!("Didn't see any new events, breaking cycle"); - break; - } else if chunk_len < usize::try_from(limit)? { - debug!( - "Got less than the limit number of events, assuming there's no more to fetch" + let frontier_changed = latest_events != frontier_before; + if unseen == 0 && !frontier_changed { + debug_warn!( + "Didn't see any new events and the frontier did not change, breaking cycle" ); break; } @@ -243,6 +228,7 @@ pub(super) async fn backfill_missing_events( /// b. Look at outlier pdu tree /// c. Ask origin server over federation /// d. TODO: Ask other servers over federation? + #[deprecated] pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>( &self, origin: &'a ServerName, diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 270b3d0d7..628b776a1 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,11 +1,18 @@ -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{HashMap, HashSet}; -use conduwuit::{Event, PduEvent, debug, debug_info, error, trace}; -use ruma::{OwnedEventId, RoomId, ServerName}; +use conduwuit::{ + Event, PduEvent, debug, debug_info, + result::DebugInspect, + utils::{BoolExt, IterStream, stream::BroadbandExt}, +}; +use futures::StreamExt; +use ruma::{RoomId, ServerName}; use crate::rooms::event_handler::build_local_dag; impl super::Service { + /// Fetches any missing prev_events for this event and persists them before + /// returning. pub(super) async fn fetch_prevs( &self, room_id: &RoomId, @@ -13,106 +20,72 @@ pub(super) async fn fetch_prevs( incoming_pdu: &PduEvent, origin: &ServerName, ) -> conduwuit::Result<()> { - let mut queue: VecDeque = VecDeque::new(); - queue.push_back(incoming_pdu.event_id().to_owned()); + let missing = incoming_pdu + .prev_events() + .stream() + .broad_filter_map(|event_id| async move { + self.services + .timeline + .get_non_outlier_pdu_json(event_id) + .await + .inspect(|_| debug!("Found prev_event {event_id} locally.")) + .inspect_err(|e| debug!(%e, "Could not find prev_event {event_id} locally.")) + .is_ok() + .or(|| event_id.to_owned()) + }) + .collect::>() + .await; + if missing.is_empty() { + debug!(event_id=%incoming_pdu.event_id(), "No missing prev events."); + return Ok(()); + } + debug!(%room_id, event_id=%incoming_pdu.event_id(), ?missing, "Fetching previous events"); + let tail = self + .services + .state + .get_forward_extremities(room_id) + .collect::>() + .await; - while let Some(event_id) = queue.pop_front() { - debug!(event_id=%incoming_pdu.event_id, "Fetching any missing prev_events"); - let mut missing_prev_events: HashSet = - incoming_pdu.prev_events().map(ToOwned::to_owned).collect(); - for pid in missing_prev_events.clone() { - if self.services.timeline.pdu_exists(&pid).await { - trace!("Found prev event {pid} for outlier event {event_id} locally"); - missing_prev_events.remove(&pid); - } else { - debug_info!( - "Could not find prev event {pid} for outlier event {event_id} locally, \ - will fetch over federation" - ); - } - } - if !missing_prev_events.is_empty() { - debug_info!( - "Fetching {} missing prev events for outlier event {event_id}", - missing_prev_events.len() - ); - let backfilled = self - .backfill_missing_events( - room_id.to_owned(), - vec![event_id.clone()], - origin.to_owned(), - ) - .await?; - debug_info!("Fetched {} missing events for {event_id}", backfilled.len()); - let mapped = backfilled - .iter() - .map(|(eid, evt)| { - let mut obj = evt.to_canonical_object(); - obj.remove("event_id"); // event_id is inserted by backfill_missing_events - (eid.clone(), obj) - }) - .collect::>(); - let local_dag = if mapped.len() == 1 { - mapped.keys().map(ToOwned::to_owned).collect() - } else { - build_local_dag(&mapped).await? - }; - debug_info!("Preparing to handle {} missing events", backfilled.len()); - for prev_event_id in local_dag { - let obj = mapped - .get(&prev_event_id) - .expect("We should have this event in memory"); - debug_info!("Handling prev event {prev_event_id}"); - match self - .handle_outlier_pdu( - origin, - create_event, - &prev_event_id, - room_id, - obj.clone(), - false, - ) - .await - { - | Ok(_) => { - debug!("Successfully handled {prev_event_id} as an outlier"); - missing_prev_events.remove(&prev_event_id); - }, - | Err(e) => - error!(error=?e, %prev_event_id, %event_id, "Failed to handle prev event"), - } - debug_info!("Finished handling prev"); - } - } - let outlier = self.services.timeline.get_pdu(&event_id).await; - if missing_prev_events.is_empty() - && let Ok(pdu) = outlier - { - // promote any prevs first - for prev_event_id in pdu.prev_events() { - debug_info!("Promoting prev event {prev_event_id} to timeline"); - let prev_pdu = self.services.timeline.get_pdu(&event_id).await?; - let val = prev_pdu.to_canonical_object(); - self.upgrade_outlier_to_timeline_pdu( - prev_pdu, - val, - create_event, - origin, - room_id, - ) - .await?; - debug_info!("Finished prev promoting {prev_event_id} to timeline"); - } - debug_info!("Promoting event {event_id} to timeline"); - let val = pdu.to_canonical_object(); - self.upgrade_outlier_to_timeline_pdu(pdu, val, create_event, origin, room_id) - .await?; - debug_info!("Finished promoting {event_id} to timeline"); - } else { - debug!(?missing_prev_events, ok=%outlier.is_ok(), "Not promoting {event_id}"); - } + let backfilled = self + .backfill_missing_events( + room_id.to_owned(), + HashSet::from_iter(vec![incoming_pdu.event_id().to_owned()]), + tail, + origin.to_owned(), + ) + .await?; + debug_info!("Fetched {} missing events", backfilled.len()); + + // Persist all fetched events + let mapped = backfilled + .iter() + .map(|(eid, evt)| { + let mut obj = evt.to_canonical_object(); + obj.remove("event_id"); // event_id is inserted by backfill_missing_events + (eid.clone(), obj) + }) + .collect::>(); + + let to_persist = if mapped.len() <= 1 { + mapped.keys().map(ToOwned::to_owned).collect() + } else { + build_local_dag(&mapped).await? + }; + + for event_id in to_persist { + debug_info!("Persisting fetched prev event {event_id}"); + let pdu_event = backfilled.get(&event_id).cloned().unwrap_or_else(|| { + panic!("Event {event_id} was in backfill response but not in map") + }); + let obj = pdu_event.to_canonical_object(); + self.upgrade_outlier_to_timeline_pdu(pdu_event, obj, create_event, origin, room_id) + .await + .debug_inspect(|_| debug_info!("Persisted fetched prev event {event_id}"))?; } + // NOTE because i keep forgetting: the caller persists incoming_pdu. + // we only care about its prev events Ok(()) } } diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index a8bba57fc..2d349a29d 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -4,7 +4,7 @@ Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, implement, state_res, trace, warn, }; -use futures::future::ready; +use futures::{StreamExt, future::ready}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, events::StateEventType, @@ -118,10 +118,17 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( "Fetching {} missing auth events for outlier event {event_id}", missing_auth_events.len() ); + let tail = self + .services + .state + .get_forward_extremities(room_id) + .collect::>() + .await; let backfilled = self .backfill_missing_events( room_id.to_owned(), - vec![event_id.to_owned()], + HashSet::from_iter(vec![event_id.to_owned()]), + tail, origin.to_owned(), ) .await?; diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 9f11ac4b8..4010c4462 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -5,7 +5,7 @@ }; use conduwuit::{ - Result, debug, err, error, implement, + Result, debug, debug_error, err, error, implement, matrix::{Event, StateMap}, trace, utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryWidebandExt}, @@ -37,6 +37,7 @@ pub(super) async fn state_at_incoming_degree_one( .pdu_shortstatehash(prev_event) .await else { + trace!("No shortstatehash for {prev_event}, cannot calculate one-degree state."); return Ok(None); }; @@ -99,6 +100,7 @@ pub(super) async fn state_at_incoming_resolved( .map_ok(move |sstatehash| (sstatehash, prev_event)) }) .try_collect::>() + .inspect_err(|e| debug_error!("failed to calculate N-degree short state hashes: {e}")) .await else { return Ok(None); diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 8b142463f..f4cae2ec7 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -173,6 +173,10 @@ pub async fn get_non_outlier_pdu_json( self.db.get_non_outlier_pdu_json(event_id).await } + pub async fn non_outlier_pdu_exists(&self, event_id: &EventId) -> bool { + self.db.non_outlier_pdu_exists(event_id).await.is_ok() + } + /// Returns the pdu's id. #[inline] pub async fn get_pdu_id(&self, event_id: &EventId) -> Result { From eeb937416c4e6db905d27bdc8ea00520ad05ed7e Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 26 May 2026 20:00:55 +0100 Subject: [PATCH 04/81] perf: Improve gap filling, handle missing auth events better --- src/admin/debug/commands.rs | 10 +- .../fetch_and_handle_outliers.rs | 202 +++++++++--------- src/service/rooms/event_handler/fetch_prev.rs | 31 ++- .../rooms/event_handler/handle_outlier_pdu.rs | 121 ++++------- 4 files changed, 166 insertions(+), 198 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 8194c726b..dd6201c74 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -1146,16 +1146,16 @@ pub(super) async fn walk_missing_events( earliest: OwnedEventId, via: String, ) -> Result { - let earliest_pdu = self.services.rooms.timeline.get_pdu(&earliest).await?; + let latest_pdu = self.services.rooms.timeline.get_pdu(&latest).await?; let events = self .services .rooms .event_handler - .backfill_missing_events( - earliest_pdu.room_id_or_hash(), - HashSet::from_iter(vec![latest]), + .get_missing_events( + &latest_pdu.room_id_or_hash(), + &latest_pdu, vec![earliest], - ServerName::parse(via)?, + &ServerName::parse(via)?, ) .await?; self.write_str(&format!("Found {} events:\n\n```\n", events.len())) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 1797ebce8..dae9968f2 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -3,14 +3,19 @@ time::Instant, }; +use assign::assign; use conduwuit::{ - Err, Event, PduEvent, debug, debug_info, debug_warn, err, - matrix::event::gen_event_id_canonical_json, state_res::lexicographical_topological_sort, - trace, utils::continue_exponential_backoff_secs, warn, + Event, PduEvent, debug, debug_info, debug_warn, err, error, + matrix::event::gen_event_id_canonical_json, + state_res::lexicographical_topological_sort, + trace, + utils::{IterStream, continue_exponential_backoff_secs, stream::BroadbandExt}, + warn, }; +use futures::StreamExt; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, - OwnedRoomId, OwnedServerName, RoomId, ServerName, UInt, + RoomId, ServerName, UInt, api::federation::event::{get_event, get_missing_events}, int, }; @@ -94,129 +99,120 @@ impl super::Service { /// exponentially incrementing limit (up to 100 per request) until it has /// filled the gap, i.e. when the remote says there's no more events. /// + /// This function will iterate until the remote returns no more events, + /// increasing the limit by a factor of 10. If 20 iterations are reached or + /// 200 events are backfilled, the function will give up and return what it + /// has, to avoid pulling in too much data (for example, absurdly large + /// gaps). + /// /// This function does not persist the events. The caller is responsible for /// passing them through handle_incoming_pdu. - pub async fn backfill_missing_events( + /// + /// ## Parameters + /// + /// - `room_id`: The room's ID. + /// - `head`: The event we are potentially missing prev_events for. + /// - `tail`: The most recently known events in the graph (typically forward + /// extremities). + /// - `via`: The server to ask for missing events. + pub async fn get_missing_events( &self, - room_id: OwnedRoomId, - head: HashSet, + room_id: &RoomId, + head: &PduEvent, tail: Vec, - via: OwnedServerName, + via: &ServerName, ) -> conduwuit::Result> { - if head.is_empty() { - return Ok(HashMap::new()); - } - // TODO: min_depth is probably necessary to avoid fetching the entire room - // history if there are very long gaps - let mut latest_events = head.clone(); - let mut loop_count: u64 = 3; - // Start with 3 so that we fetch 9, 16, 25, 36, 49, 64, 81, 100 events. - // This gives steady growth to the server's typical limit of 100. It's unlikely - // we'll end up close to that. - let mut backfilled_events = HashMap::with_capacity(10); - - while !latest_events.is_empty() { - // TODO: holy clone() - let frontier_before = latest_events.clone(); - let todo: Vec = latest_events.clone().into_iter().collect(); - let mut request = - get_missing_events::v1::Request::new(room_id.clone(), tail.clone(), todo.clone()); - let limit = loop_count.saturating_pow(2).min(100); - request.limit = limit.try_into().expect("limit must fit into UInt"); - - debug_info!( - backfilled=%backfilled_events.len(), - %loop_count, - "Asking {via} for up to {limit} missing events", - ); - trace!( - ?latest_events, - ?tail, - %via, - %limit, - "Requesting missing events" + #[cfg(debug_assertions)] + { + let missing_count = head + .prev_events() + .stream() + .broad_filter_map(|event_id| async move { + match self + .services + .timeline + .get_non_outlier_pdu_json(event_id) + .await + .inspect(|_| debug!("Found prev_event {event_id} locally.")) + .inspect_err( + |e| debug!(%e, "Could not find prev_event {event_id} locally."), + ) { + | Ok(_) => None, + | Err(_) => Some(event_id), + } + }) + .count() + .await; + debug_assert_ne!( + missing_count, 0, + "event passed to get_missing_events is not missing any events (wasteful call)" ); + }; + + let mut discovered = HashMap::with_capacity(20); + let mut latest_events = vec![head.event_id().to_owned()]; + let mut iterations = 0_u8; + loop { + iterations = iterations.saturating_add(1); + let limit = iterations.saturating_mul(10); + debug_info!(%limit, %via, %iterations, "Attempting to gap fill missing events"); let response: get_missing_events::v1::Response = self .services .sending - .send_federation_request(&via, request) + .send_federation_request( + via, + assign!( + get_missing_events::v1::Request::new( + room_id.to_owned(), + tail.clone(), + latest_events.clone() + ), + {limit: limit.into()} + ), + ) .await?; - loop_count = loop_count.saturating_add(1); - trace!(?response, "get_missing_events response"); - // Some buggy servers (including old continuwuity) may return the same events - // multiple times, which can cause this to be an infinite loop. - // In order to break this loop, if we see no new events from this response (i.e. - // all events in the response are already in backfilled_events), we stop, - // with a warning. - let mut unseen: usize = 0; - let chunk_len = response.events.len(); if response.events.is_empty() { - debug_info!("No more missing events found"); + debug_info!(%via, "Finished gap filling missing events (remote returned no more events)."); break; } + debug_info!("Got {} events back from remote", response.events.len()); - for event in response.events { - trace!("Parsing incoming event from backfill"); - let (incoming_room_id, event_id, pdu_json) = - self.parse_incoming_pdu(&event).await.map_err(|e| { - err!(BadServerResponse("{via} returned an invalid event: {e:?}")) - })?; - trace!(%incoming_room_id, %event_id, "Parsed incoming event from backfill"); - if incoming_room_id != room_id { - return Err!(BadServerResponse( - "{via} returned {event_id} in missing events which belongs to \ - {incoming_room_id}, not {room_id}" - )); - } - latest_events.remove(&event_id); - if head.contains(&event_id) || tail.contains(&event_id) { - debug!("Skipping known event {event_id}"); - continue; - } - let retransmitted = backfilled_events.contains_key(&event_id); - if retransmitted { - debug_warn!(%via, %event_id, "Remote retransmitted event"); - } else { - if let Ok(pdu) = self.services.timeline.get_pdu(&event_id).await { - debug!(%via, %event_id, "Already seen event in database"); - backfilled_events.insert(event_id.clone(), pdu); - } else { - unseen = unseen.saturating_add(1); + latest_events.clear(); + for raw_event in response.events { + let (_, event_id, pdu_json) = self.parse_incoming_pdu(&raw_event).await?; + let pdu = PduEvent::from_id_val(&event_id, pdu_json).map_err(|e| { + err!(Request(BadJson("Failed to parse backfilled event {event_id}: {e}"))) + })?; + + for prev_event_id in pdu.prev_events() { + if discovered.contains_key(prev_event_id) { + continue; } - } - let parsed = PduEvent::from_id_val(&event_id, pdu_json) - .map_err(|e| err!(BadServerResponse("Unable to parse {event_id}: {e}")))?; - for prev_event_id in parsed.prev_events() { - if !(backfilled_events.contains_key(prev_event_id) - || self.services.timeline.pdu_exists(prev_event_id).await) + if self + .services + .timeline + .non_outlier_pdu_exists(prev_event_id) + .await { - latest_events.insert(prev_event_id.to_owned()); + continue; } + latest_events.push(prev_event_id.to_owned()); + break; } - if !retransmitted { - backfilled_events.insert(event_id.clone(), parsed); - } + + discovered.insert(event_id.clone(), pdu); } - latest_events.retain(|event_id| !backfilled_events.contains_key(event_id)); - debug!( - count=%chunk_len, - new=%unseen, - remaining=%latest_events.len(), - "Got missing events" - ); - let frontier_changed = latest_events != frontier_before; - if unseen == 0 && !frontier_changed { - debug_warn!( - "Didn't see any new events and the frontier did not change, breaking cycle" - ); + + if latest_events.is_empty() { + break; + } else if discovered.len() >= 200 || iterations >= 20 { + error!("Gap too large, giving up"); break; } } - debug_info!("Successfully fetched {} missing events from {via}", backfilled_events.len()); - trace!("Missing_events: {backfilled_events:?}"); - Ok(backfilled_events) + Ok(discovered) } /// Find the event and auth it. Once the event is validated (steps 1 - 8) diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 628b776a1..bd67e4f28 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,9 +1,9 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use conduwuit::{ Event, PduEvent, debug, debug_info, - result::DebugInspect, utils::{BoolExt, IterStream, stream::BroadbandExt}, + warn, }; use futures::StreamExt; use ruma::{RoomId, ServerName}; @@ -28,8 +28,6 @@ pub(super) async fn fetch_prevs( .timeline .get_non_outlier_pdu_json(event_id) .await - .inspect(|_| debug!("Found prev_event {event_id} locally.")) - .inspect_err(|e| debug!(%e, "Could not find prev_event {event_id} locally.")) .is_ok() .or(|| event_id.to_owned()) }) @@ -48,12 +46,7 @@ pub(super) async fn fetch_prevs( .await; let backfilled = self - .backfill_missing_events( - room_id.to_owned(), - HashSet::from_iter(vec![incoming_pdu.event_id().to_owned()]), - tail, - origin.to_owned(), - ) + .get_missing_events(room_id, incoming_pdu, tail, origin) .await?; debug_info!("Fetched {} missing events", backfilled.len()); @@ -75,13 +68,19 @@ pub(super) async fn fetch_prevs( for event_id in to_persist { debug_info!("Persisting fetched prev event {event_id}"); - let pdu_event = backfilled.get(&event_id).cloned().unwrap_or_else(|| { - panic!("Event {event_id} was in backfill response but not in map") - }); - let obj = pdu_event.to_canonical_object(); - self.upgrade_outlier_to_timeline_pdu(pdu_event, obj, create_event, origin, room_id) + let obj = mapped.get(&event_id).cloned().unwrap(); + match self + .handle_outlier_pdu(origin, create_event, &event_id, room_id, obj, false) .await - .debug_inspect(|_| debug_info!("Persisted fetched prev event {event_id}"))?; + { + | Ok((pdu, val)) => + self.upgrade_outlier_to_timeline_pdu(pdu, val, create_event, origin, room_id) + .await, + | Err(e) => { + warn!("Failed to persist prev_event {event_id}: {e}"); + continue; + }, + }?; } // NOTE because i keep forgetting: the caller persists incoming_pdu. diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 2d349a29d..208f4aa08 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -1,16 +1,16 @@ -use std::collections::{BTreeMap, HashMap, HashSet, hash_map}; +use std::collections::{BTreeMap, HashMap, hash_map}; use conduwuit::{ - Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, implement, state_res, + Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, implement, info, state_res, trace, warn, }; -use futures::{StreamExt, future::ready}; +use futures::future::ready; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, - events::StateEventType, + api::federation::authorization::get_event_authorization, events::StateEventType, }; -use super::{build_local_dag, check_room_id, get_room_version_rules}; +use super::{check_room_id, get_room_version_rules}; use crate::rooms::timeline::pdu_fits; #[implement(super::Service)] @@ -107,79 +107,52 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( } // Fetch any missing ones & reject invalid ones - let mut missing_auth_events: HashSet = pdu_event - .auth_events() - .filter(|id| !auth_events.contains_key(*id)) - .map(ToOwned::to_owned) - .collect(); - - if !missing_auth_events.is_empty() { - debug_info!( - "Fetching {} missing auth events for outlier event {event_id}", - missing_auth_events.len() - ); - let tail = self + if auth_events.len() != pdu_event.auth_events().count() { + info!("Missing some auth events, asking remote for auth chain"); + let response: get_event_authorization::v1::Response = self .services - .state - .get_forward_extremities(room_id) - .collect::>() - .await; - let backfilled = self - .backfill_missing_events( - room_id.to_owned(), - HashSet::from_iter(vec![event_id.to_owned()]), - tail, - origin.to_owned(), - ) - .await?; - debug_info!("Fetched {} missing auth events for {event_id}", backfilled.len()); - let mapped = backfilled - .iter() - .map(|(eid, evt)| { - let mut obj = evt.to_canonical_object(); - obj.remove("event_id"); // event_id is inserted by backfill_missing_events - (eid.clone(), obj) - }) - .collect::>(); - let local_dag = if mapped.len() == 1 { - mapped.keys().map(ToOwned::to_owned).collect() - } else { - build_local_dag(&mapped).await? - }; - debug_info!("Preparing to handle {} missing auth events", backfilled.len()); - for prev_event_id in local_dag { - let obj = mapped - .get(&prev_event_id) - .expect("We should have this event in memory"); - debug_info!("Handling prev {prev_event_id}"); - let (prev, _) = Box::pin(self.handle_outlier_pdu( + .sending + .send_federation_request( origin, - create_event, - &prev_event_id, - room_id, - obj.clone(), - false, - )) - .await?; - if missing_auth_events.contains(&*prev_event_id) { - missing_auth_events.remove(&prev_event_id); - auth_events.insert(prev_event_id, prev); + get_event_authorization::v1::Request::new( + room_id.to_owned(), + event_id.to_owned(), + ), + ) + .await + .map_err(|e| { + err!(Request(Forbidden( + "Remote server is not divulging incoming event's auth chain: {e}" + ))) + })?; + let mut auth_chain_map = HashMap::with_capacity(response.auth_chain.len()); + for auth_pdu_json in response.auth_chain { + let (auth_event_room_id, auth_event_id, auth_pdu_json) = + self.parse_incoming_pdu(&auth_pdu_json).await?; + if auth_event_room_id != room_id { + return Err!(Request(BadJson( + "Auth event {auth_event_id} is in {auth_event_room_id}, not {room_id}." + ))); } - debug_info!("Finished handling prev auth event"); + let auth_pdu = PduEvent::from_id_val(&auth_event_id, auth_pdu_json) + .map_err(|e| err!(Request(BadJson("Invalid PDU {auth_event_id}: {e}"))))?; + auth_chain_map.insert(auth_event_id, auth_pdu); } - } else { - debug!("No missing auth events for outlier event {event_id}"); - } - // reject if we are still missing some auth events. - // If we're still missing prev events, we will fetch them individually later, - // but there's no reason for us to be missing auth events now we've gapfilled - // the DAG. - if !missing_auth_events.is_empty() { - // Don't reject: this could be a temporary condition - return Err!(Request(InvalidParam( - "Could not fetch all auth events for outlier event {event_id}, still missing: \ - {missing_auth_events:?}" - ))); + for aid in pdu_event.auth_events() { + if auth_events.contains_key(aid) { + continue; + } + if let Some(auth_event) = auth_chain_map.get(aid) { + auth_events.insert(aid.to_owned(), auth_event.clone()); + } else { + return Err!(Request(Forbidden( + "Remote server is not divulging incoming event's auth events (missing: \ + {aid})" + ))); + } + } + // TODO: do events received from auth chain need persisting? that sounds + // awfully slow } // 6. Reject "due to auth events" if the event doesn't pass auth based on the From 51d0e615f58c65ddcb1bde5f5d95971853ba33a3 Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 26 May 2026 20:11:53 +0100 Subject: [PATCH 05/81] perf: Make max gap depth fetch configurable --- .../fetch_and_handle_outliers.rs | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index dae9968f2..21f148921 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -100,10 +100,10 @@ impl super::Service { /// filled the gap, i.e. when the remote says there's no more events. /// /// This function will iterate until the remote returns no more events, - /// increasing the limit by a factor of 10. If 20 iterations are reached or - /// 200 events are backfilled, the function will give up and return what it - /// has, to avoid pulling in too much data (for example, absurdly large - /// gaps). + /// increasing the limit by a factor of 10. If 100 iterations are reached or + /// max_fetch_prev_events events are backfilled, the function will give up + /// and return what it has, to avoid pulling in too much data (for example, + /// absurdly large gaps). /// /// This function does not persist the events. The caller is responsible for /// passing them through handle_incoming_pdu. @@ -154,7 +154,7 @@ pub async fn get_missing_events( let mut iterations = 0_u8; loop { iterations = iterations.saturating_add(1); - let limit = iterations.saturating_mul(10); + let limit = iterations.saturating_mul(10).min(100); debug_info!(%limit, %via, %iterations, "Attempting to gap fill missing events"); let response: get_missing_events::v1::Response = self .services @@ -206,8 +206,15 @@ pub async fn get_missing_events( if latest_events.is_empty() { break; - } else if discovered.len() >= 200 || iterations >= 20 { - error!("Gap too large, giving up"); + } else if discovered.len() > self.services.server.config.max_fetch_prev_events.into() + || iterations >= 20 + { + error!( + filled=discovered.len(), + max_fetch_prev_events=self.services.server.config.max_fetch_prev_events, + %iterations, + "Gap too large, giving up" + ); break; } } From 9547c438d690e3943ead48d1f0096a06fdde1135 Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 26 May 2026 20:25:37 +0100 Subject: [PATCH 06/81] perf: Increase default `max_fetch_prev_events` to 256 --- conduwuit-example.toml | 2 +- src/api/server/send.rs | 1 + src/core/config/mod.rs | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 37b2aaa14..a4d60bbc7 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -297,7 +297,7 @@ # This item is undocumented. Please contribute documentation for it. # -#max_fetch_prev_events = 192 +#max_fetch_prev_events = 256 # How many incoming federation transactions the server is willing to be # processing at any given time before it becomes overloaded and starts diff --git a/src/api/server/send.rs b/src/api/server/send.rs index fd0ba490d..6f6d35d10 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -47,6 +47,7 @@ FederationTxnState, TransactionError, TxnKey, WrappedTransactionResponse, }; use tokio::sync::watch::{Receiver, Sender}; +use tracing::instrument; use crate::Ruma; diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 633536e3b..6ee251bf8 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -375,7 +375,7 @@ pub struct Config { #[serde(default = "default_max_request_size")] pub max_request_size: usize, - /// default: 192 + /// default: 256 #[serde(default = "default_max_fetch_prev_events")] pub max_fetch_prev_events: u16, @@ -2549,7 +2549,7 @@ fn default_pusher_timeout() -> u64 { 60 } fn default_pusher_idle_timeout() -> u64 { 15 } -fn default_max_fetch_prev_events() -> u16 { 192_u16 } +fn default_max_fetch_prev_events() -> u16 { 256_u16 } fn default_max_concurrent_inbound_transactions() -> usize { 150 } From 1445a8d446b4ea59a1843ab77c6b477612f5da54 Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 26 May 2026 21:04:52 +0100 Subject: [PATCH 07/81] feat: Keep track of a min_depth value Should prevent weird situations where we accidentally gapfill into backfill territory --- src/admin/debug/commands.rs | 31 +------------------ src/admin/debug/mod.rs | 6 ---- src/database/maps.rs | 4 +++ .../fetch_and_handle_outliers.rs | 17 ++++++++-- src/service/rooms/event_handler/fetch_prev.rs | 8 ++++- .../event_handler/upgrade_outlier_pdu.rs | 6 ++++ src/service/rooms/membership/mod.rs | 4 +++ src/service/rooms/metadata/mod.rs | 30 ++++++++++++++++-- 8 files changed, 65 insertions(+), 41 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index dd6201c74..f9a678f19 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -22,7 +22,7 @@ use lettre::message::Mailbox; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, - OwnedRoomOrAliasId, OwnedServerName, RoomId, RoomVersionId, ServerName, UInt, + OwnedRoomOrAliasId, OwnedServerName, RoomId, RoomVersionId, UInt, api::federation::event::get_room_state, events::AnyStateEvent, serde::Raw, }; use service::rooms::{ @@ -1138,32 +1138,3 @@ pub(super) async fn send_test_email(&self) -> Result { Ok(()) } - -#[admin_command] -pub(super) async fn walk_missing_events( - &self, - latest: OwnedEventId, - earliest: OwnedEventId, - via: String, -) -> Result { - let latest_pdu = self.services.rooms.timeline.get_pdu(&latest).await?; - let events = self - .services - .rooms - .event_handler - .get_missing_events( - &latest_pdu.room_id_or_hash(), - &latest_pdu, - vec![earliest], - &ServerName::parse(via)?, - ) - .await?; - self.write_str(&format!("Found {} events:\n\n```\n", events.len())) - .await?; - for (event_id, event) in events { - self.write_str(&format!("{event_id}: {:?}\n\n", serde_json::to_string(&event))) - .await?; - } - self.write_str("\n```").await?; - Ok(()) -} diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 608d8c5aa..a9478ce82 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -32,12 +32,6 @@ pub enum DebugCommand { event_id: OwnedEventId, }, - WalkMissingEvents { - latest: OwnedEventId, - earliest: OwnedEventId, - via: String, - }, - /// Parse and print a PDU from a JSON /// /// The PDU event is only checked for validity and is not added to the diff --git a/src/database/maps.rs b/src/database/maps.rs index 03fbbda72..33ef3e78d 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -187,6 +187,10 @@ pub(super) fn open_list(db: &Arc, maps: &[Descriptor]) -> Result { val_size_hint: Some(8), ..descriptor::RANDOM_SMALL }, + Descriptor { + name: "roomid_mindepth", + ..descriptor::RANDOM_SMALL + }, Descriptor { name: "roomserverids", ..descriptor::RANDOM_SMALL diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 21f148921..1f5fc7e78 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -115,12 +115,16 @@ impl super::Service { /// - `tail`: The most recently known events in the graph (typically forward /// extremities). /// - `via`: The server to ask for missing events. + /// - `min_depth`: Don't process events with a `depth` lower than this + /// value. Not massively useful, but can help short-circuit infinite loops + /// and weird edge paths. pub async fn get_missing_events( &self, room_id: &RoomId, head: &PduEvent, tail: Vec, via: &ServerName, + min_depth: UInt, ) -> conduwuit::Result> { #[cfg(debug_assertions)] { @@ -155,7 +159,7 @@ pub async fn get_missing_events( loop { iterations = iterations.saturating_add(1); let limit = iterations.saturating_mul(10).min(100); - debug_info!(%limit, %via, %iterations, "Attempting to gap fill missing events"); + debug_info!(%limit, %via, %iterations, discovered=discovered.len(), %min_depth, "Attempting to gap fill missing events"); let response: get_missing_events::v1::Response = self .services .sending @@ -167,7 +171,7 @@ pub async fn get_missing_events( tail.clone(), latest_events.clone() ), - {limit: limit.into()} + {limit: limit.into(), min_depth} ), ) .await?; @@ -185,6 +189,15 @@ pub async fn get_missing_events( err!(Request(BadJson("Failed to parse backfilled event {event_id}: {e}"))) })?; + if pdu.depth < min_depth { + debug_warn!( + "Received PDU with depth {} below min_depth {}, ignoring", + pdu.depth, + min_depth + ); + continue; + } + for prev_event_id in pdu.prev_events() { if discovered.contains_key(prev_event_id) { continue; diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index bd67e4f28..9be7b2e96 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -46,7 +46,13 @@ pub(super) async fn fetch_prevs( .await; let backfilled = self - .get_missing_events(room_id, incoming_pdu, tail, origin) + .get_missing_events( + room_id, + incoming_pdu, + tail, + origin, + self.services.metadata.get_mindepth(room_id).await, + ) .await?; debug_info!("Fetched {} missing events", backfilled.len()); diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index db947a803..ad81c0af1 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -64,6 +64,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( "Upgrading PDU from outlier to timeline" ); let timer = Instant::now(); + let min_depth = self.services.metadata.get_mindepth(room_id).await; let room_version_rules = get_room_version_rules(create_event)?; // 10. Fetch missing state and auth chain events by calling /state_ids at @@ -384,6 +385,11 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( // Event has passed all auth/stateres checks drop(state_lock); + if incoming_pdu.depth > min_depth { + self.services + .metadata + .set_mindepth(room_id, incoming_pdu.depth.into()); + } Ok(pdu_id) } diff --git a/src/service/rooms/membership/mod.rs b/src/service/rooms/membership/mod.rs index ff69e095e..539d63ea4 100644 --- a/src/service/rooms/membership/mod.rs +++ b/src/service/rooms/membership/mod.rs @@ -626,6 +626,10 @@ pub async fn join_remote_room( room_id, ) .await?; + self.services + .metadata + .maybe_set_mindepth(room_id, parsed_join_pdu.depth.into()) + .await; info!("Setting final room state for new room"); // We set the room state after inserting the pdu, so that we never have a moment diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 310ab2c76..da2112515 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -1,9 +1,9 @@ use std::sync::Arc; use conduwuit::{Result, implement, utils::stream::TryIgnore}; -use database::Map; +use database::{Deserialized, Map}; use futures::{Stream, StreamExt}; -use ruma::{OwnedRoomId, RoomId}; +use ruma::{OwnedRoomId, RoomId, UInt, uint}; use crate::{Dep, rooms}; @@ -17,6 +17,7 @@ struct Data { bannedroomids: Arc, roomid_shortroomid: Arc, pduid_pdu: Arc, + roomid_mindepth: Arc, } struct Services { @@ -31,6 +32,7 @@ fn build(args: crate::Args<'_>) -> Result> { bannedroomids: args.db["bannedroomids"].clone(), roomid_shortroomid: args.db["roomid_shortroomid"].clone(), pduid_pdu: args.db["pduid_pdu"].clone(), + roomid_mindepth: args.db["roomid_mindepth"].clone(), }, services: Services { short: args.depend::("rooms::short"), @@ -98,3 +100,27 @@ pub async fn is_disabled(&self, room_id: &RoomId) -> bool { pub async fn is_banned(&self, room_id: &RoomId) -> bool { self.db.bannedroomids.get(room_id).await.is_ok() } + +#[implement(Service)] +pub async fn get_mindepth(&self, room_id: &RoomId) -> UInt { + self.db + .roomid_mindepth + .get(room_id) + .await + .deserialized::() + .unwrap_or_else(|_| uint!(0)) +} + +#[implement(Service)] +pub fn set_mindepth(&self, room_id: &RoomId, min_depth: u64) { + self.db + .roomid_mindepth + .put_raw(room_id.as_bytes(), min_depth.to_be_bytes()); +} + +#[implement(Service)] +pub async fn maybe_set_mindepth(&self, room_id: &RoomId, min_depth: u64) { + if min_depth > self.get_mindepth(room_id).await.into() { + self.set_mindepth(room_id, min_depth); + } +} From e3ec1066c46d541ce8b4b162dd7db5a5206926d6 Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 26 May 2026 21:26:57 +0100 Subject: [PATCH 08/81] chore: Add newsfrag --- changelog.d/1818.feature.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog.d/1818.feature.md diff --git a/changelog.d/1818.feature.md b/changelog.d/1818.feature.md new file mode 100644 index 000000000..ce5c47254 --- /dev/null +++ b/changelog.d/1818.feature.md @@ -0,0 +1,2 @@ +Improved the performance and reliability of fetching missing events, improving network partition recovery. Contributed +by @nex. From b9dca84acf8e8a9ed6dbc6d0a94c21c68972457d Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 26 May 2026 23:29:31 +0100 Subject: [PATCH 09/81] fix: Only increment mindepth on state events --- src/service/rooms/event_handler/upgrade_outlier_pdu.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index ad81c0af1..3769bdd58 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -385,7 +385,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( // Event has passed all auth/stateres checks drop(state_lock); - if incoming_pdu.depth > min_depth { + if incoming_pdu.depth > min_depth && incoming_pdu.state_key().is_some() { self.services .metadata .set_mindepth(room_id, incoming_pdu.depth.into()); From f80e1e89a52b95a12eacfcb63a33c9fe69ee000f Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 26 May 2026 23:57:47 +0100 Subject: [PATCH 10/81] fix: Lower floor for min depth --- src/service/rooms/event_handler/fetch_prev.rs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 9be7b2e96..f25b90085 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -2,11 +2,11 @@ use conduwuit::{ Event, PduEvent, debug, debug_info, - utils::{BoolExt, IterStream, stream::BroadbandExt}, + utils::{BoolExt, IterStream, math::try_into, stream::BroadbandExt}, warn, }; use futures::StreamExt; -use ruma::{RoomId, ServerName}; +use ruma::{RoomId, ServerName, UInt}; use crate::rooms::event_handler::build_local_dag; @@ -45,19 +45,28 @@ pub(super) async fn fetch_prevs( .collect::>() .await; - let backfilled = self + let gapfilled = self .get_missing_events( room_id, incoming_pdu, tail, origin, - self.services.metadata.get_mindepth(room_id).await, + self.services + .metadata + .get_mindepth(room_id) + .await + .saturating_sub( + u8::try_from(incoming_pdu.prev_events.len()) + .unwrap() + .saturating_mul(2) + .into(), + ), ) .await?; - debug_info!("Fetched {} missing events", backfilled.len()); + debug_info!("Fetched {} missing events", gapfilled.len()); // Persist all fetched events - let mapped = backfilled + let mapped = gapfilled .iter() .map(|(eid, evt)| { let mut obj = evt.to_canonical_object(); From e8db01fc8d9630ae18e9477273298be0cc94f2a7 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 27 May 2026 00:25:56 +0100 Subject: [PATCH 11/81] style: Improve logging --- src/api/server/send.rs | 1 + src/service/rooms/event_handler/handle_incoming_pdu.rs | 1 - src/service/rooms/event_handler/upgrade_outlier_pdu.rs | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 6f6d35d10..cdea52eee 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -133,6 +133,7 @@ async fn wait_for_result( } #[instrument( + name="transaction" skip_all, fields( id = ?body.transaction_id.as_str(), diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 5033ab625..42065b343 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -108,7 +108,6 @@ async fn should_rescind_invite( #[implement(super::Service)] #[tracing::instrument( name = "pdu", - level = INFO_SPAN_LEVEL, skip_all, fields(%room_id, %event_id), )] diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 3769bdd58..b3f1d7363 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -389,6 +389,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( self.services .metadata .set_mindepth(room_id, incoming_pdu.depth.into()); + trace!("Increased room's min depth from {} to {}", min_depth, incoming_pdu.depth); } Ok(pdu_id) From 9ea9b0e04ccf8436affe61511e7f66cb2be5fc6b Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 27 May 2026 00:33:57 +0100 Subject: [PATCH 12/81] perf: Don't re-process events as outliers --- src/service/rooms/event_handler/handle_outlier_pdu.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 208f4aa08..01f5dba22 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -77,6 +77,13 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( incoming_pdu .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); + if let Ok(pdu_event) = self.services.timeline.get_pdu(event_id).await { + debug!( + "Already have event {event_id} as an outlier or timeline event, not re-processing" + ); + return Ok((pdu_event, incoming_pdu)); + } + let pdu_event = serde_json::from_value::( serde_json::to_value(&incoming_pdu).expect("CanonicalJsonObj is a valid JsonValue"), ) From d5f69c8a31a2148a2359ce8a7ad356e8e3213f83 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 27 May 2026 00:43:37 +0100 Subject: [PATCH 13/81] fix: Be smarter when re-receiving already-seen PDUs --- src/service/rooms/event_handler/fetch_prev.rs | 4 +- .../event_handler/handle_incoming_pdu.rs | 4 +- .../rooms/event_handler/handle_outlier_pdu.rs | 52 ++++++++++++------- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index f25b90085..25fdf9554 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -2,11 +2,11 @@ use conduwuit::{ Event, PduEvent, debug, debug_info, - utils::{BoolExt, IterStream, math::try_into, stream::BroadbandExt}, + utils::{BoolExt, IterStream, stream::BroadbandExt}, warn, }; use futures::StreamExt; -use ruma::{RoomId, ServerName, UInt}; +use ruma::{RoomId, ServerName}; use crate::rooms::event_handler::build_local_dag; diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 42065b343..d728370cf 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -1,8 +1,8 @@ use std::{collections::BTreeMap, time::Instant}; use conduwuit::{ - Err, Event, PduEvent, Result, debug::INFO_SPAN_LEVEL, debug_error, debug_info, defer, err, - implement, info, trace, warn, + Err, Event, PduEvent, Result, debug_error, debug_info, defer, err, implement, info, trace, + warn, }; use futures::{ FutureExt, diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 01f5dba22..e9b4ec191 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -7,7 +7,8 @@ use futures::future::ready; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, - api::federation::authorization::get_event_authorization, events::StateEventType, + api::federation::authorization::get_event_authorization, canonical_json::redact, + events::StateEventType, }; use super::{check_room_id, get_room_version_rules}; @@ -46,27 +47,45 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( .verify_event(&value, &room_version_rules) .await { - | Ok(ruma::signatures::Verified::All) => value, + | Ok(ruma::signatures::Verified::All) => { + if let Ok(pdu_event) = self.services.timeline.get_pdu(event_id).await { + debug!( + "Already have event {event_id} as an outlier or timeline event, not \ + re-processing" + ); + value.insert( + "event_id".to_owned(), + CanonicalJsonValue::String(event_id.as_str().to_owned()), + ); + check_room_id(room_id, &pdu_event)?; + return Ok((pdu_event, value)); + } + value + }, | Ok(ruma::signatures::Verified::Signatures) => { - // Redact debug_info!("Calculated hash does not match (redaction): {event_id}"); - let Ok(obj) = - ruma::canonical_json::redact(value, &room_version_rules.redaction, None) - else { - return Err!(Request(InvalidParam("Redaction failed"))); + let mut obj = match redact(value, &room_version_rules.redaction, None) { + | Ok(obj) => obj, + | Err(e) => return Err!(Request(BadJson("Failed to redact {event_id}: {e}"))), }; - // Skip the PDU if it is redacted and we already have it as an outlier event - if self.services.timeline.pdu_exists(event_id).await { - return Err!(Request(InvalidParam( - "Event was redacted and we already knew about it" - ))); + if let Ok(pdu_event) = self.services.timeline.get_pdu(event_id).await { + debug!( + "Received a redacted copy of {event_id}, but we already knew about it. \ + Re-using known content instead." + ); + obj.insert( + "event_id".to_owned(), + CanonicalJsonValue::String(event_id.as_str().to_owned()), + ); + check_room_id(room_id, &pdu_event)?; + return Ok((pdu_event, obj)); } obj }, | Err(e) => { - return Err!(Request(InvalidParam(debug_error!( + return Err!(Request(Forbidden(debug_error!( "Signature verification failed for {event_id}: {e}" )))); }, @@ -77,13 +96,6 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( incoming_pdu .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); - if let Ok(pdu_event) = self.services.timeline.get_pdu(event_id).await { - debug!( - "Already have event {event_id} as an outlier or timeline event, not re-processing" - ); - return Ok((pdu_event, incoming_pdu)); - } - let pdu_event = serde_json::from_value::( serde_json::to_value(&incoming_pdu).expect("CanonicalJsonObj is a valid JsonValue"), ) From 8edf9552b824f246c6b874e5e4a0aeb2705475eb Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 27 May 2026 00:51:12 +0100 Subject: [PATCH 14/81] fix: Don't lie about using already-known content --- .../rooms/event_handler/handle_outlier_pdu.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index e9b4ec191..64d1a9acd 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -63,26 +63,19 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( value }, | Ok(ruma::signatures::Verified::Signatures) => { - debug_info!("Calculated hash does not match (redaction): {event_id}"); - let mut obj = match redact(value, &room_version_rules.redaction, None) { - | Ok(obj) => obj, - | Err(e) => return Err!(Request(BadJson("Failed to redact {event_id}: {e}"))), - }; - if let Ok(pdu_event) = self.services.timeline.get_pdu(event_id).await { debug!( "Received a redacted copy of {event_id}, but we already knew about it. \ Re-using known content instead." ); - obj.insert( - "event_id".to_owned(), - CanonicalJsonValue::String(event_id.as_str().to_owned()), - ); check_room_id(room_id, &pdu_event)?; + let obj = pdu_event.to_canonical_object(); return Ok((pdu_event, obj)); } - obj + debug_info!("Calculated hash does not match (redaction): {event_id}"); + redact(value, &room_version_rules.redaction, None) + .map_err(|e| err!(Request(BadJson("Failed to redact {event_id}: {e}"))))? }, | Err(e) => { return Err!(Request(Forbidden(debug_error!( From aea03f2f99389631f003651320130f66d25b7c59 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 27 May 2026 02:43:08 +0100 Subject: [PATCH 15/81] perf(wip): Improve individual events fetcher --- .../fetch_and_handle_outliers.rs | 329 +++++++----------- .../rooms/event_handler/fetch_state.rs | 47 ++- .../event_handler/handle_incoming_pdu.rs | 2 - .../rooms/event_handler/parse_incoming_pdu.rs | 16 +- .../event_handler/upgrade_outlier_pdu.rs | 4 +- src/service/rooms/timeline/backfill.rs | 5 +- 6 files changed, 182 insertions(+), 221 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 1f5fc7e78..fec37ec28 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -1,26 +1,24 @@ -use std::{ - collections::{BTreeMap, HashMap, HashSet, VecDeque, hash_map}, - time::Instant, -}; +use std::collections::{HashMap, HashSet, VecDeque}; use assign::assign; use conduwuit::{ - Event, PduEvent, debug, debug_info, debug_warn, err, error, - matrix::event::gen_event_id_canonical_json, + Err, Event, PduEvent, debug, debug_info, debug_warn, err, error, state_res::lexicographical_topological_sort, trace, - utils::{IterStream, continue_exponential_backoff_secs, stream::BroadbandExt}, + utils::{IterStream, stream::BroadbandExt}, warn, }; -use futures::StreamExt; +use futures::{StreamExt, future::select_ok}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, - RoomId, ServerName, UInt, + OwnedServerName, RoomId, ServerName, UInt, api::federation::event::{get_event, get_missing_events}, int, + room_version_rules::RoomVersionRules, }; use super::get_room_version_rules; +use crate::rooms::event_handler::parse_incoming_pdu::expect_event_id_array; /// Attempts to build a localised directed acyclic graph out of the given PDUs, /// returning them in a topologically sorted order. @@ -154,7 +152,7 @@ pub async fn get_missing_events( }; let mut discovered = HashMap::with_capacity(20); - let mut latest_events = vec![head.event_id().to_owned()]; + let mut latest_events: Vec = vec![head.event_id().to_owned()]; let mut iterations = 0_u8; loop { iterations = iterations.saturating_add(1); @@ -235,222 +233,143 @@ pub async fn get_missing_events( Ok(discovered) } - /// Find the event and auth it. Once the event is validated (steps 1 - 8) - /// it is appended to the outliers Tree. - /// - /// Returns pdu and if we fetched it over federation the raw json. - /// - /// a. Look in the main timeline (pduid_pdu tree) - /// b. Look at outlier pdu tree - /// c. Ask origin server over federation - /// d. TODO: Ask other servers over federation? - #[deprecated] - pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>( + async fn fetch_and_handle_missing_event_via( + &self, + remote: OwnedServerName, + event_id: OwnedEventId, + room_version_rules: &RoomVersionRules, + ) -> conduwuit::Result<(OwnedEventId, CanonicalJsonObject)> { + let res = self + .services + .sending + .send_federation_request(&remote, get_event::v1::Request::new(event_id.clone())) + .await?; + + let (calculated_event_id, value) = self + .parse_incoming_pdu_with_known_room(&res.pdu, room_version_rules) + .await?; + + if calculated_event_id != event_id { + Err!(Request(BadJson(warn!( + expected=%event_id, + received=%calculated_event_id, + "Server didn't return event id we requested", + )))) + } else { + Ok((event_id, value)) + } + } + + /// Asks remote servers for any individual events that are missing. Should + /// only be used for fetching missing auth events or resolving missing + /// events from state_ids. For all other uses, use get_missing_events. + pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( &self, origin: &'a ServerName, - events: Events, + events: Vec, create_event: &'a Pdu, room_id: &'a RoomId, - ) -> Vec<(PduEvent, Option>)> + ) -> HashMap where Pdu: Event + Send + Sync, - Events: Iterator + Clone + Send, { - let back_off = |id| match self + let room_version_rules = + &get_room_version_rules(create_event).unwrap_or(RoomVersionRules::V1); + let mut candidates = self .services - .globals - .bad_event_ratelimiter - .write() - .entry(id) - { - | hash_map::Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - | hash_map::Entry::Occupied(mut e) => { - *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); - }, - }; - - let mut events_with_auth_events = Vec::with_capacity(events.clone().count()); - trace!("Fetching {} outlier pdus", events.clone().count()); + .timeline + .candidate_backfill_servers(room_id) + .await; + candidates.insert(origin.to_owned()); + let mut seeded_events = + HashMap::with_capacity(events.len() + (events.len().saturating_mul(3))); + trace!( + "Fetching {} unknown PDUs on demand from {} candidates", + events.len(), + candidates.len() + ); for id in events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { - trace!("Found {id} in main timeline or outlier tree"); - events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![])); - continue; - } - - // c. Ask origin server over federation - // We also handle its auth chain here so we don't get a stack overflow in - // handle_outlier_pdu. - let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into(); - let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); - - let mut events_all = HashSet::with_capacity(todo_auth_events.len()); - while let Some(next_id) = todo_auth_events.pop_front() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .get(&*next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 60 * 2; - const MAX_DURATION: u64 = 60 * 60 * 8; - if continue_exponential_backoff_secs( - MIN_DURATION, - MAX_DURATION, - time.elapsed(), - *tries, - ) { - debug_warn!( - tried = ?*tries, - elapsed = ?time.elapsed(), - "Backing off from {next_id}", - ); - continue; - } - } - - if events_all.contains(&next_id) { + let mut todo: VecDeque<_> = [id.clone()].into(); + while let Some(next_id) = todo.pop_front() { + if seeded_events.contains_key(&next_id) { continue; } - - if self.services.timeline.pdu_exists(&next_id).await { + if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { trace!("Found {next_id} in db"); + seeded_events.insert(id.clone(), local_pdu.into_canonical_object()); continue; } - debug!("Fetching {next_id} over federation from {origin}."); - match self - .services - .sending - .send_federation_request( - origin, - get_event::v1::Request::new((*next_id).to_owned()), - ) - .await - { - | Ok(res) => { - debug!("Got {next_id} over federation from {origin}"); - let Ok(room_version_rules) = get_room_version_rules(create_event) else { - back_off((*next_id).to_owned()); - continue; - }; - - let Ok((calculated_event_id, value)) = - gen_event_id_canonical_json(&res.pdu, &room_version_rules) - else { - back_off((*next_id).to_owned()); - continue; - }; - - if calculated_event_id != *next_id { - warn!( - "Server didn't return event id we requested: requested: \ - {next_id}, we got {calculated_event_id}. Event: {:?}", - &res.pdu - ); - } - - if let Some(auth_events) = value - .get("auth_events") - .and_then(CanonicalJsonValue::as_array) - { - for auth_event in auth_events { - match serde_json::from_value::( - auth_event.clone().into(), - ) { - | Ok(auth_event) => { - trace!( - "Found auth event id {auth_event} for event \ - {next_id}" - ); - todo_auth_events.push_back(auth_event); - }, - | _ => { - warn!("Auth event id is not valid"); - }, - } - } - } else { - warn!("Auth event list invalid"); - } - - events_in_reverse_order.push((next_id.clone(), value)); - events_all.insert(next_id); - }, + debug!("Fetching {next_id} over federation"); + let futures = candidates + .iter() + .map(|remote| { + Box::pin(self.fetch_and_handle_missing_event_via( + remote.clone(), + next_id.clone(), + room_version_rules, + )) + }) + .collect::>(); + let (event_id, value) = match select_ok(futures).await { + | Ok((x, _)) => x, | Err(e) => { - warn!("Failed to fetch auth event {next_id} from {origin}: {e}"); - back_off((*next_id).to_owned()); - }, - } - } - - events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order)); - } - - let mut pdus = Vec::with_capacity(events_with_auth_events.len()); - for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Some(local_pdu) = local_pdu { - trace!("Found {id} in main timeline or outlier tree"); - pdus.push((local_pdu.clone(), None)); - } - - for (next_id, value) in events_in_reverse_order.into_iter().rev() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .get(&*next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs( - MIN_DURATION, - MAX_DURATION, - time.elapsed(), - *tries, - ) { - debug!("Backing off from {next_id}"); + warn!("failed to fetch missing event {next_id} from any candidate: {e}"); continue; - } - } - - trace!("Handling outlier {next_id}"); - match Box::pin(self.handle_outlier_pdu( - origin, - create_event, - &next_id, - room_id, - value.clone(), - true, - )) - .await - { - | Ok((pdu, json)) => - if next_id == *id { - trace!("Handled outlier {next_id} (original request)"); - pdus.push((pdu, Some(json))); - }, - | Err(e) => { - warn!("Authentication of event {next_id} failed: {e:?}"); - back_off(next_id); }, - } + }; + let auth_events = + match expect_event_id_array(&value, "auth_events").map_err(|e| { + err!(Request(BadJson(warn!( + %event_id, + "Failed to parse event fetched from remote: {e}" + )))) + }) { + | Ok(auth_events) => auth_events, + | Err(e) => { + warn!( + ?e, + "event {event_id} is malformed (bad auth_events), skipping" + ); + continue; + }, + }; + todo.extend(auth_events); + seeded_events.insert(event_id, value); } } - trace!("Fetched and handled {} outlier pdus", pdus.len()); + + let seeded_ordered = build_local_dag( + &seeded_events + .iter() + .map(|(eid, e)| (eid.to_owned(), e.clone())) + .collect::>(), + ) + .await + .expect("failed to build local DAG"); + let mut pdus = HashMap::with_capacity(seeded_ordered.len()); + for id in seeded_ordered { + let pdu_json = seeded_events.remove(&id).unwrap(); + trace!("Handling missing event {id}"); + match Box::pin(self.handle_outlier_pdu( + origin, + create_event, + &id, + room_id, + pdu_json, + true, + )) + .await + { + | Ok((pdu, _)) => { + let _ = pdus.insert(id, pdu); + }, + | Err(e) => warn!("Authentication of event {id} failed: {e:?}"), + } + } + + trace!("Fetched and handled {} missing PDUs", pdus.len()); pdus } } diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index d6c20518c..e5bc78cad 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,12 +1,15 @@ use std::collections::{HashMap, hash_map}; -use conduwuit::{Err, Event, Result, debug, debug_warn, err, implement}; +use conduwuit::{ + Err, Event, PduEvent, Result, debug, debug_warn, err, implement, utils::IterStream, +}; +use futures::StreamExt; use ruma::{ EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids, events::StateEventType, }; -use crate::rooms::short::ShortStateKey; +use crate::{conduwuit::utils::stream::BroadbandExt, rooms::short::ShortStateKey}; /// Call /state_ids to find out what the state at this pdu is. We trust the /// server's response to some extend (sic), but we still do a lot of checks @@ -27,7 +30,7 @@ pub(super) async fn fetch_state( where Pdu: Event + Send + Sync, { - let res = self + let res: get_room_state_ids::v1::Response = self .services .sending .send_federation_request( @@ -38,16 +41,39 @@ pub(super) async fn fetch_state( .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; debug!("Fetching state events"); - let state_ids = res.pdu_ids.iter().map(AsRef::as_ref); - let state_vec = self - .fetch_and_handle_outliers(origin, state_ids, create_event, room_id) + let mut state_events: HashMap = + HashMap::with_capacity(res.pdu_ids.len()); + let to_fetch: Vec = res + .pdu_ids + .clone() + .into_iter() + .stream() + .broad_filter_map(|event_id| async move { + if self.services.timeline.pdu_exists(&event_id).await { + None + } else { + Some(event_id) + } + }) + .collect() .await; + if !to_fetch.is_empty() { + if to_fetch.len() >= 100 { + // That's a lot of events to fetch, just ask for the full state at + // that point. TODO: fetch /state + } + state_events.extend( + self.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) + .await, + ); + } - let mut state: HashMap = HashMap::with_capacity(state_vec.len()); - for (pdu, _) in state_vec { + let mut state: HashMap = + HashMap::with_capacity(state_events.len()); + for (event_id, pdu) in state_events { let state_key = pdu .state_key() - .ok_or_else(|| err!(Database("Found non-state pdu in state events.")))?; + .ok_or_else(|| err!(Database("Found non-state pdu in state events: {event_id}")))?; let shortstatekey = self .services @@ -61,7 +87,8 @@ pub(super) async fn fetch_state( }, | hash_map::Entry::Occupied(_) => { return Err!(Database( - "State event's type and state_key combination exists multiple times: {}, {}", + "State event's type and state_key combination exists multiple times \ + ({event_id}): {}, {}", pdu.kind(), state_key )); diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index d728370cf..305df60e9 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -15,7 +15,6 @@ room::member::{MembershipState, RoomMemberEventContent}, }, }; -use tracing::debug; use crate::rooms::timeline::{RawPduId, pdu_fits}; @@ -241,7 +240,6 @@ pub async fn handle_incoming_pdu<'a>( // 9. Fetch any missing prev events doing all checks listed here starting at 1. // These are timeline events - debug!("Handling previous events"); self.fetch_prevs(room_id, create_event, &incoming_pdu, origin) .await?; diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 34a33d83d..2e7281b3c 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -7,7 +7,7 @@ use itertools::Itertools; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, RoomId, - RoomVersionId, + RoomVersionId, room_version_rules::RoomVersionRules, }; use serde_json::value::RawValue as RawJsonValue; @@ -103,6 +103,20 @@ pub fn validate_pdu(&self, pdu: &CanonicalJsonObject) -> Result { Ok(()) } +#[implement(super::Service)] +pub async fn parse_incoming_pdu_with_known_room( + &self, + pdu: &RawJsonValue, + room_version_rules: &RoomVersionRules, +) -> Result<(OwnedEventId, CanonicalJsonObject)> { + let (event_id, value) = + gen_event_id_canonical_json(pdu, room_version_rules).map_err(|e| { + err!(Request(InvalidParam("Could not convert event to canonical json: {e}"))) + })?; + self.validate_pdu(&value)?; + Ok((event_id, value)) +} + #[implement(super::Service)] pub async fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result { let value = serde_json::from_str::(pdu.get()).map_err(|e| { diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index b3f1d7363..a96ee594b 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -54,9 +54,9 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( .is_event_soft_failed(incoming_pdu.event_id()) ); if rejected { - return Err!(Request(InvalidParam("Event has been rejected"))); + return Err!(Request(Forbidden("Event has been rejected"))); } else if soft_failed { - return Err!(Request(InvalidParam("Event has been soft-failed"))); + return Err!(Request(Forbidden("Event has been soft-failed"))); } debug!( diff --git a/src/service/rooms/timeline/backfill.rs b/src/service/rooms/timeline/backfill.rs index 9930cf466..c951d7293 100644 --- a/src/service/rooms/timeline/backfill.rs +++ b/src/service/rooms/timeline/backfill.rs @@ -221,7 +221,10 @@ pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> } #[implement(super::Service)] -async fn candidate_backfill_servers(&self, room_id: &RoomId) -> HashSet { +pub(crate) async fn candidate_backfill_servers( + &self, + room_id: &RoomId, +) -> HashSet { let mut candidate_backfill_servers = HashSet::new(); let power_levels = self From 7f64de9727a0b0d90a32c737638e1811206ce390 Mon Sep 17 00:00:00 2001 From: timedout Date: Thu, 28 May 2026 17:42:01 +0100 Subject: [PATCH 16/81] fix: Calculate max iterations dynamically, and bump max prevs --- conduwuit-example.toml | 2 +- src/core/config/mod.rs | 4 +- .../fetch_and_handle_outliers.rs | 83 ++++++++++++++++--- 3 files changed, 73 insertions(+), 16 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index a4d60bbc7..cc736826b 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -297,7 +297,7 @@ # This item is undocumented. Please contribute documentation for it. # -#max_fetch_prev_events = 256 +#max_fetch_prev_events = 1024 # How many incoming federation transactions the server is willing to be # processing at any given time before it becomes overloaded and starts diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 6ee251bf8..22c47e67d 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -375,7 +375,7 @@ pub struct Config { #[serde(default = "default_max_request_size")] pub max_request_size: usize, - /// default: 256 + /// default: 1024 #[serde(default = "default_max_fetch_prev_events")] pub max_fetch_prev_events: u16, @@ -2549,7 +2549,7 @@ fn default_pusher_timeout() -> u64 { 60 } fn default_pusher_idle_timeout() -> u64 { 15 } -fn default_max_fetch_prev_events() -> u16 { 256_u16 } +fn default_max_fetch_prev_events() -> u16 { 1024 } fn default_max_concurrent_inbound_transactions() -> usize { 150 } diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index fec37ec28..3e01defd8 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -20,6 +20,8 @@ use super::get_room_version_rules; use crate::rooms::event_handler::parse_incoming_pdu::expect_event_id_array; +const GET_MISSING_EVENTS_MAX_BATCH_SIZE: u16 = 50; // matches src/server/get_missing_events.rs#LIMIT_MAX + /// Attempts to build a localised directed acyclic graph out of the given PDUs, /// returning them in a topologically sorted order. /// @@ -150,14 +152,39 @@ pub async fn get_missing_events( "event passed to get_missing_events is not missing any events (wasteful call)" ); }; + assert!(!tail.is_empty(), "empty tail"); + assert_ne!(via, self.services.globals.server_name(), "cannot ask ourselves for events"); - let mut discovered = HashMap::with_capacity(20); + // The iteration limit is in place to ensure that if the remote server leaves us + // in a state of infinite recursion (as old versions of continuwuity and + // predecessors would), we give up. However, get_missing_events doesn't return + // that many events per-request. Synapse returns 20, and conduwuit+ return 50. + // This means with a hard iteration limit, we might give up too early, before + // we get a chance to even come close to max_fetch_prev_events. As such, we'll + // calculate the max limit based on that config option based on these averages. + let max_fetch = self.services.server.config.max_fetch_prev_events; + let iteration_limit = max_fetch.saturating_div(20).max(10); + + let mut discovered = HashMap::with_capacity(head.prev_events.len()); let mut latest_events: Vec = vec![head.event_id().to_owned()]; - let mut iterations = 0_u8; - loop { - iterations = iterations.saturating_add(1); - let limit = iterations.saturating_mul(10).min(100); - debug_info!(%limit, %via, %iterations, discovered=discovered.len(), %min_depth, "Attempting to gap fill missing events"); + debug!( + %room_id, + event_id=%head.event_id(), + %iteration_limit, + "Fetching any missing events for head event", + ); + for iteration in 0..iteration_limit { + let limit = iteration + .saturating_mul(10) + .min(GET_MISSING_EVENTS_MAX_BATCH_SIZE); + debug_info!( + %limit, + %via, + %iteration, + discovered=discovered.len(), + %min_depth, + "Attempting to gap fill missing events" + ); let response: get_missing_events::v1::Response = self .services .sending @@ -193,11 +220,13 @@ pub async fn get_missing_events( pdu.depth, min_depth ); + discovered.insert(event_id.clone(), pdu); continue; } for prev_event_id in pdu.prev_events() { if discovered.contains_key(prev_event_id) { + // We already received this event. continue; } if self @@ -206,25 +235,53 @@ pub async fn get_missing_events( .non_outlier_pdu_exists(prev_event_id) .await { + // NOTE: we explicitly check for *non*-outlier events here, as if we end + // up discovering outlier events, we will be able to upgrade them + // immediately. continue; } latest_events.push(prev_event_id.to_owned()); - break; } discovered.insert(event_id.clone(), pdu); } if latest_events.is_empty() { + debug!( + %limit, + %via, + %iteration, + discovered=discovered.len(), + "No more events to fetch." + ); break; - } else if discovered.len() > self.services.server.config.max_fetch_prev_events.into() - || iterations >= 20 - { + } + if discovered.len() >= self.services.server.config.max_fetch_prev_events.into() { + // Stupid hack, debug_error!() drops the log to a DEBUG when not in debug mode, + // which is bad because this should at least produce a warning. It's an error in + // debug mode because this can be important, but typically not much can be done + // about it as a user. + #[cfg(debug_assertions)] error!( - filled=discovered.len(), + discovered=discovered.len(), max_fetch_prev_events=self.services.server.config.max_fetch_prev_events, - %iterations, - "Gap too large, giving up" + %iteration, + %iteration_limit, + %via, + event_id=%head.event_id(), + %room_id, + "Encountered a gap too large to fill, giving up" + ); + #[cfg(not(debug_assertions))] + warn!( + discovered=discovered.len(), + max_fetch_prev_events=self.services.server.config.max_fetch_prev_events, + %iteration, + %iteration_limit, + %via, + event_id=%head.event_id(), + %room_id, + "Encountered a gap too large to fill" ); break; } From 2bcc56704b0eebdcde822265fb3e88b04020eeae Mon Sep 17 00:00:00 2001 From: timedout Date: Thu, 28 May 2026 18:38:27 +0100 Subject: [PATCH 17/81] feat: Enhance reliability by fetching full state when we're missing a lot of auth events --- .../fetch_and_handle_outliers.rs | 38 ++- .../rooms/event_handler/fetch_state.rs | 292 ++++++++++++------ .../event_handler/upgrade_outlier_pdu.rs | 9 +- 3 files changed, 241 insertions(+), 98 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 3e01defd8..287ee9553 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -338,8 +338,10 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( .candidate_backfill_servers(room_id) .await; candidates.insert(origin.to_owned()); + candidates.retain(|sn| self.services.globals.server_name() != sn); + assert_ne!(candidates.len(), 0, "no candidates to fetch missing events from"); let mut seeded_events = - HashMap::with_capacity(events.len() + (events.len().saturating_mul(3))); + HashMap::with_capacity(events.len().saturating_add(events.len().saturating_mul(3))); trace!( "Fetching {} unknown PDUs on demand from {} candidates", events.len(), @@ -392,8 +394,32 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( continue; }, }; - todo.extend(auth_events); - seeded_events.insert(event_id, value); + let mut have_all_auth = true; + for auth_event_id in auth_events { + if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { + trace!("Found auth event {next_id} in db"); + seeded_events.insert(id.clone(), local_pdu.into_canonical_object()); + continue; + } + if seeded_events.contains_key(&auth_event_id) { + trace!(%auth_event_id, "Already found auth event"); + continue; + } + todo.push_back(auth_event_id); + have_all_auth = false; + } + // Insert this PDU back at the end of the queue so that it will be resolved once + // all of its auth events have been fetched. + // TODO: This may result in infinite looping, needs a breaker + if have_all_auth { + debug!(%event_id, "Have all auth events"); + seeded_events.insert(event_id, value); + } else { + debug_warn!( + "Fetched {event_id} but missing some auth events, will have to re-fetch." + ); + todo.push_back(event_id); + } } } @@ -408,7 +434,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( let mut pdus = HashMap::with_capacity(seeded_ordered.len()); for id in seeded_ordered { let pdu_json = seeded_events.remove(&id).unwrap(); - trace!("Handling missing event {id}"); + debug_info!("Handling missing event {id} as outlier"); match Box::pin(self.handle_outlier_pdu( origin, create_event, @@ -424,6 +450,10 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( }, | Err(e) => warn!("Authentication of event {id} failed: {e:?}"), } + + // TODO: should this try to promote to timeline? + // If we got here, we probably weren't able to promote it before + // now. } trace!("Fetched and handled {} missing PDUs", pdus.len()); diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index e5bc78cad..ea7ed3c3b 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,111 +1,227 @@ use std::collections::{HashMap, hash_map}; -use conduwuit::{ - Err, Event, PduEvent, Result, debug, debug_warn, err, implement, utils::IterStream, -}; +use conduwuit::{Err, Event, PduEvent, Result, debug, debug_warn, err, utils::IterStream, warn}; use futures::StreamExt; use ruma::{ - EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids, + EventId, OwnedEventId, RoomId, ServerName, + api::federation::event::{get_room_state, get_room_state_ids}, events::StateEventType, }; use crate::{conduwuit::utils::stream::BroadbandExt, rooms::short::ShortStateKey}; -/// Call /state_ids to find out what the state at this pdu is. We trust the -/// server's response to some extend (sic), but we still do a lot of checks -/// on the events -#[implement(super::Service)] -#[tracing::instrument( - level = "debug", - skip_all, - fields(%origin), -)] -pub(super) async fn fetch_state( - &self, - origin: &ServerName, - create_event: &Pdu, - room_id: &RoomId, - event_id: &EventId, -) -> Result>> -where - Pdu: Event + Send + Sync, -{ - let res: get_room_state_ids::v1::Response = self - .services - .sending - .send_federation_request( - origin, - get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()), - ) - .await - .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; +impl super::Service { + /// Asks a remote server what the state at this event is. + /// It first attempts to call `GET /_matrix/federation/v1/state_ids` (fast). + /// If any events are missing, they are fetched from the remote, and + /// persisted as outliers, before being returned back to this function. If + /// we are missing a lot of events locally (>=50), this function falls back + /// to requesting the full state in PDU format from the remote (`GET + /// /_matrix/federation/v1/state, very slow in large rooms), and persists + /// them directly. + /// + /// The end result is a result containing a map of shortstatekeys to event + /// IDs. The underlying `Option` is always `Some`. + #[tracing::instrument(skip(self, create_event))] + pub(super) async fn fetch_state( + &self, + origin: &ServerName, + create_event: &PduEvent, + room_id: &RoomId, + event_id: &EventId, + ) -> Result>> { + let res: get_room_state_ids::v1::Response = self + .services + .sending + .send_federation_request( + origin, + get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()), + ) + .await + .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; - debug!("Fetching state events"); - let mut state_events: HashMap = - HashMap::with_capacity(res.pdu_ids.len()); - let to_fetch: Vec = res - .pdu_ids - .clone() - .into_iter() - .stream() - .broad_filter_map(|event_id| async move { - if self.services.timeline.pdu_exists(&event_id).await { - None + debug!("Fetching state events"); + let mut state_events: HashMap = + HashMap::with_capacity(res.pdu_ids.len()); + let to_fetch: Vec = res + .pdu_ids + .clone() + .into_iter() + .stream() + .broad_filter_map(|event_id| async move { + if self.services.timeline.pdu_exists(&event_id).await { + None + } else { + Some(event_id) + } + }) + .collect() + .await; + if !to_fetch.is_empty() { + if to_fetch.len() >= 100 { + // That's a lot of events to fetch, just ask for the full state + // at that point. + debug_warn!( + to_fetch = to_fetch.len(), + "Fetching full state from remote server for event" + ); + state_events.extend( + self.fetch_full_state(origin, create_event, room_id, event_id) + .await?, + ); } else { - Some(event_id) + debug!( + to_fetch = to_fetch.len(), + "Fetching missing events for state from remote" + ); + state_events.extend( + self.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) + .await, + ); } - }) - .collect() - .await; - if !to_fetch.is_empty() { - if to_fetch.len() >= 100 { - // That's a lot of events to fetch, just ask for the full state at - // that point. TODO: fetch /state } - state_events.extend( - self.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) - .await, - ); - } - let mut state: HashMap = - HashMap::with_capacity(state_events.len()); - for (event_id, pdu) in state_events { - let state_key = pdu - .state_key() - .ok_or_else(|| err!(Database("Found non-state pdu in state events: {event_id}")))?; + let mut state: HashMap = + HashMap::with_capacity(state_events.len()); + for (event_id, pdu) in state_events { + let state_key = pdu.state_key().ok_or_else(|| { + err!(Database("Found non-state pdu in state events: {event_id}")) + })?; - let shortstatekey = self + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&pdu.kind().to_string().into(), state_key) + .await; + + match state.entry(shortstatekey) { + | hash_map::Entry::Vacant(v) => { + v.insert(pdu.event_id().to_owned()); + }, + | hash_map::Entry::Occupied(_) => { + return Err!(Database( + "State event's type and state_key combination exists multiple times \ + ({event_id}): {}, {}", + pdu.kind(), + state_key + )); + }, + } + } + + // The original create event must still be in the state + let create_shortstatekey = self .services .short - .get_or_create_shortstatekey(&pdu.kind().to_string().into(), state_key) - .await; + .get_shortstatekey(&StateEventType::RoomCreate, "") + .await?; - match state.entry(shortstatekey) { - | hash_map::Entry::Vacant(v) => { - v.insert(pdu.event_id().to_owned()); - }, - | hash_map::Entry::Occupied(_) => { - return Err!(Database( - "State event's type and state_key combination exists multiple times \ - ({event_id}): {}, {}", - pdu.kind(), - state_key - )); - }, + if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(create_event.event_id()) { + return Err!(Database("Incoming event refers to wrong create event.")); } + + Ok(Some(state)) } - // The original create event must still be in the state - let create_shortstatekey = self - .services - .short - .get_shortstatekey(&StateEventType::RoomCreate, "") - .await?; - - if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(create_event.event_id()) { - return Err!(Database("Incoming event refers to wrong create event.")); + /// Fetches the full state via `GET /_matrix/federation/v1/state` from a + /// remote server, and persists all the incoming auth chain events and + /// state events as outliers, for use later. + /// + /// Any events that cannot be persisted are dropped with a warning. + /// TODO: make it noisy? + pub(super) async fn fetch_full_state( + &self, + origin: &ServerName, + create_event: &PduEvent, + room_id: &RoomId, + event_id: &EventId, + ) -> Result> { + let res: get_room_state::v1::Response = self + .services + .sending + .send_federation_request( + origin, + get_room_state::v1::Request::new(event_id.to_owned(), room_id.to_owned()), + ) + .await + .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; + debug!(count = res.auth_chain.len(), "Handling incoming auth chain..."); + res.auth_chain + .iter() + .stream() + .broad_filter_map(|raw_event_json| async { + if let Some(parsed) = self.parse_incoming_pdu(raw_event_json).await.ok() + && parsed.0 == room_id + { + Some(parsed) + } else { + None + } + }) + .for_each_concurrent( + None, + |(incoming_room_id, incoming_event_id, incoming_event_json)| async move { + self.handle_outlier_pdu( + origin, + create_event, + &incoming_event_id, + &incoming_room_id, + incoming_event_json, + true, + ) + .await + .inspect_err(|e| { + warn!( + %incoming_room_id, + %incoming_event_id, + ?e, + "Failed to handle auth chain event from state fetch" + ); + }) + .ok(); + }, + ) + .await; + debug!(count = res.pdus.len(), "Handling incoming state PDUs..."); + Ok(res + .pdus + .iter() + .stream() + .broad_filter_map(|raw_event_json| async { + if let Some(parsed) = self.parse_incoming_pdu(raw_event_json).await.ok() + && parsed.0 == room_id + { + Some(parsed) + } else { + None + } + }) + .broad_filter_map( + |(incoming_room_id, incoming_event_id, incoming_event_json)| async move { + self.handle_outlier_pdu( + origin, + create_event, + &incoming_event_id, + &incoming_room_id, + incoming_event_json, + true, + ) + .await + .inspect_err(|e| { + warn!( + %incoming_room_id, + %incoming_event_id, + ?e, + "Failed to handle state event from state fetch" + ); + }) + .ok() + }, + ) + .fold(HashMap::new(), |mut acc, (event, _)| async move { + acc.insert(event.event_id().to_owned(), event); + acc + }) + .await) } - - Ok(Some(state)) } diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index a96ee594b..253a6905e 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -23,17 +23,14 @@ }; #[implement(super::Service)] -pub(super) async fn upgrade_outlier_to_timeline_pdu( +pub(super) async fn upgrade_outlier_to_timeline_pdu( &self, incoming_pdu: PduEvent, mut val: CanonicalJsonObject, - create_event: &Pdu, + create_event: &PduEvent, origin: &ServerName, room_id: &RoomId, -) -> Result> -where - Pdu: Event + Send + Sync, -{ +) -> Result> { // Skip the PDU if we already have it as a timeline event if let Ok(pduid) = self .services From 316a0b7d58a41f164df19c7cf158ca388b8a0119 Mon Sep 17 00:00:00 2001 From: timedout Date: Thu, 28 May 2026 18:46:27 +0100 Subject: [PATCH 18/81] fix: Don't repeat already-included metadata in fetch_state instrument --- src/service/rooms/event_handler/fetch_and_handle_outliers.rs | 2 +- src/service/rooms/event_handler/fetch_state.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 287ee9553..c7faa14fb 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -29,7 +29,7 @@ /// dependencies, however it is ultimately the sender's responsibility to send /// them in a processable order, so this is just a best effort attempt. It does /// not account for power levels or other tie breaks. -pub async fn build_local_dag( +pub async fn build_local_dag( pdu_map: &HashMap, ) -> conduwuit::Result> { debug_assert!(pdu_map.len() >= 2, "needless call to build_local_dag with less than 2 PDUs"); diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index ea7ed3c3b..f4e19fe96 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -22,7 +22,7 @@ impl super::Service { /// /// The end result is a result containing a map of shortstatekeys to event /// IDs. The underlying `Option` is always `Some`. - #[tracing::instrument(skip(self, create_event))] + #[tracing::instrument(skip_all)] pub(super) async fn fetch_state( &self, origin: &ServerName, From cef4ebe38eb4c4bc5d39ba4362f1bc3dd938acac Mon Sep 17 00:00:00 2001 From: timedout Date: Thu, 28 May 2026 19:30:17 +0100 Subject: [PATCH 19/81] fix: Don't try to fetch the same event endlessly --- .../fetch_and_handle_outliers.rs | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index c7faa14fb..d80aa9d9f 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -2,7 +2,7 @@ use assign::assign; use conduwuit::{ - Err, Event, PduEvent, debug, debug_info, debug_warn, err, error, + Err, Event, PduEvent, debug, debug_error, debug_info, debug_warn, err, error, state_res::lexicographical_topological_sort, trace, utils::{IterStream, stream::BroadbandExt}, @@ -348,15 +348,21 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( candidates.len() ); + let mut seen: HashMap = HashMap::new(); for id in events { - let mut todo: VecDeque<_> = [id.clone()].into(); + let mut todo: VecDeque = [id.clone()].into(); while let Some(next_id) = todo.pop_front() { if seeded_events.contains_key(&next_id) { continue; } if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { trace!("Found {next_id} in db"); - seeded_events.insert(id.clone(), local_pdu.into_canonical_object()); + seeded_events.insert(next_id.clone(), local_pdu.into_canonical_object()); + continue; + } + let attempts = seen.get(&*next_id).copied().unwrap_or_default(); + if attempts >= 5 { + debug_error!(%attempts, %next_id, "Could not fetch missing event after 5 attempts, giving up"); continue; } @@ -405,6 +411,8 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( trace!(%auth_event_id, "Already found auth event"); continue; } + debug!("Missing auth event {auth_event_id} for event {next_id}"); + seen.insert(auth_event_id.clone(), attempts.saturating_add(1)); todo.push_back(auth_event_id); have_all_auth = false; } @@ -412,13 +420,14 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( // all of its auth events have been fetched. // TODO: This may result in infinite looping, needs a breaker if have_all_auth { - debug!(%event_id, "Have all auth events"); - seeded_events.insert(event_id, value); + debug!(%next_id, "Have all auth events"); + seeded_events.insert(next_id, value); } else { debug_warn!( - "Fetched {event_id} but missing some auth events, will have to re-fetch." + "Fetched {next_id} but missing some auth events, will have to re-fetch." ); - todo.push_back(event_id); + seen.insert(next_id.clone(), attempts.saturating_add(1)); + todo.push_back(next_id); } } } From 162e6eb92fef40614b5fd4ae101cefc1cbcd93e1 Mon Sep 17 00:00:00 2001 From: timedout Date: Thu, 28 May 2026 19:59:33 +0100 Subject: [PATCH 20/81] fix: Remove short-term memory loss I keep writing forgetful code, it's a problem --- .../rooms/event_handler/fetch_state.rs | 66 ++++++++++++------- .../event_handler/upgrade_outlier_pdu.rs | 8 ++- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index f4e19fe96..710ee6254 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,6 +1,8 @@ use std::collections::{HashMap, hash_map}; -use conduwuit::{Err, Event, PduEvent, Result, debug, debug_warn, err, utils::IterStream, warn}; +use conduwuit::{ + Err, Event, PduEvent, Result, debug, debug_warn, err, trace, utils::IterStream, warn, +}; use futures::StreamExt; use ruma::{ EventId, OwnedEventId, RoomId, ServerName, @@ -30,6 +32,7 @@ pub(super) async fn fetch_state( room_id: &RoomId, event_id: &EventId, ) -> Result>> { + trace!(%origin, "Asking remote for state_ids"); let res: get_room_state_ids::v1::Response = self .services .sending @@ -40,7 +43,7 @@ pub(super) async fn fetch_state( .await .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; - debug!("Fetching state events"); + debug!(events = res.pdu_ids.len(), "Fetching state events"); let mut state_events: HashMap = HashMap::with_capacity(res.pdu_ids.len()); let to_fetch: Vec = res @@ -57,32 +60,46 @@ pub(super) async fn fetch_state( }) .collect() .await; - if !to_fetch.is_empty() { - if to_fetch.len() >= 100 { - // That's a lot of events to fetch, just ask for the full state - // at that point. - debug_warn!( - to_fetch = to_fetch.len(), - "Fetching full state from remote server for event" - ); - state_events.extend( - self.fetch_full_state(origin, create_event, room_id, event_id) - .await?, - ); - } else { - debug!( - to_fetch = to_fetch.len(), - "Fetching missing events for state from remote" - ); - state_events.extend( - self.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) - .await, - ); - } + if to_fetch.is_empty() { + debug!("All required state events are already known."); + state_events = res + .pdu_ids + .iter() + .stream() + .broad_filter_map(|event_id| async move { + Some(( + event_id.clone(), + self.services + .timeline + .get_pdu(event_id) + .await + .expect("Event disappeared between filtering and fetching"), + )) + }) + .collect() + .await; + } else if to_fetch.len() >= 100 { + // That's a lot of events to fetch, just ask for the full state + // at that point. + debug_warn!( + to_fetch = to_fetch.len(), + "Fetching full state from remote server for event" + ); + state_events.extend( + self.fetch_full_state(origin, create_event, room_id, event_id) + .await?, + ); + } else { + debug!(to_fetch = to_fetch.len(), "Fetching missing events for state from remote"); + state_events.extend( + self.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) + .await, + ); } let mut state: HashMap = HashMap::with_capacity(state_events.len()); + debug!(events = state_events.len(), "Processing state events"); for (event_id, pdu) in state_events { let state_key = pdu.state_key().ok_or_else(|| { err!(Database("Found non-state pdu in state events: {event_id}")) @@ -136,6 +153,7 @@ pub(super) async fn fetch_full_state( room_id: &RoomId, event_id: &EventId, ) -> Result> { + trace!("Fetching full state from remote server"); let res: get_room_state::v1::Response = self .services .sending diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 253a6905e..89966b43b 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -1,8 +1,9 @@ use std::{borrow::Borrow, sync::Arc, time::Instant}; use conduwuit::{ - Err, Result, debug, debug_info, err, implement, info, is_equal_to, + Err, Result, debug, debug_error, debug_info, err, implement, info, is_equal_to, matrix::{Event, EventTypeExt, PduEvent, StateKey, state_res}, + result::DebugInspect, trace, utils::{ IterStream, @@ -83,11 +84,14 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( trace!("Could not calculate incoming state, asking remote {origin} for it"); state_at_incoming_event = self .fetch_state(origin, create_event, room_id, incoming_pdu.event_id()) - .await?; + .await + .debug_inspect_err(|e| debug_error!("Could not fetch state from {origin}: {e}"))?; } let state_at_incoming_event = state_at_incoming_event.expect("we always set this to some above"); + assert!(!state_at_incoming_event.is_empty(), "Event has no incoming state"); + trace!(state_events = state_at_incoming_event.len(), "Calculated incoming state"); debug!( event_id = %incoming_pdu.event_id, From 573d5bc50e7db244f0e7336257bae655646b8675 Mon Sep 17 00:00:00 2001 From: timedout Date: Thu, 28 May 2026 21:13:42 +0100 Subject: [PATCH 21/81] fix: Fall back to atomic fetch when full-state fetch fails --- .../rooms/event_handler/fetch_state.rs | 67 ++++++++++++++----- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 710ee6254..f2601856f 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,7 +1,10 @@ -use std::collections::{HashMap, hash_map}; +use std::{ + cmp::max, + collections::{HashMap, hash_map}, +}; use conduwuit::{ - Err, Event, PduEvent, Result, debug, debug_warn, err, trace, utils::IterStream, warn, + Err, Event, PduEvent, Result, debug, debug_warn, err, error, trace, utils::IterStream, warn, }; use futures::StreamExt; use ruma::{ @@ -78,23 +81,51 @@ pub(super) async fn fetch_state( }) .collect() .await; - } else if to_fetch.len() >= 100 { - // That's a lot of events to fetch, just ask for the full state - // at that point. - debug_warn!( - to_fetch = to_fetch.len(), - "Fetching full state from remote server for event" - ); - state_events.extend( - self.fetch_full_state(origin, create_event, room_id, event_id) - .await?, - ); } else { - debug!(to_fetch = to_fetch.len(), "Fetching missing events for state from remote"); - state_events.extend( - self.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) - .await, - ); + let total_count = res.pdu_ids.len(); + let missing_count = to_fetch.len(); + if missing_count >= max(50, total_count >> 2) { + // If there's more than 50 events to fetch, or we're missing 25% or more of the + // state, we would need to make a lot of atomic requests, so we'll just try + // to fetch the full state from the remote instead. + // Since this endpoint might fail in huge rooms, we fall back to atomic fetch + // anyway. + debug_warn!( + %missing_count, + %total_count, + "Fetching full state from remote server for event" + ); + let fetched_state = match self + .fetch_full_state(origin, create_event, room_id, event_id) + .await + { + | Ok(state) => state, + | Err(e) => { + error!( + error=?e, + %origin, + "Failed to fetch full state from remote, falling back to atomic fetch" + ); + self.fetch_and_handle_missing_events( + origin, + to_fetch, + create_event, + room_id, + ) + .await + }, + }; + state_events.extend(fetched_state); + } else { + debug!( + to_fetch = to_fetch.len(), + "Fetching missing events for state from remote" + ); + state_events.extend( + self.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) + .await, + ); + } } let mut state: HashMap = From 61066bb0c6b03c511fff9b025ebf9c67d3041f3f Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 29 May 2026 14:22:08 +0100 Subject: [PATCH 22/81] fix: Don't try and fetch zero events --- src/service/rooms/event_handler/fetch_and_handle_outliers.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index d80aa9d9f..7b39ec296 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -5,7 +5,7 @@ Err, Event, PduEvent, debug, debug_error, debug_info, debug_warn, err, error, state_res::lexicographical_topological_sort, trace, - utils::{IterStream, stream::BroadbandExt}, + utils::{IterStream, math::Expected, stream::BroadbandExt}, warn, }; use futures::{StreamExt, future::select_ok}; @@ -175,6 +175,7 @@ pub async fn get_missing_events( ); for iteration in 0..iteration_limit { let limit = iteration + .expected_add(1) .saturating_mul(10) .min(GET_MISSING_EVENTS_MAX_BATCH_SIZE); debug_info!( From 9509080e0dfd2d96797452461ba178615cd274c2 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 29 May 2026 14:45:37 +0100 Subject: [PATCH 23/81] fix: Replace our local extremity tracking when joining a disconnected room remotely --- src/service/rooms/membership/mod.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/membership/mod.rs b/src/service/rooms/membership/mod.rs index 539d63ea4..a4a7338cb 100644 --- a/src/service/rooms/membership/mod.rs +++ b/src/service/rooms/membership/mod.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use conduwuit::{ - Err, Pdu, Result, Server, debug, debug_info, debug_warn, err, error, info, is_true, + Err, Event, Pdu, Result, Server, debug, debug_info, debug_warn, err, error, info, is_true, matrix::{ StateKey, event::{gen_event_id, gen_event_id_canonical_json}, @@ -577,6 +577,11 @@ pub async fn join_remote_room( if !auth_check { return Err!(Request(Forbidden("Auth check failed"))); } + let resident_before = self + .services + .state_cache + .server_in_room(self.services.globals.server_name(), room_id) + .await; info!("Compressing state from send_join"); let compressed: CompressedState = self @@ -637,6 +642,22 @@ pub async fn join_remote_room( self.services .state .set_room_state(room_id, statehash_after_join, &state_lock); + if !resident_before { + // NOTE: We replace local extremities for this room if we were not a resident + // before. We might be doing a remote join to satisfy restricted join rules, + // so we don't want to do this if we're already a resident. Otherwise, we + // want to replace our forward extremities whole-sale in case we were + // desynced. + info!("Replacing local forward extremities"); + self.services + .state + .set_forward_extremities( + room_id, + std::iter::once(parsed_join_pdu.event_id()), + &state_lock, + ) + .await; + } Ok(()) } From 9e0bcd3be838c41a1caff0841bc901df42e54c18 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 29 May 2026 14:53:15 +0100 Subject: [PATCH 24/81] fix: Hold a federation room lock while remotely joining a room --- src/service/rooms/membership/mod.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/service/rooms/membership/mod.rs b/src/service/rooms/membership/mod.rs index a4a7338cb..996f55436 100644 --- a/src/service/rooms/membership/mod.rs +++ b/src/service/rooms/membership/mod.rs @@ -34,7 +34,7 @@ use crate::{ Dep, antispam, globals, rooms::{ - metadata, outlier, pdu_metadata, short, + event_handler, metadata, outlier, pdu_metadata, short, state::{self, RoomMutexGuard}, state_accessor, state_cache, state_compressor::{self, CompressedState, HashSetCompressStateEvent}, @@ -51,6 +51,7 @@ struct Services { server: Arc, db: Arc, antispam: Dep, + event_handler: Dep, globals: Dep, metadata: Dep, outlier: Dep, @@ -73,6 +74,7 @@ fn build(args: crate::Args<'_>) -> Result> { server: args.server.clone(), db: args.db.clone(), antispam: args.depend::("antispam"), + event_handler: args.depend::("rooms::event_handler"), globals: args.depend::("globals"), metadata: args.depend::("rooms::metadata"), outlier: args.depend::("rooms::outlier"), @@ -381,8 +383,6 @@ pub async fn join_remote_room( // It has enough fields to be called a proper event now let mut join_event = join_event_stub; - - info!("Asking {remote_server} for send_join in room {room_id}"); let send_join_request = federation::membership::create_join_event::v2::Request::new( room_id.to_owned(), event_id.clone(), @@ -392,6 +392,18 @@ pub async fn join_remote_room( .await, ); + // NOTE: send_join can take a long time to respond, but from the point of view + // of other servers, we may already have finished joining. This means they + // sometimes end up sending PDUs to us that we aren't yet ready to accept, and + // consequently drop. Holding the mutex over the room while processing mitigates + // this. + let _room_lock = self + .services + .event_handler + .mutex_federation + .lock(room_id.as_str()) + .await; + info!("Asking {remote_server} for send_join in room {room_id}"); let send_join_response = match self .services .sending From 8b85b04d1029962ae12c5a44f845889c8d1871f1 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 29 May 2026 17:15:07 +0100 Subject: [PATCH 25/81] fix: Properly remove `event_id` from the PDU JSON before upgrading it --- .../rooms/event_handler/fetch_and_handle_outliers.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 7b39ec296..46dc734d3 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -358,7 +358,9 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( } if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { trace!("Found {next_id} in db"); - seeded_events.insert(next_id.clone(), local_pdu.into_canonical_object()); + let mut obj = local_pdu.into_canonical_object(); + obj.remove("event_id"); + seeded_events.insert(next_id.clone(), obj); continue; } let attempts = seen.get(&*next_id).copied().unwrap_or_default(); @@ -405,7 +407,9 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( for auth_event_id in auth_events { if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { trace!("Found auth event {next_id} in db"); - seeded_events.insert(id.clone(), local_pdu.into_canonical_object()); + let mut obj = local_pdu.into_canonical_object(); + obj.remove("event_id"); + seeded_events.insert(id.clone(), obj); continue; } if seeded_events.contains_key(&auth_event_id) { @@ -445,6 +449,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( for id in seeded_ordered { let pdu_json = seeded_events.remove(&id).unwrap(); debug_info!("Handling missing event {id} as outlier"); + assert_eq!(pdu_json.get("event_id"), None, "pdu_json had event_id"); match Box::pin(self.handle_outlier_pdu( origin, create_event, From 4df2097e6cb18b4560f260d75dce797f8fa1ca41 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 29 May 2026 17:30:46 +0100 Subject: [PATCH 26/81] style: Rename gapfill helpers instruments --- src/service/rooms/event_handler/fetch_and_handle_outliers.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 46dc734d3..2cda58b57 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -118,6 +118,7 @@ impl super::Service { /// - `min_depth`: Don't process events with a `depth` lower than this /// value. Not massively useful, but can help short-circuit infinite loops /// and weird edge paths. + #[tracing::instrument(name = "get_missing_events_bulk", skip_all)] pub async fn get_missing_events( &self, room_id: &RoomId, @@ -321,6 +322,7 @@ async fn fetch_and_handle_missing_event_via( /// Asks remote servers for any individual events that are missing. Should /// only be used for fetching missing auth events or resolving missing /// events from state_ids. For all other uses, use get_missing_events. + #[tracing::instrument(name = "get_missing_events_atomic", skip_all)] pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( &self, origin: &'a ServerName, From 04fce56381544a5a9c3ac86953447f1f31db4678 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 29 May 2026 20:57:05 +0100 Subject: [PATCH 27/81] fix: Don't silence PDU handle logs --- src/api/server/send.rs | 12 +++++++++--- src/service/rooms/membership/mod.rs | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index cdea52eee..bba4fb93e 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -7,7 +7,7 @@ use axum::extract::State; use axum_client_ip::ClientIp; use conduwuit::{ - Err, Error, Result, debug, debug_warn, err, error, + Err, Error, Result, debug, debug_error, debug_warn, err, error, result::LogErr, state_res::lexicographical_topological_sort, trace, @@ -175,8 +175,14 @@ async fn process_inbound_transaction( for (id, result) in &results { if let Err(e) = result { - if matches!(e, Error::BadRequest(ErrorKind::NotFound, _)) { - debug_warn!("Incoming PDU failed {id}: {e:?}"); + match e { + | Error::BadRequest( + ErrorKind::Forbidden | ErrorKind::InvalidParam | ErrorKind::BadJson, + .., + ) => { + debug_warn!("Incoming PDU {id} failed: {e:?}"); + }, + | _ => debug_error!("Incoming PDU {id} failed: {e:?}"), } } } diff --git a/src/service/rooms/membership/mod.rs b/src/service/rooms/membership/mod.rs index 996f55436..dfb947250 100644 --- a/src/service/rooms/membership/mod.rs +++ b/src/service/rooms/membership/mod.rs @@ -595,6 +595,7 @@ pub async fn join_remote_room( .server_in_room(self.services.globals.server_name(), room_id) .await; + let cork = self.services.db.cork_and_flush(); info!("Compressing state from send_join"); let compressed: CompressedState = self .services @@ -670,6 +671,7 @@ pub async fn join_remote_room( ) .await; } + drop(cork); Ok(()) } From 43adff926f21bcb41db5d7b34226fdb03daf5721 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 29 May 2026 21:47:36 +0100 Subject: [PATCH 28/81] fix: Make dedupe noisy, don't allow non-create event as create event --- src/service/rooms/event_handler/fetch_state.rs | 15 ++++++++------- .../rooms/event_handler/upgrade_outlier_pdu.rs | 6 ++++++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index f2601856f..c23a562a0 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -133,7 +133,7 @@ pub(super) async fn fetch_state( debug!(events = state_events.len(), "Processing state events"); for (event_id, pdu) in state_events { let state_key = pdu.state_key().ok_or_else(|| { - err!(Database("Found non-state pdu in state events: {event_id}")) + err!(Request(BadJson("Found non-state pdu in state events: {event_id}"))) })?; let shortstatekey = self @@ -146,13 +146,14 @@ pub(super) async fn fetch_state( | hash_map::Entry::Vacant(v) => { v.insert(pdu.event_id().to_owned()); }, - | hash_map::Entry::Occupied(_) => { - return Err!(Database( + | hash_map::Entry::Occupied(existing) => { + return Err!(Request(Forbidden( "State event's type and state_key combination exists multiple times \ - ({event_id}): {}, {}", + ({event_id} + {}): ({}, \"{}\")", + existing.get(), pdu.kind(), - state_key - )); + state_key, + ))); }, } } @@ -165,7 +166,7 @@ pub(super) async fn fetch_state( .await?; if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(create_event.event_id()) { - return Err!(Database("Incoming event refers to wrong create event.")); + return Err!(Request(Forbidden("Incoming event refers to wrong create event."))); } Ok(Some(state)) diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 89966b43b..1dd6f0b0c 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -57,6 +57,12 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( return Err!(Request(Forbidden("Event has been soft-failed"))); } + assert_eq!( + *create_event.kind(), + StateEventType::RoomCreate.into(), + "tried to upgrade a PDU with a create_event that is not a room create event" + ); + debug!( event_id = %incoming_pdu.event_id, "Upgrading PDU from outlier to timeline" From c94a395bf0c4b291c559f46c3027fcc641a1df97 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 16:55:32 +0100 Subject: [PATCH 29/81] fix: Make PDU handle errors noisier & correct error types --- .../rooms/event_handler/fetch_state.rs | 9 +++-- .../event_handler/handle_incoming_pdu.rs | 13 +++++--- .../rooms/event_handler/handle_outlier_pdu.rs | 33 +++++++++++-------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index c23a562a0..a4feaaa66 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -165,8 +165,13 @@ pub(super) async fn fetch_state( .get_shortstatekey(&StateEventType::RoomCreate, "") .await?; - if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(create_event.event_id()) { - return Err!(Request(Forbidden("Incoming event refers to wrong create event."))); + let create_event_id = state.get(&create_shortstatekey).map(AsRef::as_ref); + if create_event_id != Some(create_event.event_id()) { + return Err!(Request(Forbidden( + "Incoming event refers to wrong create event: expected {}, got: \ + {create_event_id:?}", + create_event.event_id() + ))); } Ok(Some(state)) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 305df60e9..4b2927ec2 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -1,8 +1,8 @@ use std::{collections::BTreeMap, time::Instant}; use conduwuit::{ - Err, Event, PduEvent, Result, debug_error, debug_info, defer, err, implement, info, trace, - warn, + Err, Event, PduEvent, Result, debug_error, debug_info, defer, err, error, implement, info, + trace, warn, }; use futures::{ FutureExt, @@ -146,7 +146,7 @@ pub async fn handle_incoming_pdu<'a>( .and_then(|v| v.as_str()) .ok_or_else(|| err!("No sender in object")) .and_then(|v| Ok(UserId::parse(v)?)) - .map_err(|e| err!(Request(InvalidParam("PDU does not have a valid sender key: {e}"))))?; + .map_err(|e| err!(Request(BadJson("PDU does not have a valid sender key: {e}"))))?; let sender_acl_check: OptionFuture<_> = sender .server_name() @@ -223,7 +223,8 @@ pub async fn handle_incoming_pdu<'a>( let (incoming_pdu, val) = self .handle_outlier_pdu(origin, create_event, event_id, room_id, value, false) - .await?; + .await + .inspect(|e| error!("[TODO] Failed to handle outlier PDU: {e:?}"))?; // 8. if not timeline event: stop if !is_timeline_event { @@ -242,9 +243,11 @@ pub async fn handle_incoming_pdu<'a>( // These are timeline events self.fetch_prevs(room_id, create_event, &incoming_pdu, origin) - .await?; + .await + .inspect_err(|e| error!("[TODO] Failed to fetch_prevs: {e:?}"))?; // Done with prev events, now handling the incoming event self.upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) .await + .inspect_err(|e| error!("[TODO] Failed to upgrade outlier to timeline pdu: {e:?}")) } diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 64d1a9acd..91e9d91ba 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -2,7 +2,7 @@ use conduwuit::{ Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, implement, info, state_res, - trace, warn, + state_res::EventTypeExt, trace, warn, }; use futures::future::ready; use ruma::{ @@ -106,7 +106,7 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( event_id, ); self.services.pdu_metadata.mark_event_rejected(event_id); - return Err!(Request(InvalidParam("Event has rejected auth event: {aid}"))); + return Err!(Request(Forbidden("Event has rejected auth event: {aid}"))); } if let Ok(auth_event) = self.services.timeline.get_pdu(aid).await { @@ -142,7 +142,7 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( let (auth_event_room_id, auth_event_id, auth_pdu_json) = self.parse_incoming_pdu(&auth_pdu_json).await?; if auth_event_room_id != room_id { - return Err!(Request(BadJson( + return Err!(Request(Forbidden( "Auth event {auth_event_id} is in {auth_event_room_id}, not {room_id}." ))); } @@ -195,7 +195,7 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( .outlier .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); self.services.pdu_metadata.mark_event_rejected(event_id); - return Err!(Request(InvalidParam( + return Err!(Request(Forbidden( "Auth event's type and state_key combination exists multiple times: {}, {}", auth_event.kind, auth_event.state_key().unwrap_or("") @@ -205,15 +205,22 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( } // The original create event must be in the auth events - if !matches!( - auth_events_by_key.get(&(StateEventType::RoomCreate, String::new().into())), - Some(_) | None - ) { - self.services.pdu_metadata.mark_event_rejected(event_id); - self.services - .outlier - .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); - return Err!(Request(InvalidParam("Incoming event refers to wrong create event."))); + let claimed_create_event = + auth_events_by_key.get(&StateEventType::RoomCreate.with_state_key("")); + if let Some(claimed_create_event) = claimed_create_event { + if claimed_create_event.event_id() != create_event.event_id() { + self.services.pdu_metadata.mark_event_rejected(event_id); + self.services + .outlier + .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); + return Err!(Request(Forbidden( + "Incoming event refers to wrong create event (expected {}, got {})", + create_event.event_id(), + claimed_create_event.event_id(), + ))); + } + } else if !pdu_event.auth_events.is_empty() { + return Err!(Request(Forbidden("Incoming event does not refer to any create event"))); } let state_fetch = |ty: &StateEventType, sk: &str| { From 970958652a68f9049f45be35e3359563cc6d46e1 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 17:07:27 +0100 Subject: [PATCH 30/81] fix: Remove redundant check that accidentally banned everyone --- .../event_handler/handle_incoming_pdu.rs | 2 +- .../rooms/event_handler/handle_outlier_pdu.rs | 21 +------------------ 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 4b2927ec2..2d105755c 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -224,7 +224,7 @@ pub async fn handle_incoming_pdu<'a>( let (incoming_pdu, val) = self .handle_outlier_pdu(origin, create_event, event_id, room_id, value, false) .await - .inspect(|e| error!("[TODO] Failed to handle outlier PDU: {e:?}"))?; + .inspect_err(|e| error!("[TODO] Failed to handle outlier PDU: {e:?}"))?; // 8. if not timeline event: stop if !is_timeline_event { diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 91e9d91ba..740b83c3c 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -2,7 +2,7 @@ use conduwuit::{ Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, implement, info, state_res, - state_res::EventTypeExt, trace, warn, + trace, warn, }; use futures::future::ready; use ruma::{ @@ -204,25 +204,6 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( } } - // The original create event must be in the auth events - let claimed_create_event = - auth_events_by_key.get(&StateEventType::RoomCreate.with_state_key("")); - if let Some(claimed_create_event) = claimed_create_event { - if claimed_create_event.event_id() != create_event.event_id() { - self.services.pdu_metadata.mark_event_rejected(event_id); - self.services - .outlier - .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); - return Err!(Request(Forbidden( - "Incoming event refers to wrong create event (expected {}, got {})", - create_event.event_id(), - claimed_create_event.event_id(), - ))); - } - } else if !pdu_event.auth_events.is_empty() { - return Err!(Request(Forbidden("Incoming event does not refer to any create event"))); - } - let state_fetch = |ty: &StateEventType, sk: &str| { let key = (ty.to_owned(), sk.into()); ready(auth_events_by_key.get(&key).map(ToOwned::to_owned)) From 31982e84de470117d10c941a776872bb4b6f4384 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 17:12:40 +0100 Subject: [PATCH 31/81] fix: Remove redundant check 2 This may look scary, but this is safe because event auth performs the same check, and will reject the event if it doesn't reference the create event correctly. --- src/service/rooms/event_handler/fetch_state.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index a4feaaa66..1fed99d52 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -10,7 +10,6 @@ use ruma::{ EventId, OwnedEventId, RoomId, ServerName, api::federation::event::{get_room_state, get_room_state_ids}, - events::StateEventType, }; use crate::{conduwuit::utils::stream::BroadbandExt, rooms::short::ShortStateKey}; @@ -158,22 +157,6 @@ pub(super) async fn fetch_state( } } - // The original create event must still be in the state - let create_shortstatekey = self - .services - .short - .get_shortstatekey(&StateEventType::RoomCreate, "") - .await?; - - let create_event_id = state.get(&create_shortstatekey).map(AsRef::as_ref); - if create_event_id != Some(create_event.event_id()) { - return Err!(Request(Forbidden( - "Incoming event refers to wrong create event: expected {}, got: \ - {create_event_id:?}", - create_event.event_id() - ))); - } - Ok(Some(state)) } From 2bf5876778414be41c15d99c9a1f903cf6d10c95 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 17:41:23 +0100 Subject: [PATCH 32/81] fix: Ask more servers for state_ids when origin fails to provide Some servers reference events in prev_events that they might not yet have finished processing, so this allows us to at least attempt to get the state from another trustworthy server in the room that might be faster. I don't think this is too effective, however it's more effective than giving up immediately. --- .../rooms/event_handler/fetch_state.rs | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 1fed99d52..ebfac081c 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -6,9 +6,9 @@ use conduwuit::{ Err, Event, PduEvent, Result, debug, debug_warn, err, error, trace, utils::IterStream, warn, }; -use futures::StreamExt; +use futures::{StreamExt, TryFutureExt, future::select_ok}; use ruma::{ - EventId, OwnedEventId, RoomId, ServerName, + EventId, OwnedEventId, OwnedRoomId, RoomId, ServerName, api::federation::event::{get_room_state, get_room_state_ids}, }; @@ -35,7 +35,7 @@ pub(super) async fn fetch_state( event_id: &EventId, ) -> Result>> { trace!(%origin, "Asking remote for state_ids"); - let res: get_room_state_ids::v1::Response = self + let res: get_room_state_ids::v1::Response = match self .services .sending .send_federation_request( @@ -43,7 +43,20 @@ pub(super) async fn fetch_state( get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()), ) .await - .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; + .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}")) + { + | Ok(resp) => Ok(resp), + | Err(e) => + if e.is_not_found() { + self.fetch_state_ids_from_backfill_servers( + event_id.to_owned(), + room_id.to_owned(), + ) + .await + } else { + Err(e) + }, + }?; debug!(events = res.pdu_ids.len(), "Fetching state events"); let mut state_events: HashMap = @@ -160,6 +173,33 @@ pub(super) async fn fetch_state( Ok(Some(state)) } + async fn fetch_state_ids_from_backfill_servers( + &self, + event_id: OwnedEventId, + room_id: OwnedRoomId, + ) -> Result { + let candidates = self + .services + .timeline + .candidate_backfill_servers(&room_id) + .await; + debug!(%room_id, ?candidates, "Asking backfill servers for state_ids"); + let futures = candidates.iter().map(|server_name| { + Box::pin( + self.services + .sending + .send_federation_request( + server_name, + get_room_state_ids::v1::Request::new(event_id.clone(), room_id.clone()), + ) + .inspect_err(|e| { + debug_warn!("Fallback fetching state for event failed: {e}"); + }), + ) + }); + Ok(select_ok(futures).await?.0) + } + /// Fetches the full state via `GET /_matrix/federation/v1/state` from a /// remote server, and persists all the incoming auth chain events and /// state events as outliers, for use later. From 0efa6ed1f280d71440323a47888672810ff223c0 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 18:02:09 +0100 Subject: [PATCH 33/81] fix: Don't treat prev outlier upgrades as fetch failures --- src/service/rooms/event_handler/fetch_prev.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 25fdf9554..9cacf2f40 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,9 +1,8 @@ use std::collections::HashMap; use conduwuit::{ - Event, PduEvent, debug, debug_info, + Event, PduEvent, debug, debug_info, debug_warn, utils::{BoolExt, IterStream, stream::BroadbandExt}, - warn, }; use futures::StreamExt; use ruma::{RoomId, ServerName}; @@ -88,14 +87,16 @@ pub(super) async fn fetch_prevs( .handle_outlier_pdu(origin, create_event, &event_id, room_id, obj, false) .await { - | Ok((pdu, val)) => + | Ok((pdu, val)) => { self.upgrade_outlier_to_timeline_pdu(pdu, val, create_event, origin, room_id) - .await, - | Err(e) => { - warn!("Failed to persist prev_event {event_id}: {e}"); - continue; + .await + .inspect_err(|e| { + debug_warn!("Failed to upgrade prev event {event_id}: {e}"); + }) + .ok(); }, - }?; + | Err(e) => debug_warn!("Failed to persist prev event {event_id}: {e}"), + } } // NOTE because i keep forgetting: the caller persists incoming_pdu. From f34f84832b6670e63ae19f99708226800bbbe515 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 18:33:44 +0100 Subject: [PATCH 34/81] feat: Include timing information in debug logs --- .../fetch_and_handle_outliers.rs | 59 +++++++++++-------- src/service/rooms/event_handler/fetch_prev.rs | 23 +++++--- .../rooms/event_handler/fetch_state.rs | 30 ++++++---- 3 files changed, 72 insertions(+), 40 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 2cda58b57..68ed45fae 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -1,4 +1,7 @@ -use std::collections::{HashMap, HashSet, VecDeque}; +use std::{ + collections::{HashMap, HashSet, VecDeque}, + time::Instant, +}; use assign::assign; use conduwuit::{ @@ -127,6 +130,7 @@ pub async fn get_missing_events( via: &ServerName, min_depth: UInt, ) -> conduwuit::Result> { + let start = Instant::now(); #[cfg(debug_assertions)] { let missing_count = head @@ -138,9 +142,11 @@ pub async fn get_missing_events( .timeline .get_non_outlier_pdu_json(event_id) .await - .inspect(|_| debug!("Found prev_event {event_id} locally.")) + .inspect( + |_| debug!(elapsed=?start.elapsed(),"Found prev_event {event_id} locally."), + ) .inspect_err( - |e| debug!(%e, "Could not find prev_event {event_id} locally."), + |e| debug!(elapsed=?start.elapsed(),%e, "Could not find prev_event {event_id} locally."), ) { | Ok(_) => None, | Err(_) => Some(event_id), @@ -168,7 +174,7 @@ pub async fn get_missing_events( let mut discovered = HashMap::with_capacity(head.prev_events.len()); let mut latest_events: Vec = vec![head.event_id().to_owned()]; - debug!( + debug!(elapsed=?start.elapsed(), %room_id, event_id=%head.event_id(), %iteration_limit, @@ -179,7 +185,7 @@ pub async fn get_missing_events( .expected_add(1) .saturating_mul(10) .min(GET_MISSING_EVENTS_MAX_BATCH_SIZE); - debug_info!( + debug_info!(elapsed=?start.elapsed(), %limit, %via, %iteration, @@ -204,10 +210,10 @@ pub async fn get_missing_events( .await?; if response.events.is_empty() { - debug_info!(%via, "Finished gap filling missing events (remote returned no more events)."); + debug_info!(elapsed=?start.elapsed(),%via, "Finished gap filling missing events (remote returned no more events)."); break; } - debug_info!("Got {} events back from remote", response.events.len()); + debug_info!(elapsed=?start.elapsed(),"Got {} events back from remote", response.events.len()); latest_events.clear(); for raw_event in response.events { @@ -218,7 +224,7 @@ pub async fn get_missing_events( if pdu.depth < min_depth { debug_warn!( - "Received PDU with depth {} below min_depth {}, ignoring", + elapsed=?start.elapsed(),"Received PDU with depth {} below min_depth {}, ignoring", pdu.depth, min_depth ); @@ -249,7 +255,7 @@ pub async fn get_missing_events( } if latest_events.is_empty() { - debug!( + debug!(elapsed=?start.elapsed(), %limit, %via, %iteration, @@ -264,7 +270,7 @@ pub async fn get_missing_events( // debug mode because this can be important, but typically not much can be done // about it as a user. #[cfg(debug_assertions)] - error!( + error!(elapsed=?start.elapsed(), discovered=discovered.len(), max_fetch_prev_events=self.services.server.config.max_fetch_prev_events, %iteration, @@ -275,7 +281,7 @@ pub async fn get_missing_events( "Encountered a gap too large to fill, giving up" ); #[cfg(not(debug_assertions))] - warn!( + warn!(elapsed=?start.elapsed(), discovered=discovered.len(), max_fetch_prev_events=self.services.server.config.max_fetch_prev_events, %iteration, @@ -289,6 +295,7 @@ pub async fn get_missing_events( } } + trace!(elapsed=?start.elapsed(), "Finished get_missing_events"); Ok(discovered) } @@ -333,6 +340,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( where Pdu: Event + Send + Sync, { + let start = Instant::now(); let room_version_rules = &get_room_version_rules(create_event).unwrap_or(RoomVersionRules::V1); let mut candidates = self @@ -346,6 +354,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( let mut seeded_events = HashMap::with_capacity(events.len().saturating_add(events.len().saturating_mul(3))); trace!( + elapsed=?start.elapsed(), "Fetching {} unknown PDUs on demand from {} candidates", events.len(), candidates.len() @@ -359,7 +368,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( continue; } if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { - trace!("Found {next_id} in db"); + trace!(elapsed=?start.elapsed(),"Found {next_id} in db"); let mut obj = local_pdu.into_canonical_object(); obj.remove("event_id"); seeded_events.insert(next_id.clone(), obj); @@ -367,11 +376,11 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( } let attempts = seen.get(&*next_id).copied().unwrap_or_default(); if attempts >= 5 { - debug_error!(%attempts, %next_id, "Could not fetch missing event after 5 attempts, giving up"); + debug_error!(elapsed=?start.elapsed(),%attempts, %next_id, "Could not fetch missing event after 5 attempts, giving up"); continue; } - debug!("Fetching {next_id} over federation"); + debug!(elapsed=?start.elapsed(),"Fetching {next_id} over federation"); let futures = candidates .iter() .map(|remote| { @@ -385,13 +394,14 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( let (event_id, value) = match select_ok(futures).await { | Ok((x, _)) => x, | Err(e) => { - warn!("failed to fetch missing event {next_id} from any candidate: {e}"); + warn!(elapsed=?start.elapsed(),"failed to fetch missing event {next_id} from any candidate: {e}"); continue; }, }; let auth_events = match expect_event_id_array(&value, "auth_events").map_err(|e| { err!(Request(BadJson(warn!( + elapsed=?start.elapsed(), %event_id, "Failed to parse event fetched from remote: {e}" )))) @@ -399,6 +409,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( | Ok(auth_events) => auth_events, | Err(e) => { warn!( + elapsed=?start.elapsed(), ?e, "event {event_id} is malformed (bad auth_events), skipping" ); @@ -408,17 +419,17 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( let mut have_all_auth = true; for auth_event_id in auth_events { if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { - trace!("Found auth event {next_id} in db"); + trace!(elapsed=?start.elapsed(),"Found auth event {next_id} in db"); let mut obj = local_pdu.into_canonical_object(); obj.remove("event_id"); seeded_events.insert(id.clone(), obj); continue; } if seeded_events.contains_key(&auth_event_id) { - trace!(%auth_event_id, "Already found auth event"); + trace!(elapsed=?start.elapsed(),%auth_event_id, "Already found auth event"); continue; } - debug!("Missing auth event {auth_event_id} for event {next_id}"); + debug!(elapsed=?start.elapsed(),"Missing auth event {auth_event_id} for event {next_id}"); seen.insert(auth_event_id.clone(), attempts.saturating_add(1)); todo.push_back(auth_event_id); have_all_auth = false; @@ -427,10 +438,10 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( // all of its auth events have been fetched. // TODO: This may result in infinite looping, needs a breaker if have_all_auth { - debug!(%next_id, "Have all auth events"); + debug!(elapsed=?start.elapsed(),%next_id, "Have all auth events"); seeded_events.insert(next_id, value); } else { - debug_warn!( + debug_warn!(elapsed=?start.elapsed(), "Fetched {next_id} but missing some auth events, will have to re-fetch." ); seen.insert(next_id.clone(), attempts.saturating_add(1)); @@ -450,7 +461,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( let mut pdus = HashMap::with_capacity(seeded_ordered.len()); for id in seeded_ordered { let pdu_json = seeded_events.remove(&id).unwrap(); - debug_info!("Handling missing event {id} as outlier"); + debug_info!(elapsed=?start.elapsed(),"Handling missing event {id} as outlier"); assert_eq!(pdu_json.get("event_id"), None, "pdu_json had event_id"); match Box::pin(self.handle_outlier_pdu( origin, @@ -463,9 +474,11 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( .await { | Ok((pdu, _)) => { + trace!(elapsed=?start.elapsed(), "Persisted {id}"); let _ = pdus.insert(id, pdu); }, - | Err(e) => warn!("Authentication of event {id} failed: {e:?}"), + | Err(e) => + warn!(elapsed=?start.elapsed(),"Authentication of event {id} failed: {e:?}"), } // TODO: should this try to promote to timeline? @@ -473,7 +486,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( // now. } - trace!("Fetched and handled {} missing PDUs", pdus.len()); + trace!(elapsed=?start.elapsed(),"Finished fetch_and_handle_missing_events: fetched and handled {} missing PDUs", pdus.len()); pdus } } diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 9cacf2f40..729339e6e 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,7 +1,7 @@ -use std::collections::HashMap; +use std::{collections::HashMap, time::Instant}; use conduwuit::{ - Event, PduEvent, debug, debug_info, debug_warn, + Event, PduEvent, debug, debug_info, debug_warn, trace, utils::{BoolExt, IterStream, stream::BroadbandExt}, }; use futures::StreamExt; @@ -19,6 +19,7 @@ pub(super) async fn fetch_prevs( incoming_pdu: &PduEvent, origin: &ServerName, ) -> conduwuit::Result<()> { + let start = Instant::now(); let missing = incoming_pdu .prev_events() .stream() @@ -33,10 +34,10 @@ pub(super) async fn fetch_prevs( .collect::>() .await; if missing.is_empty() { - debug!(event_id=%incoming_pdu.event_id(), "No missing prev events."); + debug!(elapsed=?start.elapsed(), event_id=%incoming_pdu.event_id(), "No missing prev events."); return Ok(()); } - debug!(%room_id, event_id=%incoming_pdu.event_id(), ?missing, "Fetching previous events"); + debug!(elapsed=?start.elapsed(), %room_id, event_id=%incoming_pdu.event_id(), ?missing, "Fetching previous events"); let tail = self .services .state @@ -62,7 +63,7 @@ pub(super) async fn fetch_prevs( ), ) .await?; - debug_info!("Fetched {} missing events", gapfilled.len()); + debug_info!(elapsed=?start.elapsed(), "Fetched {} missing events", gapfilled.len()); // Persist all fetched events let mapped = gapfilled @@ -80,9 +81,12 @@ pub(super) async fn fetch_prevs( build_local_dag(&mapped).await? }; + let job_start = Instant::now(); + trace!("Starting to persist {} prev events", to_persist.len()); for event_id in to_persist { debug_info!("Persisting fetched prev event {event_id}"); let obj = mapped.get(&event_id).cloned().unwrap(); + let persist_start = Instant::now(); match self .handle_outlier_pdu(origin, create_event, &event_id, room_id, obj, false) .await @@ -91,16 +95,21 @@ pub(super) async fn fetch_prevs( self.upgrade_outlier_to_timeline_pdu(pdu, val, create_event, origin, room_id) .await .inspect_err(|e| { - debug_warn!("Failed to upgrade prev event {event_id}: {e}"); + debug_warn!(total_elapsed=?start.elapsed(), job_elapsed=?job_start.elapsed(), task_elapsed=?persist_start.elapsed(), "Failed to upgrade prev event {event_id}: {e}"); }) .ok(); }, - | Err(e) => debug_warn!("Failed to persist prev event {event_id}: {e}"), + | Err(e) => + debug_warn!(total_elapsed=?start.elapsed(), job_elapsed=?job_start.elapsed(), task_elapsed=?persist_start.elapsed(), "Failed to persist prev event {event_id}: {e}"), } } // NOTE because i keep forgetting: the caller persists incoming_pdu. // we only care about its prev events + trace!( + total_elapsed=?start.elapsed(), + persist_elapsed=?job_start.elapsed(), + ); Ok(()) } } diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index ebfac081c..f399e1696 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,6 +1,7 @@ use std::{ cmp::max, collections::{HashMap, hash_map}, + time::Instant, }; use conduwuit::{ @@ -34,6 +35,7 @@ pub(super) async fn fetch_state( room_id: &RoomId, event_id: &EventId, ) -> Result>> { + let start = Instant::now(); trace!(%origin, "Asking remote for state_ids"); let res: get_room_state_ids::v1::Response = match self .services @@ -43,8 +45,9 @@ pub(super) async fn fetch_state( get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()), ) .await - .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}")) - { + .inspect_err( + |e| debug_warn!(elapsed=?start.elapsed(), "Fetching state for event failed: {e}"), + ) { | Ok(resp) => Ok(resp), | Err(e) => if e.is_not_found() { @@ -58,7 +61,7 @@ pub(super) async fn fetch_state( }, }?; - debug!(events = res.pdu_ids.len(), "Fetching state events"); + debug!(elapsed=?start.elapsed(), events = res.pdu_ids.len(), "Fetching state events"); let mut state_events: HashMap = HashMap::with_capacity(res.pdu_ids.len()); let to_fetch: Vec = res @@ -76,7 +79,7 @@ pub(super) async fn fetch_state( .collect() .await; if to_fetch.is_empty() { - debug!("All required state events are already known."); + debug!(elapsed=?start.elapsed(), "All required state events are already known."); state_events = res .pdu_ids .iter() @@ -103,6 +106,7 @@ pub(super) async fn fetch_state( // Since this endpoint might fail in huge rooms, we fall back to atomic fetch // anyway. debug_warn!( + elapsed=?start.elapsed(), %missing_count, %total_count, "Fetching full state from remote server for event" @@ -114,6 +118,7 @@ pub(super) async fn fetch_state( | Ok(state) => state, | Err(e) => { error!( + elapsed=?start.elapsed(), error=?e, %origin, "Failed to fetch full state from remote, falling back to atomic fetch" @@ -130,6 +135,7 @@ pub(super) async fn fetch_state( state_events.extend(fetched_state); } else { debug!( + elapsed=?start.elapsed(), to_fetch = to_fetch.len(), "Fetching missing events for state from remote" ); @@ -142,7 +148,7 @@ pub(super) async fn fetch_state( let mut state: HashMap = HashMap::with_capacity(state_events.len()); - debug!(events = state_events.len(), "Processing state events"); + debug!(elapsed=?start.elapsed(), events = state_events.len(), "Processing state events"); for (event_id, pdu) in state_events { let state_key = pdu.state_key().ok_or_else(|| { err!(Request(BadJson("Found non-state pdu in state events: {event_id}"))) @@ -169,7 +175,7 @@ pub(super) async fn fetch_state( }, } } - + trace!(elapsed=?start.elapsed(), "fetch_state finished"); Ok(Some(state)) } @@ -213,6 +219,7 @@ pub(super) async fn fetch_full_state( room_id: &RoomId, event_id: &EventId, ) -> Result> { + let start = Instant::now(); trace!("Fetching full state from remote server"); let res: get_room_state::v1::Response = self .services @@ -223,7 +230,7 @@ pub(super) async fn fetch_full_state( ) .await .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; - debug!(count = res.auth_chain.len(), "Handling incoming auth chain..."); + debug!(elapsed=?start.elapsed(), count = res.auth_chain.len(), "Handling incoming auth chain..."); res.auth_chain .iter() .stream() @@ -260,8 +267,8 @@ pub(super) async fn fetch_full_state( }, ) .await; - debug!(count = res.pdus.len(), "Handling incoming state PDUs..."); - Ok(res + debug!(elapsed=?start.elapsed(), count = res.pdus.len(), "Handling incoming state PDUs..."); + let r = res .pdus .iter() .stream() @@ -287,6 +294,7 @@ pub(super) async fn fetch_full_state( .await .inspect_err(|e| { warn!( + elapsed=?start.elapsed(), %incoming_room_id, %incoming_event_id, ?e, @@ -300,6 +308,8 @@ pub(super) async fn fetch_full_state( acc.insert(event.event_id().to_owned(), event); acc }) - .await) + .await; + trace!(elapsed=?start.elapsed(), "fetch_full_state finished"); + Ok(r) } } From 2a598af8880733325875c5069e16d7b55bcfc42e Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 19:24:48 +0100 Subject: [PATCH 35/81] feat: Make logging more verbose to diagnose the aranjesplosion --- .../rooms/event_handler/fetch_and_handle_outliers.rs | 6 +++--- src/service/rooms/event_handler/handle_incoming_pdu.rs | 6 +++++- src/service/rooms/event_handler/handle_outlier_pdu.rs | 1 + src/service/rooms/event_handler/parse_incoming_pdu.rs | 1 + src/service/rooms/event_handler/upgrade_outlier_pdu.rs | 1 + 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 68ed45fae..a68b5e3e3 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -224,12 +224,12 @@ pub async fn get_missing_events( if pdu.depth < min_depth { debug_warn!( - elapsed=?start.elapsed(),"Received PDU with depth {} below min_depth {}, ignoring", + elapsed=?start.elapsed(),"Received PDU with depth {} below min_depth {}", pdu.depth, min_depth ); - discovered.insert(event_id.clone(), pdu); - continue; + // discovered.insert(event_id.clone(), pdu); + // continue; } for prev_event_id in pdu.prev_events() { diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 2d105755c..6f90ac602 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -221,13 +221,15 @@ pub async fn handle_incoming_pdu<'a>( .remove(room_id); }}; + info!("[TODO] Handling PDU as outlier"); let (incoming_pdu, val) = self .handle_outlier_pdu(origin, create_event, event_id, room_id, value, false) .await .inspect_err(|e| error!("[TODO] Failed to handle outlier PDU: {e:?}"))?; - + info!("[TODO] Finished handling PDU as outlier"); // 8. if not timeline event: stop if !is_timeline_event { + info!("[TODO] Not upgrading PDU"); return Ok(None); } @@ -242,10 +244,12 @@ pub async fn handle_incoming_pdu<'a>( // 9. Fetch any missing prev events doing all checks listed here starting at 1. // These are timeline events + info!("[TODO] Fetching prev events"); self.fetch_prevs(room_id, create_event, &incoming_pdu, origin) .await .inspect_err(|e| error!("[TODO] Failed to fetch_prevs: {e:?}"))?; + info!("[TODO] Finished fetching prev events, attempting to upgrade"); // Done with prev events, now handling the incoming event self.upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) .await diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 740b83c3c..4f167fcd3 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -16,6 +16,7 @@ #[implement(super::Service)] #[allow(clippy::too_many_arguments)] +#[tracing::instrument(name="handle_outlier", skip_all, fields(%event_id))] pub(super) async fn handle_outlier_pdu<'a, Pdu>( &self, origin: &'a ServerName, diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 2e7281b3c..7c7e229fe 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -118,6 +118,7 @@ pub async fn parse_incoming_pdu_with_known_room( } #[implement(super::Service)] +#[tracing::instrument(name = "parse", skip_all)] pub async fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result { let value = serde_json::from_str::(pdu.get()).map_err(|e| { err!(BadServerResponse(debug_warn!("Error parsing incoming event {e:?}"))) diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 1dd6f0b0c..4794de8f3 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -24,6 +24,7 @@ }; #[implement(super::Service)] +#[tracing::instrument(name="upgrade_outlier", skip_all, fields(event_id=%incoming_pdu.event_id()))] pub(super) async fn upgrade_outlier_to_timeline_pdu( &self, incoming_pdu: PduEvent, From 3907589b6caac7a0856b601c7972a8d1de03a16e Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 19:30:35 +0100 Subject: [PATCH 36/81] fix: Don't download the world --- src/service/rooms/event_handler/fetch_and_handle_outliers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index a68b5e3e3..5b615ee4e 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -228,8 +228,8 @@ pub async fn get_missing_events( pdu.depth, min_depth ); - // discovered.insert(event_id.clone(), pdu); - // continue; + discovered.insert(event_id.clone(), pdu); + continue; } for prev_event_id in pdu.prev_events() { From 36ed20cb04b2b8ba205c582a288b41af1e2f3b05 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 20:29:16 +0100 Subject: [PATCH 37/81] fix: Downgrade safe assert to debug assert --- src/service/rooms/event_handler/upgrade_outlier_pdu.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 4794de8f3..5d47d03ad 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -97,7 +97,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( let state_at_incoming_event = state_at_incoming_event.expect("we always set this to some above"); - assert!(!state_at_incoming_event.is_empty(), "Event has no incoming state"); + debug_assert!(!state_at_incoming_event.is_empty(), "Event has no incoming state"); trace!(state_events = state_at_incoming_event.len(), "Calculated incoming state"); debug!( From 9930e549a048398fa1801ce325446af2ef70a660 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 21:03:01 +0100 Subject: [PATCH 38/81] fix: Progress log in fetch_prev --- src/service/rooms/event_handler/fetch_prev.rs | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 729339e6e..7b15396a3 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,10 +1,11 @@ use std::{collections::HashMap, time::Instant}; +use askama::filters::e; use conduwuit::{ - Event, PduEvent, debug, debug_info, debug_warn, trace, + Event, PduEvent, debug, debug_info, debug_warn, info, trace, utils::{BoolExt, IterStream, stream::BroadbandExt}, }; -use futures::StreamExt; +use futures::{StreamExt, future::ok}; use ruma::{RoomId, ServerName}; use crate::rooms::event_handler::build_local_dag; @@ -83,24 +84,47 @@ pub(super) async fn fetch_prevs( let job_start = Instant::now(); trace!("Starting to persist {} prev events", to_persist.len()); - for event_id in to_persist { - debug_info!("Persisting fetched prev event {event_id}"); - let obj = mapped.get(&event_id).cloned().unwrap(); + for (i, event_id) in to_persist.iter().enumerate() { + info!( + elapsed=?start.elapsed(), + "[TODO] Persisting fetched prev event: {event_id} ({}/{})", + i.saturating_add(1), + to_persist.len(), + ); + debug_info!(elapsed=?start.elapsed(), "Persisting fetched prev event {event_id}"); + let obj = mapped.get(event_id).cloned().unwrap(); let persist_start = Instant::now(); match self - .handle_outlier_pdu(origin, create_event, &event_id, room_id, obj, false) + .handle_outlier_pdu(origin, create_event, event_id, room_id, obj, false) .await { | Ok((pdu, val)) => { self.upgrade_outlier_to_timeline_pdu(pdu, val, create_event, origin, room_id) .await .inspect_err(|e| { - debug_warn!(total_elapsed=?start.elapsed(), job_elapsed=?job_start.elapsed(), task_elapsed=?persist_start.elapsed(), "Failed to upgrade prev event {event_id}: {e}"); + debug_warn!( + total_elapsed=?start.elapsed(), + job_elapsed=?job_start.elapsed(), + task_elapsed=?persist_start.elapsed(), + "Failed to upgrade prev event {event_id}: {e}", + ); + }) + .inspect(|_| { + debug_info!( + total_elapsed=?start.elapsed(), + job_elapsed=?job_start.elapsed(), + task_elapsed=?persist_start.elapsed(), + "Upgraded prev event {event_id}", + ); }) .ok(); }, - | Err(e) => - debug_warn!(total_elapsed=?start.elapsed(), job_elapsed=?job_start.elapsed(), task_elapsed=?persist_start.elapsed(), "Failed to persist prev event {event_id}: {e}"), + | Err(e) => debug_warn!( + total_elapsed=?start.elapsed(), + job_elapsed=?job_start.elapsed(), + task_elapsed=?persist_start.elapsed(), + "Failed to persist prev event {event_id}: {e}", + ), } } From cd46070bd306c0ae188d811dcd0bc708707134c3 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 21:57:30 +0100 Subject: [PATCH 39/81] fix: Elide auth chain from fetch_and_handle_outliers --- .../fetch_and_handle_outliers.rs | 28 +++++++++---------- src/service/rooms/event_handler/fetch_prev.rs | 3 +- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 5b615ee4e..606754015 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -349,9 +349,8 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( .candidate_backfill_servers(room_id) .await; candidates.insert(origin.to_owned()); - candidates.retain(|sn| self.services.globals.server_name() != sn); - assert_ne!(candidates.len(), 0, "no candidates to fetch missing events from"); - let mut seeded_events = + assert!(!candidates.is_empty(), "no candidates to fetch missing events from"); + let mut discovered_events = HashMap::with_capacity(events.len().saturating_add(events.len().saturating_mul(3))); trace!( elapsed=?start.elapsed(), @@ -361,17 +360,17 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( ); let mut seen: HashMap = HashMap::new(); - for id in events { - let mut todo: VecDeque = [id.clone()].into(); + for id in &events { + let mut todo: VecDeque = [id.to_owned()].into(); while let Some(next_id) = todo.pop_front() { - if seeded_events.contains_key(&next_id) { + if discovered_events.contains_key(&next_id) { continue; } if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { trace!(elapsed=?start.elapsed(),"Found {next_id} in db"); let mut obj = local_pdu.into_canonical_object(); obj.remove("event_id"); - seeded_events.insert(next_id.clone(), obj); + discovered_events.insert(next_id.clone(), obj); continue; } let attempts = seen.get(&*next_id).copied().unwrap_or_default(); @@ -422,10 +421,10 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( trace!(elapsed=?start.elapsed(),"Found auth event {next_id} in db"); let mut obj = local_pdu.into_canonical_object(); obj.remove("event_id"); - seeded_events.insert(id.clone(), obj); + discovered_events.insert(id.clone(), obj); continue; } - if seeded_events.contains_key(&auth_event_id) { + if discovered_events.contains_key(&auth_event_id) { trace!(elapsed=?start.elapsed(),%auth_event_id, "Already found auth event"); continue; } @@ -439,7 +438,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( // TODO: This may result in infinite looping, needs a breaker if have_all_auth { debug!(elapsed=?start.elapsed(),%next_id, "Have all auth events"); - seeded_events.insert(next_id, value); + discovered_events.insert(next_id, value); } else { debug_warn!(elapsed=?start.elapsed(), "Fetched {next_id} but missing some auth events, will have to re-fetch." @@ -451,16 +450,14 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( } let seeded_ordered = build_local_dag( - &seeded_events - .iter() - .map(|(eid, e)| (eid.to_owned(), e.clone())) - .collect::>(), + &discovered_events.clone(), /* TODO: this clones like several megabytes of data, + * owie :( */ ) .await .expect("failed to build local DAG"); let mut pdus = HashMap::with_capacity(seeded_ordered.len()); for id in seeded_ordered { - let pdu_json = seeded_events.remove(&id).unwrap(); + let pdu_json = discovered_events.remove(&id).unwrap(); debug_info!(elapsed=?start.elapsed(),"Handling missing event {id} as outlier"); assert_eq!(pdu_json.get("event_id"), None, "pdu_json had event_id"); match Box::pin(self.handle_outlier_pdu( @@ -487,6 +484,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( } trace!(elapsed=?start.elapsed(),"Finished fetch_and_handle_missing_events: fetched and handled {} missing PDUs", pdus.len()); + pdus.retain(|id, _| events.contains(id)); // Only return state events pdus } } diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 7b15396a3..e49f6e7d9 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,11 +1,10 @@ use std::{collections::HashMap, time::Instant}; -use askama::filters::e; use conduwuit::{ Event, PduEvent, debug, debug_info, debug_warn, info, trace, utils::{BoolExt, IterStream, stream::BroadbandExt}, }; -use futures::{StreamExt, future::ok}; +use futures::StreamExt; use ruma::{RoomId, ServerName}; use crate::rooms::event_handler::build_local_dag; From 8a5708b9f91d31fca9947dda10a2f340bc45a3f1 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 22:05:19 +0100 Subject: [PATCH 40/81] fix: Be noisy when there's no incoming state --- .../event_handler/upgrade_outlier_pdu.rs | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 5d47d03ad..182169755 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -33,18 +33,8 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( origin: &ServerName, room_id: &RoomId, ) -> Result> { - // Skip the PDU if we already have it as a timeline event - if let Ok(pduid) = self - .services - .timeline - .get_pdu_id(incoming_pdu.event_id()) - .await - { - trace!(event_id=%incoming_pdu.event_id(), "Skipping upgrade of already upgraded PDU"); - return Ok(Some(pduid)); - } - - let (rejected, soft_failed) = join!( + let (pduid, rejected, soft_failed) = join!( + self.services.timeline.get_pdu_id(incoming_pdu.event_id()), self.services .pdu_metadata .is_event_rejected(incoming_pdu.event_id()), @@ -52,7 +42,10 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( .pdu_metadata .is_event_soft_failed(incoming_pdu.event_id()) ); - if rejected { + if let Ok(id) = pduid { + trace!(event_id=%incoming_pdu.event_id(), "Skipping upgrade of already upgraded PDU"); + return Ok(Some(id)); + } else if rejected { return Err!(Request(Forbidden("Event has been rejected"))); } else if soft_failed { return Err!(Request(Forbidden("Event has been soft-failed"))); @@ -97,7 +90,11 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( let state_at_incoming_event = state_at_incoming_event.expect("we always set this to some above"); - debug_assert!(!state_at_incoming_event.is_empty(), "Event has no incoming state"); + if state_at_incoming_event.is_empty() + && *incoming_pdu.event_type() != StateEventType::RoomCreate.into() + { + return Err!(Request(Forbidden("Incoming event has empty incoming state at"))); + } trace!(state_events = state_at_incoming_event.len(), "Calculated incoming state"); debug!( From 2ec739478597d9db0962c21516c72513ac2da977 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 22:48:23 +0100 Subject: [PATCH 41/81] perf: Don't add trees we already have to latest boundary --- .../fetch_and_handle_outliers.rs | 63 ++++++++++++------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 606754015..a30799442 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -8,10 +8,10 @@ Err, Event, PduEvent, debug, debug_error, debug_info, debug_warn, err, error, state_res::lexicographical_topological_sort, trace, - utils::{IterStream, math::Expected, stream::BroadbandExt}, + utils::{BoolExt, IterStream, math::Expected}, warn, }; -use futures::{StreamExt, future::select_ok}; +use futures::{StreamExt, TryStreamExt, future::select_ok}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedServerName, RoomId, ServerName, UInt, @@ -136,23 +136,13 @@ pub async fn get_missing_events( let missing_count = head .prev_events() .stream() - .broad_filter_map(|event_id| async move { - match self - .services - .timeline - .get_non_outlier_pdu_json(event_id) - .await - .inspect( - |_| debug!(elapsed=?start.elapsed(),"Found prev_event {event_id} locally."), - ) - .inspect_err( - |e| debug!(elapsed=?start.elapsed(),%e, "Could not find prev_event {event_id} locally."), - ) { - | Ok(_) => None, - | Err(_) => Some(event_id), + .fold(0_u8, |i, event_id| async move { + if self.services.timeline.pdu_exists(event_id).await { + i.expected_add(1) + } else { + i } }) - .count() .await; debug_assert_ne!( missing_count, 0, @@ -189,6 +179,7 @@ pub async fn get_missing_events( %limit, %via, %iteration, + %iteration_limit, discovered=discovered.len(), %min_depth, "Attempting to gap fill missing events" @@ -210,21 +201,30 @@ pub async fn get_missing_events( .await?; if response.events.is_empty() { - debug_info!(elapsed=?start.elapsed(),%via, "Finished gap filling missing events (remote returned no more events)."); + debug_info!( + elapsed=?start.elapsed(), + %via, + "Finished gap filling missing events (remote returned no more events)." + ); break; } - debug_info!(elapsed=?start.elapsed(),"Got {} events back from remote", response.events.len()); + debug_info!( + elapsed=?start.elapsed(), + "Got {} events back from remote", + response.events.len() + ); latest_events.clear(); for raw_event in response.events { let (_, event_id, pdu_json) = self.parse_incoming_pdu(&raw_event).await?; let pdu = PduEvent::from_id_val(&event_id, pdu_json).map_err(|e| { - err!(Request(BadJson("Failed to parse backfilled event {event_id}: {e}"))) + err!(Request(BadJson("Failed to parse gapfilled event {event_id}: {e}"))) })?; if pdu.depth < min_depth { debug_warn!( - elapsed=?start.elapsed(),"Received PDU with depth {} below min_depth {}", + elapsed=?start.elapsed(), + "Received PDU with depth {} below min_depth {}", pdu.depth, min_depth ); @@ -248,6 +248,27 @@ pub async fn get_missing_events( // immediately. continue; } + if let Ok(outlier) = self.services.timeline.get_pdu(prev_event_id).await { + // We already have this PDU as an outlier, don't ask for + // it. However, if we are missing any prev events for it, add it to the + // latest events anyway. + let outlier_missing_prevs = outlier + .prev_events() + .stream() + .fold(0_u8, |i, event_id| async move { + if self.services.timeline.pdu_exists(event_id).await { + i.expected_add(1) + } else { + i + } + }) + .await; + if outlier_missing_prevs > 0 { + latest_events.push(prev_event_id.to_owned()); + } + discovered.insert(prev_event_id.to_owned(), outlier); + continue; + } latest_events.push(prev_event_id.to_owned()); } From 5a0d6461d1c844ac85032bcec0ad6840626be144 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 30 May 2026 22:56:00 +0100 Subject: [PATCH 42/81] perf: Don't try to re-persist non-outliers we already have --- .../fetch_and_handle_outliers.rs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index a30799442..93ef42e09 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -220,6 +220,23 @@ pub async fn get_missing_events( let pdu = PduEvent::from_id_val(&event_id, pdu_json).map_err(|e| { err!(Request(BadJson("Failed to parse gapfilled event {event_id}: {e}"))) })?; + if discovered.contains_key(&event_id) { + // We already received this event. + trace!("Already received {event_id}"); + continue; + } + if self + .services + .timeline + .non_outlier_pdu_exists(&event_id) + .await + { + // NOTE: we explicitly check for *non*-outlier events here, as if we end + // up discovering outlier events, we will be able to upgrade them + // immediately. + trace!("Already have {event_id} as a timeline PDU"); + continue; + } if pdu.depth < min_depth { debug_warn!( @@ -235,6 +252,7 @@ pub async fn get_missing_events( for prev_event_id in pdu.prev_events() { if discovered.contains_key(prev_event_id) { // We already received this event. + trace!("Already received prev event {prev_event_id}"); continue; } if self @@ -246,6 +264,7 @@ pub async fn get_missing_events( // NOTE: we explicitly check for *non*-outlier events here, as if we end // up discovering outlier events, we will be able to upgrade them // immediately. + trace!("Already have prev event {prev_event_id} as a timeline PDU"); continue; } if let Ok(outlier) = self.services.timeline.get_pdu(prev_event_id).await { @@ -264,14 +283,17 @@ pub async fn get_missing_events( }) .await; if outlier_missing_prevs > 0 { + trace!("Missing {outlier_missing_prevs} PDU(s) for prev event"); latest_events.push(prev_event_id.to_owned()); } + trace!("Had {prev_event_id} as an outlier already, skipping discovery"); discovered.insert(prev_event_id.to_owned(), outlier); continue; } + trace!("Missing prev {prev_event_id} of {event_id}"); latest_events.push(prev_event_id.to_owned()); } - + trace!("Discovered {event_id}"); discovered.insert(event_id.clone(), pdu); } From d461c6977a415d784ffe88ccc70b32a8aa7038d8 Mon Sep 17 00:00:00 2001 From: timedout Date: Sun, 31 May 2026 00:17:30 +0100 Subject: [PATCH 43/81] fix: Friendly assertations --- .../rooms/event_handler/fetch_state.rs | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index f399e1696..2ec208bb4 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -96,6 +96,12 @@ pub(super) async fn fetch_state( }) .collect() .await; + assert_eq!( + state_events.len(), + res.pdu_ids.len(), + "Failed to load all required state events despite allegedly knowing all of them \ + already", + ); } else { let total_count = res.pdu_ids.len(); let missing_count = to_fetch.len(); @@ -132,6 +138,11 @@ pub(super) async fn fetch_state( .await }, }; + assert!( + !fetched_state.is_empty(), + "fetch_full_state or fetch_and_handle_missing_events returned empty state \ + map" + ); state_events.extend(fetched_state); } else { debug!( @@ -139,10 +150,14 @@ pub(super) async fn fetch_state( to_fetch = to_fetch.len(), "Fetching missing events for state from remote" ); - state_events.extend( - self.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) - .await, + let fetched_state = self + .fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) + .await; + assert!( + !fetched_state.is_empty(), + "fetch_and_handle_missing_events returned empty state map" ); + state_events.extend(fetched_state); } } From 29ce21cd2ee814727083862b5bbd818b127127c0 Mon Sep 17 00:00:00 2001 From: timedout Date: Sun, 31 May 2026 15:55:08 +0100 Subject: [PATCH 44/81] fix: Correctly pre-populate state events vec with known events --- .../fetch_and_handle_outliers.rs | 1 + .../rooms/event_handler/fetch_state.rs | 46 +++++++++++++++---- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 93ef42e09..18eaa0095 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -528,6 +528,7 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( trace!(elapsed=?start.elapsed(),"Finished fetch_and_handle_missing_events: fetched and handled {} missing PDUs", pdus.len()); pdus.retain(|id, _| events.contains(id)); // Only return state events + trace!(elapsed=?start.elapsed(), "Filtered return value down to {} PDUs", pdus.len()); pdus } } diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 2ec208bb4..bf8388236 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -5,7 +5,8 @@ }; use conduwuit::{ - Err, Event, PduEvent, Result, debug, debug_warn, err, error, trace, utils::IterStream, warn, + Err, Event, PduEvent, Result, debug, debug_warn, err, error, info, trace, utils::IterStream, + warn, }; use futures::{StreamExt, TryFutureExt, future::select_ok}; use ruma::{ @@ -70,11 +71,11 @@ pub(super) async fn fetch_state( .into_iter() .stream() .broad_filter_map(|event_id| async move { - if self.services.timeline.pdu_exists(&event_id).await { - None - } else { - Some(event_id) - } + self.services + .timeline + .pdu_exists(&event_id) + .await + .then_some(event_id) }) .collect() .await; @@ -105,22 +106,32 @@ pub(super) async fn fetch_state( } else { let total_count = res.pdu_ids.len(); let missing_count = to_fetch.len(); - if missing_count >= max(50, total_count >> 2) { + let missing_threshold = max(50, total_count >> 2); + if missing_count >= missing_threshold { // If there's more than 50 events to fetch, or we're missing 25% or more of the // state, we would need to make a lot of atomic requests, so we'll just try // to fetch the full state from the remote instead. // Since this endpoint might fail in huge rooms, we fall back to atomic fetch // anyway. - debug_warn!( + warn!( elapsed=?start.elapsed(), %missing_count, %total_count, + %missing_threshold, "Fetching full state from remote server for event" ); let fetched_state = match self .fetch_full_state(origin, create_event, room_id, event_id) .await - { + .inspect(|_| { + info!( + elapsed=?start.elapsed(), + %missing_count, + %total_count, + %missing_threshold, + "Fetched full state from remote server for event" + ); + }) { | Ok(state) => state, | Err(e) => { error!( @@ -138,6 +149,9 @@ pub(super) async fn fetch_state( .await }, }; + // TODO: Should probably verify that fetched_state solely consists of PDUs we're + // expecting, otherwise this might be vulnerable to the "me when I lie attack", + // maybe? assert!( !fetched_state.is_empty(), "fetch_full_state or fetch_and_handle_missing_events returned empty state \ @@ -145,6 +159,20 @@ pub(super) async fn fetch_state( ); state_events.extend(fetched_state); } else { + state_events = res + .pdu_ids + .iter() + .stream() + .broad_filter_map(|event_id| async move { + self.services + .timeline + .get_pdu(event_id) + .await + .map(|p| (event_id.to_owned(), p)) + .ok() + }) + .collect() + .await; debug!( elapsed=?start.elapsed(), to_fetch = to_fetch.len(), From 3e4d6b25653f50e0840b857703e233d806b1e8d0 Mon Sep 17 00:00:00 2001 From: timedout Date: Sun, 31 May 2026 16:24:03 +0100 Subject: [PATCH 45/81] perf: Always fetch at least N events per GME --- .../rooms/event_handler/fetch_and_handle_outliers.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 18eaa0095..938321887 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -174,7 +174,13 @@ pub async fn get_missing_events( let limit = iteration .expected_add(1) .saturating_mul(10) - .min(GET_MISSING_EVENTS_MAX_BATCH_SIZE); + .min(GET_MISSING_EVENTS_MAX_BATCH_SIZE) + .max( + // This max call ensures we fetch *at least* all the prev events the + // head has. + u16::try_from(head.prev_events.len()) + .expect("cannot have more than 20 prev events, which fits in u16"), + ); debug_info!(elapsed=?start.elapsed(), %limit, %via, From c28ea44e1127cf81dd4dcdb066876801828ab8bf Mon Sep 17 00:00:00 2001 From: timedout Date: Sun, 31 May 2026 16:35:09 +0100 Subject: [PATCH 46/81] fix: Correct inverted boolean condition, add explicit timeout on /state fetch --- .../rooms/event_handler/fetch_state.rs | 57 ++++++++++++------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index bf8388236..40db2b831 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,14 +1,15 @@ use std::{ cmp::max, collections::{HashMap, hash_map}, - time::Instant, + time::{Duration, Instant}, }; use conduwuit::{ - Err, Event, PduEvent, Result, debug, debug_warn, err, error, info, trace, utils::IterStream, + Err, Event, PduEvent, Result, debug, debug_warn, err, info, trace, + utils::{BoolExt, IterStream}, warn, }; -use futures::{StreamExt, TryFutureExt, future::select_ok}; +use futures::{FutureExt, StreamExt, TryFutureExt, future::select_ok}; use ruma::{ EventId, OwnedEventId, OwnedRoomId, RoomId, ServerName, api::federation::event::{get_room_state, get_room_state_ids}, @@ -75,7 +76,7 @@ pub(super) async fn fetch_state( .timeline .pdu_exists(&event_id) .await - .then_some(event_id) + .or_some(event_id) }) .collect() .await; @@ -120,25 +121,43 @@ pub(super) async fn fetch_state( %missing_threshold, "Fetching full state from remote server for event" ); - let fetched_state = match self - .fetch_full_state(origin, create_event, room_id, event_id) - .await - .inspect(|_| { - info!( - elapsed=?start.elapsed(), - %missing_count, - %total_count, - %missing_threshold, - "Fetched full state from remote server for event" - ); - }) { - | Ok(state) => state, + let state_response = tokio::time::timeout( + Duration::from_secs(30), + self.fetch_full_state(origin, create_event, room_id, event_id), + ) + .await; + info!( + elapsed=?start.elapsed(), + %missing_count, + %total_count, + %missing_threshold, + "Fetched full state from remote server for event" + ); + let fetched_state = match state_response { + | Ok(wrapped) => match wrapped { + | Ok(state) => state, + | Err(e) => { + warn!( + elapsed=?start.elapsed(), + error=?e, + %origin, + "Failed to fetch full state from remote, falling back to atomic fetch" + ); + self.fetch_and_handle_missing_events( + origin, + to_fetch, + create_event, + room_id, + ) + .await + }, + }, | Err(e) => { - error!( + warn!( elapsed=?start.elapsed(), error=?e, %origin, - "Failed to fetch full state from remote, falling back to atomic fetch" + "Remote did not return room state in an acceptable timeframe, falling back to atomic fetch" ); self.fetch_and_handle_missing_events( origin, From a945a4b2ada87e1c3e17e83550fe7ecfcfbdbc6c Mon Sep 17 00:00:00 2001 From: timedout Date: Sun, 31 May 2026 17:26:26 +0100 Subject: [PATCH 47/81] fix: Correctly handle still-missing state, always fetch full state atomically if regular fetch fails --- .../event_handler/fetch_and_handle_outliers.rs | 7 ++++++- src/service/rooms/event_handler/fetch_state.rs | 16 ++++++++++------ .../rooms/event_handler/upgrade_outlier_pdu.rs | 6 +++++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 938321887..286cce0b8 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -424,7 +424,12 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( } let attempts = seen.get(&*next_id).copied().unwrap_or_default(); if attempts >= 5 { - debug_error!(elapsed=?start.elapsed(),%attempts, %next_id, "Could not fetch missing event after 5 attempts, giving up"); + debug_error!( + elapsed=?start.elapsed(), + %attempts, + %next_id, + "Could not fetch missing event after 5 attempts, giving up" + ); continue; } diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 40db2b831..0a46129c9 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -145,7 +145,7 @@ pub(super) async fn fetch_state( ); self.fetch_and_handle_missing_events( origin, - to_fetch, + res.pdu_ids.clone(), create_event, room_id, ) @@ -161,7 +161,7 @@ pub(super) async fn fetch_state( ); self.fetch_and_handle_missing_events( origin, - to_fetch, + res.pdu_ids.clone(), create_event, room_id, ) @@ -192,6 +192,11 @@ pub(super) async fn fetch_state( }) .collect() .await; + assert!( + !state_events.is_empty(), + "Only missing {} events but read-ahead state vec was empty", + to_fetch.len() + ); debug!( elapsed=?start.elapsed(), to_fetch = to_fetch.len(), @@ -200,13 +205,12 @@ pub(super) async fn fetch_state( let fetched_state = self .fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) .await; - assert!( - !fetched_state.is_empty(), - "fetch_and_handle_missing_events returned empty state map" - ); state_events.extend(fetched_state); } } + if state_events.is_empty() { + return Ok(Some(HashMap::new())); + } let mut state: HashMap = HashMap::with_capacity(state_events.len()); diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 182169755..24bc81f6a 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -93,7 +93,11 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( if state_at_incoming_event.is_empty() && *incoming_pdu.event_type() != StateEventType::RoomCreate.into() { - return Err!(Request(Forbidden("Incoming event has empty incoming state at"))); + // This can happen if the remote sends an event but cannot be reached to fetch + // the state at it, and all other servers in the room (which might just be the + // unreachable server) are unable to provide required info. + // returning an error here allows the upgrade to be attempted at another time. + return Err!(Request(Forbidden("Could not resolve incoming state at event"))); } trace!(state_events = state_at_incoming_event.len(), "Calculated incoming state"); From 486dcd208c1869ca1234a86d43757e1664427d3f Mon Sep 17 00:00:00 2001 From: timedout Date: Sun, 31 May 2026 17:41:15 +0100 Subject: [PATCH 48/81] style: Resolve lint complaints --- .../rooms/event_handler/fetch_and_handle_outliers.rs | 8 +++++--- src/service/rooms/event_handler/fetch_state.rs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 286cce0b8..acb5cc2b7 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -4,14 +4,16 @@ }; use assign::assign; +#[cfg(debug_assertions)] +use conduwuit::error; use conduwuit::{ - Err, Event, PduEvent, debug, debug_error, debug_info, debug_warn, err, error, + Err, Event, PduEvent, debug, debug_error, debug_info, debug_warn, err, state_res::lexicographical_topological_sort, trace, - utils::{BoolExt, IterStream, math::Expected}, + utils::{IterStream, math::Expected}, warn, }; -use futures::{StreamExt, TryStreamExt, future::select_ok}; +use futures::{StreamExt, future::select_ok}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedServerName, RoomId, ServerName, UInt, diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 0a46129c9..7a82bdb06 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -9,7 +9,7 @@ utils::{BoolExt, IterStream}, warn, }; -use futures::{FutureExt, StreamExt, TryFutureExt, future::select_ok}; +use futures::{StreamExt, TryFutureExt, future::select_ok}; use ruma::{ EventId, OwnedEventId, OwnedRoomId, RoomId, ServerName, api::federation::event::{get_room_state, get_room_state_ids}, From 31737e127edec26808bf3fc5e11298d4e7248178 Mon Sep 17 00:00:00 2001 From: timedout Date: Sun, 31 May 2026 17:47:22 +0100 Subject: [PATCH 49/81] fix: Make `fetch_state_ids_from_backfill_servers` candidate-free safe --- src/service/rooms/event_handler/fetch_state.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 7a82bdb06..d9908a625 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -255,6 +255,11 @@ async fn fetch_state_ids_from_backfill_servers( .timeline .candidate_backfill_servers(&room_id) .await; + if candidates.is_empty() { + return Err!(Request(NotFound( + "Cannot ask any other servers for the state at this event" + ))); + } debug!(%room_id, ?candidates, "Asking backfill servers for state_ids"); let futures = candidates.iter().map(|server_name| { Box::pin( From b0612397d368375b4af1acbe317493a7cdff97e3 Mon Sep 17 00:00:00 2001 From: timedout Date: Sun, 7 Jun 2026 02:05:37 +0100 Subject: [PATCH 50/81] style: Tidy up --- src/service/rooms/event_handler/fetch_prev.rs | 5 +-- .../event_handler/handle_incoming_pdu.rs | 37 ++++++++++--------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index e49f6e7d9..e9a883c33 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, time::Instant}; use conduwuit::{ - Event, PduEvent, debug, debug_info, debug_warn, info, trace, + Event, PduEvent, debug, debug_info, debug_warn, trace, utils::{BoolExt, IterStream, stream::BroadbandExt}, }; use futures::StreamExt; @@ -84,13 +84,12 @@ pub(super) async fn fetch_prevs( let job_start = Instant::now(); trace!("Starting to persist {} prev events", to_persist.len()); for (i, event_id) in to_persist.iter().enumerate() { - info!( + debug!( elapsed=?start.elapsed(), "[TODO] Persisting fetched prev event: {event_id} ({}/{})", i.saturating_add(1), to_persist.len(), ); - debug_info!(elapsed=?start.elapsed(), "Persisting fetched prev event {event_id}"); let obj = mapped.get(event_id).cloned().unwrap(); let persist_start = Instant::now(); match self diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 6f90ac602..71fddde76 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -1,8 +1,8 @@ use std::{collections::BTreeMap, time::Instant}; use conduwuit::{ - Err, Event, PduEvent, Result, debug_error, debug_info, defer, err, error, implement, info, - trace, warn, + Err, Event, PduEvent, Result, debug, debug_error, debug_info, defer, err, error, implement, + info, result::DebugInspect, trace, warn, }; use futures::{ FutureExt, @@ -221,37 +221,38 @@ pub async fn handle_incoming_pdu<'a>( .remove(room_id); }}; - info!("[TODO] Handling PDU as outlier"); let (incoming_pdu, val) = self .handle_outlier_pdu(origin, create_event, event_id, room_id, value, false) - .await - .inspect_err(|e| error!("[TODO] Failed to handle outlier PDU: {e:?}"))?; - info!("[TODO] Finished handling PDU as outlier"); + .await?; + // 8. if not timeline event: stop if !is_timeline_event { - info!("[TODO] Not upgrading PDU"); return Ok(None); } - // Skip old events - // let first_ts_in_room = self - // .services - // .timeline - // .first_pdu_in_room(room_id) - // .await? - // .origin_server_ts(); + // Skip events sent before we joined (they need to be persisted as backfilled + // events, not timeline events, which is handled elsewhere). + let first_ts_in_room = self + .services + .timeline + .first_pdu_in_room(room_id) + .await? + .origin_server_ts(); + if incoming_pdu.origin_server_ts() < first_ts_in_room { + return Ok(None); + } // 9. Fetch any missing prev events doing all checks listed here starting at 1. // These are timeline events - info!("[TODO] Fetching prev events"); + debug!("Fetching and persisting any missing prev events"); self.fetch_prevs(room_id, create_event, &incoming_pdu, origin) .await - .inspect_err(|e| error!("[TODO] Failed to fetch_prevs: {e:?}"))?; + .debug_inspect_err(|e| { + error!("Failed to fetch and persist incoming event's prev_events: {e:?}"); + })?; - info!("[TODO] Finished fetching prev events, attempting to upgrade"); // Done with prev events, now handling the incoming event self.upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) .await - .inspect_err(|e| error!("[TODO] Failed to upgrade outlier to timeline pdu: {e:?}")) } From a3544353babd6e99e47b4720f6f3b7e65af974d1 Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 16 Jun 2026 18:12:39 +0100 Subject: [PATCH 51/81] feat: Automatically squash extremities when they exceed a threshold Attempts to tackle #1844 --- conduwuit-example.toml | 8 +++ src/core/config/mod.rs | 12 ++++ .../event_handler/handle_incoming_pdu.rs | 66 +++++++++++++++++-- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index cc736826b..1742db047 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -645,6 +645,14 @@ # #default_room_acl_deny = +# The number of forward extremities to tolerate in a room before +# attempting to manually squash them with a "dummy event". Setting this +# above 20 will hinder its efficacy, and setting it below 5 will cause +# more dummy events to be sent than necessary (which increases federation +# traffic). +# +#dummy_event_threshold = 10 + # Enable OpenTelemetry OTLP tracing export. This replaces the deprecated # Jaeger exporter. Traces will be sent via OTLP to a collector (such as # Jaeger) that supports the OpenTelemetry Protocol. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 22c47e67d..f8a0846d0 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -781,6 +781,16 @@ pub struct Config { /// a substitute for moderation bots. pub default_room_acl_deny: Option>, + /// The number of forward extremities to tolerate in a room before + /// attempting to manually squash them with a "dummy event". Setting this + /// above 20 will hinder its efficacy, and setting it below 5 will cause + /// more dummy events to be sent than necessary (which increases federation + /// traffic). + /// + /// default: 10 + #[serde(default = "default_extremity_threshold")] + pub dummy_event_threshold: u8, + /// display: nested #[serde(default)] pub well_known: WellKnownConfig, @@ -2652,6 +2662,8 @@ fn default_rocksdb_stats_level() -> u8 { 1 } #[inline] pub fn default_default_room_version() -> RoomVersionId { RoomVersionId::V12 } +fn default_extremity_threshold() -> u8 { 10 } + fn default_ip_range_denylist() -> Vec { vec![ "127.0.0.0/8".to_owned(), diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 71fddde76..750de59d8 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -1,11 +1,11 @@ use std::{collections::BTreeMap, time::Instant}; use conduwuit::{ - Err, Event, PduEvent, Result, debug, debug_error, debug_info, defer, err, error, implement, - info, result::DebugInspect, trace, warn, + Err, Event, PduEvent, Result, debug, debug_error, debug_info, debug_warn, defer, err, error, + implement, info, matrix::PartialPdu, result::DebugInspect, trace, warn, }; use futures::{ - FutureExt, + FutureExt, StreamExt, future::{OptionFuture, try_join4}, }; use ruma::{ @@ -15,6 +15,7 @@ room::member::{MembershipState, RoomMemberEventContent}, }, }; +use serde_json::{json, value::to_raw_value}; use crate::rooms::timeline::{RawPduId, pdu_fits}; @@ -219,7 +220,7 @@ pub async fn handle_incoming_pdu<'a>( self.federation_handletime .write() .remove(room_id); - }}; + }} let (incoming_pdu, val) = self .handle_outlier_pdu(origin, create_event, event_id, room_id, value, false) @@ -253,6 +254,59 @@ pub async fn handle_incoming_pdu<'a>( })?; // Done with prev events, now handling the incoming event - self.upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) - .await + let pdu_id = self + .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) + .await?; + + let extremities_count = self + .services + .state + .get_forward_extremities(room_id) + .count() + .await; + + if extremities_count >= self.services.server.config.dummy_event_threshold.into() { + debug_warn!( + count=%extremities_count, + threshold=%self.services.server.config.dummy_event_threshold, + "Attempting to squash extremities after upgrading pdu" + ); + // Try to send a dummy event to squash extremities. See issue #1844 + while let Some(user_id) = self + .services + .state_cache + .local_users_in_room(room_id) + .next() + .await + { + let state_lock = self.services.state.mutex.lock(room_id).await; + if self + .services + .timeline + .build_and_append_pdu( + PartialPdu { + event_type: "org.matrix.dummy_event".into(), + content: to_raw_value(&json!({})).expect("john json"), + unsigned: None, + state_key: None, + redacts: None, + timestamp: None, + }, + &user_id, + Some(room_id), + &state_lock, + ) + .await + .inspect(|_| debug!(sender=%user_id, "Successfully sent a dummy event")) + .inspect_err( + |e| debug!(sender=%user_id, ?e, "Failed to send a dummy event via user"), + ) + .is_ok() + { + break; + } + } + } + + Ok(pdu_id) } From 3faedc4581a2848264e90095aa45727301557013 Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 16 Jun 2026 18:27:57 +0100 Subject: [PATCH 52/81] perf: Remove huge clone and tackle TODOs --- .../fetch_and_handle_outliers.rs | 20 ++++++++----------- src/service/rooms/event_handler/fetch_prev.rs | 6 ++++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index acb5cc2b7..e649abcb7 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -35,7 +35,7 @@ /// them in a processable order, so this is just a best effort attempt. It does /// not account for power levels or other tie breaks. pub async fn build_local_dag( - pdu_map: &HashMap, + pdu_map: &HashMap, ) -> conduwuit::Result> { debug_assert!(pdu_map.len() >= 2, "needless call to build_local_dag with less than 2 PDUs"); let mut dag: HashMap> = @@ -491,7 +491,6 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( } // Insert this PDU back at the end of the queue so that it will be resolved once // all of its auth events have been fetched. - // TODO: This may result in infinite looping, needs a breaker if have_all_auth { debug!(elapsed=?start.elapsed(),%next_id, "Have all auth events"); discovered_events.insert(next_id, value); @@ -505,12 +504,13 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( } } - let seeded_ordered = build_local_dag( - &discovered_events.clone(), /* TODO: this clones like several megabytes of data, - * owie :( */ - ) - .await - .expect("failed to build local DAG"); + let refmap: HashMap = discovered_events + .iter() + .map(|(id, data)| (id.clone(), data)) + .collect(); + let seeded_ordered = build_local_dag(&refmap) + .await + .expect("failed to build local DAG"); let mut pdus = HashMap::with_capacity(seeded_ordered.len()); for id in seeded_ordered { let pdu_json = discovered_events.remove(&id).unwrap(); @@ -533,10 +533,6 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( | Err(e) => warn!(elapsed=?start.elapsed(),"Authentication of event {id} failed: {e:?}"), } - - // TODO: should this try to promote to timeline? - // If we got here, we probably weren't able to promote it before - // now. } trace!(elapsed=?start.elapsed(),"Finished fetch_and_handle_missing_events: fetched and handled {} missing PDUs", pdus.len()); diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index e9a883c33..5f618a66c 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -5,7 +5,7 @@ utils::{BoolExt, IterStream, stream::BroadbandExt}, }; use futures::StreamExt; -use ruma::{RoomId, ServerName}; +use ruma::{CanonicalJsonObject, OwnedEventId, RoomId, ServerName}; use crate::rooms::event_handler::build_local_dag; @@ -78,7 +78,9 @@ pub(super) async fn fetch_prevs( let to_persist = if mapped.len() <= 1 { mapped.keys().map(ToOwned::to_owned).collect() } else { - build_local_dag(&mapped).await? + let refmap: HashMap = + mapped.iter().map(|(id, data)| (id.clone(), data)).collect(); + build_local_dag(&refmap).await? }; let job_start = Instant::now(); From ddb4ef539f8a08aa0acfb49bd5b064a3a0273ca2 Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 16 Jun 2026 18:31:09 +0100 Subject: [PATCH 53/81] fix: un-forget how streams work --- .../rooms/event_handler/handle_incoming_pdu.rs | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 750de59d8..a082b370b 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -15,7 +15,6 @@ room::member::{MembershipState, RoomMemberEventContent}, }, }; -use serde_json::{json, value::to_raw_value}; use crate::rooms::timeline::{RawPduId, pdu_fits}; @@ -272,13 +271,8 @@ pub async fn handle_incoming_pdu<'a>( "Attempting to squash extremities after upgrading pdu" ); // Try to send a dummy event to squash extremities. See issue #1844 - while let Some(user_id) = self - .services - .state_cache - .local_users_in_room(room_id) - .next() - .await - { + let mut local_users = self.services.state_cache.local_users_in_room(room_id); + while let Some(user_id) = local_users.next().await { let state_lock = self.services.state.mutex.lock(room_id).await; if self .services @@ -286,11 +280,7 @@ pub async fn handle_incoming_pdu<'a>( .build_and_append_pdu( PartialPdu { event_type: "org.matrix.dummy_event".into(), - content: to_raw_value(&json!({})).expect("john json"), - unsigned: None, - state_key: None, - redacts: None, - timestamp: None, + ..PartialPdu::default() }, &user_id, Some(room_id), From c0666a1793fdcbc9452caad84b649945c8024ce9 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 14:55:12 +0100 Subject: [PATCH 54/81] fix: Default PDU content to empty object instead of literal NULL --- src/core/matrix/pdu/partial.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/matrix/pdu/partial.rs b/src/core/matrix/pdu/partial.rs index db3010672..88db067fa 100644 --- a/src/core/matrix/pdu/partial.rs +++ b/src/core/matrix/pdu/partial.rs @@ -62,7 +62,7 @@ impl Default for PartialPdu { fn default() -> Self { Self { event_type: "m.room.message".into(), - content: Box::::default(), + content: to_raw_value("{}").unwrap(), unsigned: None, state_key: None, redacts: None, From 832ee8650b2e090c820a29ad591e5a6c6fa7a4d3 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 15:34:49 +0100 Subject: [PATCH 55/81] style: Adjust docstrings and dodgy comment --- .../fetch_and_handle_outliers.rs | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index e649abcb7..63bb6deeb 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -96,22 +96,20 @@ pub async fn build_local_dag( } impl super::Service { - /// Uses `/_matrix/federation/v1/get_missing_events` to fill gaps in the - /// DAG. + /// Uses `POST /_matrix/federation/v1/get_missing_events/{room_id}` to fill + /// gaps in the DAG. /// - /// When this function is called, the "earliest events" (current forward - /// extremities) will be collected, and the function will loop with an - /// exponentially incrementing limit (up to 100 per request) until it has - /// filled the gap, i.e. when the remote says there's no more events. + /// This function walks backwards from `head`, fetching incrementally (by a + /// factor of 10) more events until the remote we're fetching from either + /// stops returning new events, or the min_depth is reached. /// - /// This function will iterate until the remote returns no more events, - /// increasing the limit by a factor of 10. If 100 iterations are reached or - /// max_fetch_prev_events events are backfilled, the function will give up - /// and return what it has, to avoid pulling in too much data (for example, - /// absurdly large gaps). + /// This function does not persist the events, but does validate them. The + /// caller is responsible for passing them through handle_incoming_pdu or + /// related functions. /// - /// This function does not persist the events. The caller is responsible for - /// passing them through handle_incoming_pdu. + /// Only the one `via` is asked for missing events, as multiplexing remotes + /// may result in the event tree being walked in a gappy or disordered + /// manner. /// /// ## Parameters /// @@ -160,7 +158,8 @@ pub async fn get_missing_events( // that many events per-request. Synapse returns 20, and conduwuit+ return 50. // This means with a hard iteration limit, we might give up too early, before // we get a chance to even come close to max_fetch_prev_events. As such, we'll - // calculate the max limit based on that config option based on these averages. + // calculate the limit based on that config option and the aforementioned + // averages. let max_fetch = self.services.server.config.max_fetch_prev_events; let iteration_limit = max_fetch.saturating_div(20).max(10); @@ -350,6 +349,11 @@ pub async fn get_missing_events( Ok(discovered) } + /// Sends a `GET /_matrix/federation/v1/event/{event_id}` request to the + /// target `remote`, parses the resulting PDU, and ensures the remote + /// returned the correct event. + /// Allows `fetch_and_handle_missing_events` to atomically fetch events from + /// multiple remotes in parallel. async fn fetch_and_handle_missing_event_via( &self, remote: OwnedServerName, @@ -377,9 +381,10 @@ async fn fetch_and_handle_missing_event_via( } } - /// Asks remote servers for any individual events that are missing. Should - /// only be used for fetching missing auth events or resolving missing - /// events from state_ids. For all other uses, use get_missing_events. + /// Asks remote servers for any individual events that are missing, also + /// known as "atomic fetch". Should only be used for fetching missing auth + /// events or resolving missing events from state_ids. For all other uses, + /// use get_missing_events. #[tracing::instrument(name = "get_missing_events_atomic", skip_all)] pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( &self, From f1cde5f323bff7e21f59a150127e9667ebe734f0 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 15:36:49 +0100 Subject: [PATCH 56/81] style: Fix up some TODOs --- src/service/rooms/event_handler/fetch_prev.rs | 2 +- src/service/rooms/event_handler/fetch_state.rs | 1 - src/service/rooms/event_handler/handle_outlier_pdu.rs | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 5f618a66c..c36a0f3b5 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -88,7 +88,7 @@ pub(super) async fn fetch_prevs( for (i, event_id) in to_persist.iter().enumerate() { debug!( elapsed=?start.elapsed(), - "[TODO] Persisting fetched prev event: {event_id} ({}/{})", + "Persisting fetched prev event: {event_id} ({}/{})", i.saturating_add(1), to_persist.len(), ); diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index d9908a625..198270e63 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -282,7 +282,6 @@ async fn fetch_state_ids_from_backfill_servers( /// state events as outliers, for use later. /// /// Any events that cannot be persisted are dropped with a warning. - /// TODO: make it noisy? pub(super) async fn fetch_full_state( &self, origin: &ServerName, diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 4f167fcd3..58d5c9113 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -164,8 +164,6 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( ))); } } - // TODO: do events received from auth chain need persisting? that sounds - // awfully slow } // 6. Reject "due to auth events" if the event doesn't pass auth based on the From 968328d78840562e5a115e376568a448f393941d Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 15:44:15 +0100 Subject: [PATCH 57/81] perf: Squash weird mutable variable --- .../rooms/event_handler/fetch_state.rs | 9 +++----- .../event_handler/upgrade_outlier_pdu.rs | 21 +++++++++---------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 198270e63..be2940175 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -26,9 +26,6 @@ impl super::Service { /// to requesting the full state in PDU format from the remote (`GET /// /_matrix/federation/v1/state, very slow in large rooms), and persists /// them directly. - /// - /// The end result is a result containing a map of shortstatekeys to event - /// IDs. The underlying `Option` is always `Some`. #[tracing::instrument(skip_all)] pub(super) async fn fetch_state( &self, @@ -36,7 +33,7 @@ pub(super) async fn fetch_state( create_event: &PduEvent, room_id: &RoomId, event_id: &EventId, - ) -> Result>> { + ) -> Result> { let start = Instant::now(); trace!(%origin, "Asking remote for state_ids"); let res: get_room_state_ids::v1::Response = match self @@ -209,7 +206,7 @@ pub(super) async fn fetch_state( } } if state_events.is_empty() { - return Ok(Some(HashMap::new())); + return Ok(HashMap::new()); } let mut state: HashMap = @@ -242,7 +239,7 @@ pub(super) async fn fetch_state( } } trace!(elapsed=?start.elapsed(), "fetch_state finished"); - Ok(Some(state)) + Ok(state) } async fn fetch_state_ids_from_backfill_servers( diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 24bc81f6a..2b3ef3f98 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -73,23 +73,22 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( event_id = %incoming_pdu.event_id, "Resolving state at event" ); - let mut state_at_incoming_event = if incoming_pdu.prev_events().count() == 1 { + let state_at_incoming_event = if incoming_pdu.prev_events().count() == 1 { self.state_at_incoming_degree_one(&incoming_pdu).await? } else { self.state_at_incoming_resolved(&incoming_pdu, room_id, &room_version_rules) .await? }; + let state_at_incoming_event = match state_at_incoming_event { + | Some(s) => s, + | None => { + trace!("Could not calculate incoming state, asking remote {origin} for it"); + self.fetch_state(origin, create_event, room_id, incoming_pdu.event_id()) + .await + .debug_inspect_err(|e| debug_error!("Could not fetch state from {origin}: {e}"))? + }, + }; - if state_at_incoming_event.is_none() { - trace!("Could not calculate incoming state, asking remote {origin} for it"); - state_at_incoming_event = self - .fetch_state(origin, create_event, room_id, incoming_pdu.event_id()) - .await - .debug_inspect_err(|e| debug_error!("Could not fetch state from {origin}: {e}"))?; - } - - let state_at_incoming_event = - state_at_incoming_event.expect("we always set this to some above"); if state_at_incoming_event.is_empty() && *incoming_pdu.event_type() != StateEventType::RoomCreate.into() { From 115a2e802e6ff5a9f8ba02d315dbe8926f6ccd86 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 15:54:56 +0100 Subject: [PATCH 58/81] style: Check power levels before attempting to send extremity squashes Solves a problem where the console screams in agony when local users can't send dummy events --- .../rooms/event_handler/handle_incoming_pdu.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index a082b370b..696de0300 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -271,8 +271,19 @@ pub async fn handle_incoming_pdu<'a>( "Attempting to squash extremities after upgrading pdu" ); // Try to send a dummy event to squash extremities. See issue #1844 + let power_levels = self + .services + .state_accessor + .get_room_power_levels(room_id) + .await; let mut local_users = self.services.state_cache.local_users_in_room(room_id); while let Some(user_id) = local_users.next().await { + if power_levels.for_user(&user_id) + < power_levels.for_message("org.matrix.dummy_event".into()) + { + trace!(%user_id, "user does not have power level to send dummy event, skipping"); + continue; + } let state_lock = self.services.state.mutex.lock(room_id).await; if self .services From aa1281a9e028ab70d4b15ce55948c1c58c0fc027 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 16:17:17 +0100 Subject: [PATCH 59/81] fix: Prevent arbitrary state injection attack --- .../rooms/event_handler/fetch_state.rs | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index be2940175..7424d57cc 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -4,6 +4,7 @@ time::{Duration, Instant}, }; +use askama::filters::e; use conduwuit::{ Err, Event, PduEvent, Result, debug, debug_warn, err, info, trace, utils::{BoolExt, IterStream}, @@ -131,23 +132,35 @@ pub(super) async fn fetch_state( "Fetched full state from remote server for event" ); let fetched_state = match state_response { - | Ok(wrapped) => match wrapped { - | Ok(state) => state, - | Err(e) => { - warn!( - elapsed=?start.elapsed(), - error=?e, - %origin, - "Failed to fetch full state from remote, falling back to atomic fetch" - ); - self.fetch_and_handle_missing_events( - origin, - res.pdu_ids.clone(), - create_event, - room_id, - ) + | Ok(Ok(state)) => { + // Filter to ensure we only use the PDUs we were expecting, preventing + // arbitrary state injection. + // Atomic fetch does not have this problem as each PDU is evaluated + // individually. + let expected = &res.pdu_ids; + state + .into_iter() + .stream() + .broad_filter_map(|(event_id, pdu)| async move { + expected.contains(&event_id).then_some((event_id, pdu)) + }) + .collect() .await - }, + }, + | Ok(Err(e)) => { + warn!( + elapsed=?start.elapsed(), + error=?e, + %origin, + "Failed to fetch full state from remote, falling back to atomic fetch" + ); + self.fetch_and_handle_missing_events( + origin, + res.pdu_ids.clone(), + create_event, + room_id, + ) + .await }, | Err(e) => { warn!( @@ -165,9 +178,7 @@ pub(super) async fn fetch_state( .await }, }; - // TODO: Should probably verify that fetched_state solely consists of PDUs we're - // expecting, otherwise this might be vulnerable to the "me when I lie attack", - // maybe? + assert!( !fetched_state.is_empty(), "fetch_full_state or fetch_and_handle_missing_events returned empty state \ From ab3be337cb7d4929098b0430095737019809b76d Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 16:19:39 +0100 Subject: [PATCH 60/81] chore: Reformat --- src/service/rooms/event_handler/fetch_state.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 7424d57cc..414c3cbf9 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -4,7 +4,6 @@ time::{Duration, Instant}, }; -use askama::filters::e; use conduwuit::{ Err, Event, PduEvent, Result, debug, debug_warn, err, info, trace, utils::{BoolExt, IterStream}, From bcadecdc3b844e83253167c34e1689f6ad87919c Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 17:59:28 +0100 Subject: [PATCH 61/81] feat: Add `!admin debug rooms-by-extremity-count` command --- src/admin/debug/commands.rs | 52 +++++++++++++++++++++++++++++++++- src/admin/debug/mod.rs | 5 ++++ src/service/rooms/state/mod.rs | 10 +++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index f9a678f19..87a81d893 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -31,7 +31,7 @@ }; use tracing_subscriber::EnvFilter; -use crate::admin_command; +use crate::{PAGE_SIZE, admin_command}; #[admin_command] pub(super) async fn echo(&self, message: Vec) -> Result { @@ -1138,3 +1138,53 @@ pub(super) async fn send_test_email(&self) -> Result { Ok(()) } + +#[admin_command] +pub(super) async fn rooms_by_extremity_count(&self, page: Option) -> Result { + let page = page.unwrap_or(1); + // My Giant Chain:tm: + let mapped: HashMap = self + .services + .rooms + .state + .all_forward_extremities() + .ready_fold(HashMap::new(), move |mut map, (room_id, _)| { + let count: u64 = map.get(&room_id).copied().unwrap_or(0); + map.insert(room_id, count.saturating_add(1)); + map + }) + .await + .into_iter() + .filter_map(|(room_id, count)| (count >= 2).then_some((room_id, count))) + .collect(); + if mapped.is_empty() { + return Err!("No more rooms."); + } + + let mut rooms = mapped.keys().collect::>(); + rooms.sort_by_key(|room_id| { + mapped + .get(*room_id) + .copied() + .expect("keys must have values") + }); + rooms.reverse(); + + let body = rooms + .into_iter() + .stream() + .skip(page.saturating_sub(1).saturating_mul(PAGE_SIZE)) + .take(PAGE_SIZE) + .map(|room_id| { + format!("{room_id}: {}", mapped.get(room_id).copied().expect("keys must have values")) + }) + .collect::>() + .await; + + self.write_str(&format!( + "Rooms by extremity count ({}):\n```\n{}\n```", + body.len(), + body.join("\n") + )) + .await +} diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index a9478ce82..0a2e0ae7e 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -237,6 +237,11 @@ pub enum DebugCommand { /// Send a test email to the invoking admin's email address SendTestEmail, + /// Lists room IDs by forward extremity count in descending order + RoomsByExtremityCount { + page: Option, + }, + /// Developer test stubs #[command(subcommand)] #[allow(non_snake_case)] diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index ccc4be87e..bb0061f08 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -371,6 +371,16 @@ pub async fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result impl Stream + Send { + self.db + .roomid_pduleaves + .keys() + .map_ok(|(room_id, event_id): (OwnedRoomId, OwnedEventId)| (room_id, event_id)) + .ignore_err() + } + pub fn get_forward_extremities<'a>( &'a self, room_id: &'a RoomId, From c141503ccb381b9b7c25b1a3164619834e206257 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 18:10:51 +0100 Subject: [PATCH 62/81] style: Re-use `GET_MISSING_EVENTS_MAX_BATCH_SIZE` --- src/api/server/get_missing_events.rs | 10 +++------- .../rooms/event_handler/fetch_and_handle_outliers.rs | 6 ++++-- src/service/rooms/event_handler/mod.rs | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index a4915bb8b..9a42db4c5 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -4,15 +4,11 @@ use conduwuit::{Err, Event, Result, debug, info, trace, utils::to_canonical_object, warn}; use ruma::{OwnedEventId, api::federation::event::get_missing_events}; use serde_json::{json, value::RawValue}; +use service::rooms::event_handler::GET_MISSING_EVENTS_MAX_BATCH_SIZE; use super::AccessCheck; use crate::Ruma; -/// arbitrary number but synapse's is 20 and we can handle lots of these anyways -const LIMIT_MAX: usize = 50; -/// spec says default is 10 -const LIMIT_DEFAULT: usize = 10; - /// # `POST /_matrix/federation/v1/get_missing_events/{roomId}` /// /// Retrieves events that the sender is missing. @@ -45,8 +41,8 @@ pub(crate) async fn get_missing_events_route( let limit = body .limit .try_into() - .unwrap_or(LIMIT_DEFAULT) - .min(LIMIT_MAX); + .unwrap_or(10) + .min(GET_MISSING_EVENTS_MAX_BATCH_SIZE); let room_version = services.rooms.state.get_room_version(&body.room_id).await?; diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 63bb6deeb..35d96962a 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -25,7 +25,7 @@ use super::get_room_version_rules; use crate::rooms::event_handler::parse_incoming_pdu::expect_event_id_array; -const GET_MISSING_EVENTS_MAX_BATCH_SIZE: u16 = 50; // matches src/server/get_missing_events.rs#LIMIT_MAX +pub const GET_MISSING_EVENTS_MAX_BATCH_SIZE: usize = 50; /// Attempts to build a localised directed acyclic graph out of the given PDUs, /// returning them in a topologically sorted order. @@ -175,7 +175,9 @@ pub async fn get_missing_events( let limit = iteration .expected_add(1) .saturating_mul(10) - .min(GET_MISSING_EVENTS_MAX_BATCH_SIZE) + .min(GET_MISSING_EVENTS_MAX_BATCH_SIZE.try_into().expect( + "GET_MISSING_EVENTS_MAX_BATCH_SIZE (usize) should fit in u16 (<=65536)", + )) .max( // This max call ensures we fetch *at least* all the prev events the // head has. diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index f51d701f0..c11853ecd 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -14,7 +14,7 @@ use async_trait::async_trait; use conduwuit::{Err, Event, PduEvent, Result, Server, SyncRwLock, utils::MutexMap}; -pub use fetch_and_handle_outliers::build_local_dag; +pub use fetch_and_handle_outliers::{GET_MISSING_EVENTS_MAX_BATCH_SIZE, build_local_dag}; use ruma::{ OwnedEventId, OwnedRoomId, RoomId, events::room::create::RoomCreateEventContent, room_version_rules::RoomVersionRules, From 689a1ce59bcd69cda23767949c56cf8429a385c3 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 18:49:06 +0100 Subject: [PATCH 63/81] style: Use `user_can_send_message` --- src/service/rooms/event_handler/handle_incoming_pdu.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 696de0300..152f17f13 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -278,9 +278,7 @@ pub async fn handle_incoming_pdu<'a>( .await; let mut local_users = self.services.state_cache.local_users_in_room(room_id); while let Some(user_id) = local_users.next().await { - if power_levels.for_user(&user_id) - < power_levels.for_message("org.matrix.dummy_event".into()) - { + if power_levels.user_can_send_message(&user_id, "org.matrix.dummy_event".into()) { trace!(%user_id, "user does not have power level to send dummy event, skipping"); continue; } From 439bc2784deb63b1861f1f98bb23d253353f5569 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 18:58:15 +0100 Subject: [PATCH 64/81] style: Use more explicit variable names --- .../rooms/event_handler/handle_outlier_pdu.rs | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 58d5c9113..b9f8f3c10 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -100,22 +100,30 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( // Fetch all auth events let mut auth_events: HashMap = HashMap::new(); - for aid in pdu_event.auth_events() { - if self.services.pdu_metadata.is_event_rejected(aid).await { + for auth_event_id in pdu_event.auth_events() { + if self + .services + .pdu_metadata + .is_event_rejected(auth_event_id) + .await + { debug_warn!( - "Rejecting incoming event {} which depends on rejected auth event {aid}", + "Rejecting incoming event {} which depends on rejected auth event \ + {auth_event_id}", event_id, ); self.services.pdu_metadata.mark_event_rejected(event_id); - return Err!(Request(Forbidden("Event has rejected auth event: {aid}"))); + return Err!(Request(Forbidden("Event has rejected auth event: {auth_event_id}"))); } - if let Ok(auth_event) = self.services.timeline.get_pdu(aid).await { + if let Ok(auth_event) = self.services.timeline.get_pdu(auth_event_id).await { check_room_id(room_id, &auth_event)?; - trace!("Found auth event {aid} for outlier event {event_id} locally"); - auth_events.insert(aid.to_owned(), auth_event); + trace!("Found auth event {auth_event_id} for outlier event {event_id} locally"); + auth_events.insert(auth_event_id.to_owned(), auth_event); } else { - debug_warn!("Could not find auth event {aid} for outlier event {event_id} locally"); + debug_warn!( + "Could not find auth event {auth_event_id} for outlier event {event_id} locally" + ); } } @@ -151,16 +159,16 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( .map_err(|e| err!(Request(BadJson("Invalid PDU {auth_event_id}: {e}"))))?; auth_chain_map.insert(auth_event_id, auth_pdu); } - for aid in pdu_event.auth_events() { - if auth_events.contains_key(aid) { + for auth_event_id in pdu_event.auth_events() { + if auth_events.contains_key(auth_event_id) { continue; } - if let Some(auth_event) = auth_chain_map.get(aid) { - auth_events.insert(aid.to_owned(), auth_event.clone()); + if let Some(auth_event) = auth_chain_map.get(auth_event_id) { + auth_events.insert(auth_event_id.to_owned(), auth_event.clone()); } else { return Err!(Request(Forbidden( "Remote server is not divulging incoming event's auth events (missing: \ - {aid})" + {auth_event_id})" ))); } } From 0c03195aec0ff4092de44c900007a176e23147d4 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 19:51:06 +0100 Subject: [PATCH 65/81] fix: Fall back to legacy behaviour when prev events are missed from get_missing_events --- .../fetch_and_handle_outliers.rs | 137 +++++++++++++++--- src/service/rooms/event_handler/fetch_prev.rs | 18 ++- .../rooms/event_handler/fetch_state.rs | 6 +- .../event_handler/handle_incoming_pdu.rs | 2 +- 4 files changed, 136 insertions(+), 27 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 35d96962a..b986b16de 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -10,7 +10,7 @@ Err, Event, PduEvent, debug, debug_error, debug_info, debug_warn, err, state_res::lexicographical_topological_sort, trace, - utils::{IterStream, math::Expected}, + utils::{IterStream, math::Expected, stream::BroadbandExt}, warn, }; use futures::{StreamExt, future::select_ok}; @@ -356,7 +356,7 @@ pub async fn get_missing_events( /// returned the correct event. /// Allows `fetch_and_handle_missing_events` to atomically fetch events from /// multiple remotes in parallel. - async fn fetch_and_handle_missing_event_via( + async fn fetch_event_via( &self, remote: OwnedServerName, event_id: OwnedEventId, @@ -383,17 +383,42 @@ async fn fetch_and_handle_missing_event_via( } } + async fn fetch_event_vias( + &self, + candidates: impl Iterator, + event_id: &EventId, + room_version_rules: &RoomVersionRules, + ) -> conduwuit::Result<(OwnedEventId, CanonicalJsonObject)> { + if let Ok(pdu_json) = self.services.timeline.get_pdu_json(event_id).await { + return Ok((event_id.to_owned(), pdu_json)); + } + let futures = candidates + .map(|remote| { + Box::pin(self.fetch_event_via( + remote.to_owned(), + event_id.to_owned(), + room_version_rules, + )) + }) + .collect::>(); + select_ok(futures).await.map(|(res, _)| res) + } + /// Asks remote servers for any individual events that are missing, also /// known as "atomic fetch". Should only be used for fetching missing auth /// events or resolving missing events from state_ids. For all other uses, /// use get_missing_events. - #[tracing::instrument(name = "get_missing_events_atomic", skip_all)] - pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( + /// + /// This function manually walks auth_events trees in a breadth-first + /// search, and persists all fetched events as outliers when all the + /// backwards extremities have been resolved. + #[tracing::instrument(name = "get_missing_auth_events_atomic", skip_all)] + pub(super) async fn fetch_and_handle_auth_events( &self, - origin: &'a ServerName, + origin: &ServerName, events: Vec, - create_event: &'a Pdu, - room_id: &'a RoomId, + create_event: &Pdu, + room_id: &RoomId, ) -> HashMap where Pdu: Event + Send + Sync, @@ -443,18 +468,11 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( } debug!(elapsed=?start.elapsed(),"Fetching {next_id} over federation"); - let futures = candidates - .iter() - .map(|remote| { - Box::pin(self.fetch_and_handle_missing_event_via( - remote.clone(), - next_id.clone(), - room_version_rules, - )) - }) - .collect::>(); - let (event_id, value) = match select_ok(futures).await { - | Ok((x, _)) => x, + let (event_id, value) = match self + .fetch_event_vias(candidates.iter(), &next_id, room_version_rules) + .await + { + | Ok(x) => x, | Err(e) => { warn!(elapsed=?start.elapsed(),"failed to fetch missing event {next_id} from any candidate: {e}"); continue; @@ -547,4 +565,85 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>( trace!(elapsed=?start.elapsed(), "Filtered return value down to {} PDUs", pdus.len()); pdus } + + /// Similar to `fetch_and_handle_missing_events`, but simply walks the + /// prev events tree instead of the auth events tree. Additionally, it does + /// not *handle* fetched PDUs in any capacity. + #[tracing::instrument(name = "get_missing_prev_events_atomic", skip_all)] + pub(super) async fn fetch_prev_events( + &self, + origin: &ServerName, + events: Vec, + create_event: &Pdu, + room_id: &RoomId, + ) -> HashMap + where + Pdu: Event + Send + Sync, + { + let room_version_rules = + &get_room_version_rules(create_event).unwrap_or(RoomVersionRules::V1); + let mut candidates = self + .services + .timeline + .candidate_backfill_servers(room_id) + .await; + candidates.insert(origin.to_owned()); + + let mut todo: VecDeque = VecDeque::from(events); + let mut discovered_events = HashMap::new(); + while let Some(next_id) = todo.pop_front() { + if discovered_events.len() >= self.services.server.config.max_fetch_prev_events.into() + { + debug_warn!( + "Encountered a gap too large to fill, giving up (fetched {} events)", + discovered_events.len() + ); + break; + } + let pdu = match self + .fetch_event_vias(candidates.iter(), &next_id, room_version_rules) + .await + { + | Ok((_, data)) => data, + | Err(e) => { + warn!("Failed to fetch prev event {next_id} from any candidate: {e}"); + continue; + }, + }; + + let prev_events = match expect_event_id_array(&pdu, "prev_events").map_err(|e| { + err!(Request(BadJson(warn!( + event_id=%next_id, + "Failed to parse event fetched from remote: {e}" + )))) + }) { + | Ok(auth_events) => auth_events, + | Err(e) => { + warn!(?e, "event {next_id} is malformed (bad prev_events), skipping"); + continue; + }, + }; + let missing_prev = prev_events + .iter() + .stream() + .broad_filter_map(|event_id| async { + if discovered_events.contains_key(event_id) + || self.services.timeline.pdu_exists(event_id).await + { + None + } else { + Some(event_id.to_owned()) + } + }) + .collect::>() + .await; + todo.extend(missing_prev); + discovered_events.insert( + next_id.clone(), + PduEvent::from_id_val(&next_id, pdu).expect("fetched PDU was already validated"), + ); + } + + discovered_events + } } diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index c36a0f3b5..2e95f39ab 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -5,7 +5,7 @@ utils::{BoolExt, IterStream, stream::BroadbandExt}, }; use futures::StreamExt; -use ruma::{CanonicalJsonObject, OwnedEventId, RoomId, ServerName}; +use ruma::{CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName}; use crate::rooms::event_handler::build_local_dag; @@ -18,9 +18,10 @@ pub(super) async fn fetch_prevs( create_event: &PduEvent, incoming_pdu: &PduEvent, origin: &ServerName, + first_ts_in_room: MilliSecondsSinceUnixEpoch, ) -> conduwuit::Result<()> { let start = Instant::now(); - let missing = incoming_pdu + let mut missing = incoming_pdu .prev_events() .stream() .broad_filter_map(|event_id| async move { @@ -45,7 +46,7 @@ pub(super) async fn fetch_prevs( .collect::>() .await; - let gapfilled = self + let mut gapfilled = self .get_missing_events( room_id, incoming_pdu, @@ -64,6 +65,14 @@ pub(super) async fn fetch_prevs( ) .await?; debug_info!(elapsed=?start.elapsed(), "Fetched {} missing events", gapfilled.len()); + missing.retain(|eid| !gapfilled.contains_key(eid)); + if !missing.is_empty() { + debug_warn!(elapsed=?start.elapsed(), "Still missing {} events, falling back to atomic fetch.", missing.len()); + gapfilled.extend( + self.fetch_prev_events(origin, missing, create_event, room_id) + .await, + ); + } // Persist all fetched events let mapped = gapfilled @@ -98,7 +107,7 @@ pub(super) async fn fetch_prevs( .handle_outlier_pdu(origin, create_event, event_id, room_id, obj, false) .await { - | Ok((pdu, val)) => { + | Ok((pdu, val)) if pdu.origin_server_ts() >= first_ts_in_room => { self.upgrade_outlier_to_timeline_pdu(pdu, val, create_event, origin, room_id) .await .inspect_err(|e| { @@ -125,6 +134,7 @@ pub(super) async fn fetch_prevs( task_elapsed=?persist_start.elapsed(), "Failed to persist prev event {event_id}: {e}", ), + | _ => {}, } } diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 414c3cbf9..ae306014f 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -153,7 +153,7 @@ pub(super) async fn fetch_state( %origin, "Failed to fetch full state from remote, falling back to atomic fetch" ); - self.fetch_and_handle_missing_events( + self.fetch_and_handle_auth_events( origin, res.pdu_ids.clone(), create_event, @@ -168,7 +168,7 @@ pub(super) async fn fetch_state( %origin, "Remote did not return room state in an acceptable timeframe, falling back to atomic fetch" ); - self.fetch_and_handle_missing_events( + self.fetch_and_handle_auth_events( origin, res.pdu_ids.clone(), create_event, @@ -210,7 +210,7 @@ pub(super) async fn fetch_state( "Fetching missing events for state from remote" ); let fetched_state = self - .fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id) + .fetch_and_handle_auth_events(origin, to_fetch, create_event, room_id) .await; state_events.extend(fetched_state); } diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 152f17f13..c05b114e1 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -246,7 +246,7 @@ pub async fn handle_incoming_pdu<'a>( // These are timeline events debug!("Fetching and persisting any missing prev events"); - self.fetch_prevs(room_id, create_event, &incoming_pdu, origin) + self.fetch_prevs(room_id, create_event, &incoming_pdu, origin, first_ts_in_room) .await .debug_inspect_err(|e| { error!("Failed to fetch and persist incoming event's prev_events: {e:?}"); From 4673282ca1abcac32a3e29d1654a074141791ec5 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 20:08:48 +0100 Subject: [PATCH 66/81] perf: Don't try to fetch prevs we already fetched graphs are hard --- src/service/rooms/event_handler/fetch_and_handle_outliers.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index b986b16de..4121ed087 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -600,6 +600,9 @@ pub(super) async fn fetch_prev_events( ); break; } + if discovered_events.contains_key(&next_id) { + continue; + } let pdu = match self .fetch_event_vias(candidates.iter(), &next_id, room_version_rules) .await From d1aa911739b74885f5f5e2f4aff39ad57c584c8e Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 21:06:40 +0100 Subject: [PATCH 67/81] fix: Rename variables in auth event fetcher Also fixes a couple bugs where events were being misattributed --- .../fetch_and_handle_outliers.rs | 86 +++++++++++-------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 4121ed087..03e351464 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -443,38 +443,39 @@ pub(super) async fn fetch_and_handle_auth_events( ); let mut seen: HashMap = HashMap::new(); - for id in &events { - let mut todo: VecDeque = [id.to_owned()].into(); - while let Some(next_id) = todo.pop_front() { - if discovered_events.contains_key(&next_id) { + for apex_event_id in &events { + let mut todo: VecDeque = [apex_event_id.to_owned()].into(); + + while let Some(target_id) = todo.pop_front() { + if discovered_events.contains_key(&target_id) { continue; } - if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { - trace!(elapsed=?start.elapsed(),"Found {next_id} in db"); + if let Ok(local_pdu) = self.services.timeline.get_pdu(&target_id).await { + trace!(elapsed=?start.elapsed(), "Found {target_id} in db"); let mut obj = local_pdu.into_canonical_object(); obj.remove("event_id"); - discovered_events.insert(next_id.clone(), obj); + discovered_events.insert(target_id.clone(), obj); continue; } - let attempts = seen.get(&*next_id).copied().unwrap_or_default(); + let attempts = seen.get(&*target_id).copied().unwrap_or_default(); if attempts >= 5 { debug_error!( elapsed=?start.elapsed(), %attempts, - %next_id, + %target_id, "Could not fetch missing event after 5 attempts, giving up" ); continue; } - debug!(elapsed=?start.elapsed(),"Fetching {next_id} over federation"); - let (event_id, value) = match self - .fetch_event_vias(candidates.iter(), &next_id, room_version_rules) + debug!(elapsed=?start.elapsed(),"Fetching {target_id} over federation"); + let value = match self + .fetch_event_vias(candidates.iter(), &target_id, room_version_rules) .await { - | Ok(x) => x, + | Ok((_, x)) => x, | Err(e) => { - warn!(elapsed=?start.elapsed(),"failed to fetch missing event {next_id} from any candidate: {e}"); + warn!(elapsed=?start.elapsed(),"failed to fetch missing event {target_id} from any candidate: {e}"); continue; }, }; @@ -482,7 +483,7 @@ pub(super) async fn fetch_and_handle_auth_events( match expect_event_id_array(&value, "auth_events").map_err(|e| { err!(Request(BadJson(warn!( elapsed=?start.elapsed(), - %event_id, + event_id=%target_id, "Failed to parse event fetched from remote: {e}" )))) }) { @@ -491,40 +492,46 @@ pub(super) async fn fetch_and_handle_auth_events( warn!( elapsed=?start.elapsed(), ?e, - "event {event_id} is malformed (bad auth_events), skipping" + "event {target_id} is malformed (bad auth_events), skipping" ); continue; }, }; let mut have_all_auth = true; for auth_event_id in auth_events { - if let Ok(local_pdu) = self.services.timeline.get_pdu(&next_id).await { - trace!(elapsed=?start.elapsed(),"Found auth event {next_id} in db"); + if let Ok(local_pdu) = self.services.timeline.get_pdu(&auth_event_id).await { + trace!(elapsed=?start.elapsed(),"Found auth event {auth_event_id} in db"); let mut obj = local_pdu.into_canonical_object(); obj.remove("event_id"); - discovered_events.insert(id.clone(), obj); + discovered_events.insert(auth_event_id.clone(), obj); continue; } if discovered_events.contains_key(&auth_event_id) { trace!(elapsed=?start.elapsed(),%auth_event_id, "Already found auth event"); continue; } - debug!(elapsed=?start.elapsed(),"Missing auth event {auth_event_id} for event {next_id}"); - seen.insert(auth_event_id.clone(), attempts.saturating_add(1)); + debug!(elapsed=?start.elapsed(),"Missing auth event {auth_event_id} for event {target_id}"); + seen.insert( + auth_event_id.clone(), + seen.get(&auth_event_id) + .copied() + .unwrap_or_default() + .saturating_add(1), + ); todo.push_back(auth_event_id); have_all_auth = false; } // Insert this PDU back at the end of the queue so that it will be resolved once // all of its auth events have been fetched. if have_all_auth { - debug!(elapsed=?start.elapsed(),%next_id, "Have all auth events"); - discovered_events.insert(next_id, value); + debug!(elapsed=?start.elapsed(),%target_id, "Have all auth events"); + discovered_events.insert(target_id, value); } else { debug_warn!(elapsed=?start.elapsed(), - "Fetched {next_id} but missing some auth events, will have to re-fetch." + "Fetched {target_id} but missing some auth events, will have to re-fetch." ); - seen.insert(next_id.clone(), attempts.saturating_add(1)); - todo.push_back(next_id); + seen.insert(target_id.clone(), attempts.saturating_add(1)); + todo.push_back(target_id); } } } @@ -537,14 +544,17 @@ pub(super) async fn fetch_and_handle_auth_events( .await .expect("failed to build local DAG"); let mut pdus = HashMap::with_capacity(seeded_ordered.len()); - for id in seeded_ordered { - let pdu_json = discovered_events.remove(&id).unwrap(); - debug_info!(elapsed=?start.elapsed(),"Handling missing event {id} as outlier"); + for discovered_event_id in seeded_ordered { + let pdu_json = discovered_events.remove(&discovered_event_id).unwrap(); + debug_info!( + elapsed=?start.elapsed(), + "Handling missing event {discovered_event_id} as outlier" + ); assert_eq!(pdu_json.get("event_id"), None, "pdu_json had event_id"); match Box::pin(self.handle_outlier_pdu( origin, create_event, - &id, + &discovered_event_id, room_id, pdu_json, true, @@ -552,15 +562,21 @@ pub(super) async fn fetch_and_handle_auth_events( .await { | Ok((pdu, _)) => { - trace!(elapsed=?start.elapsed(), "Persisted {id}"); - let _ = pdus.insert(id, pdu); + trace!(elapsed=?start.elapsed(), "Persisted {discovered_event_id}"); + let _ = pdus.insert(discovered_event_id, pdu); }, - | Err(e) => - warn!(elapsed=?start.elapsed(),"Authentication of event {id} failed: {e:?}"), + | Err(e) => warn!( + elapsed=?start.elapsed(), + "Authentication of event {discovered_event_id} failed: {e:?}" + ), } } - trace!(elapsed=?start.elapsed(),"Finished fetch_and_handle_missing_events: fetched and handled {} missing PDUs", pdus.len()); + trace!( + elapsed=?start.elapsed(), + "Finished fetch_and_handle_missing_events: fetched and handled {} missing PDUs", + pdus.len() + ); pdus.retain(|id, _| events.contains(id)); // Only return state events trace!(elapsed=?start.elapsed(), "Filtered return value down to {} PDUs", pdus.len()); pdus From 2efe8f2ec01c12d1868b512669706c4cef58d541 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 21:06:56 +0100 Subject: [PATCH 68/81] fix: Inverted power level check in extremity squash --- src/service/rooms/event_handler/handle_incoming_pdu.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index c05b114e1..019ef5f89 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -278,7 +278,7 @@ pub async fn handle_incoming_pdu<'a>( .await; let mut local_users = self.services.state_cache.local_users_in_room(room_id); while let Some(user_id) = local_users.next().await { - if power_levels.user_can_send_message(&user_id, "org.matrix.dummy_event".into()) { + if !power_levels.user_can_send_message(&user_id, "org.matrix.dummy_event".into()) { trace!(%user_id, "user does not have power level to send dummy event, skipping"); continue; } From b8ca06029f4b20a71b5bd7338dc73aca3d04e572 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 21:20:27 +0100 Subject: [PATCH 69/81] perf: Use a hashmap for full-state filtering --- src/service/rooms/event_handler/fetch_state.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index ae306014f..22f8c6acd 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,6 +1,7 @@ use std::{ cmp::max, - collections::{HashMap, hash_map}, + collections::{HashMap, HashSet, hash_map}, + hash::{BuildHasherDefault, DefaultHasher}, time::{Duration, Instant}, }; @@ -136,7 +137,8 @@ pub(super) async fn fetch_state( // arbitrary state injection. // Atomic fetch does not have this problem as each PDU is evaluated // individually. - let expected = &res.pdu_ids; + let expected: &HashSet> = + &HashSet::from_iter(res.pdu_ids.clone()); state .into_iter() .stream() From 7ca00e4ab9a51fad2b1faa6df7f1db52e92b9f80 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 17 Jun 2026 21:27:34 +0100 Subject: [PATCH 70/81] chore: Drop unused param from handle_outlier_pdu --- src/service/rooms/event_handler/fetch_and_handle_outliers.rs | 1 - src/service/rooms/event_handler/fetch_prev.rs | 2 +- src/service/rooms/event_handler/fetch_state.rs | 2 -- src/service/rooms/event_handler/handle_incoming_pdu.rs | 2 +- src/service/rooms/event_handler/handle_outlier_pdu.rs | 1 - 5 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 03e351464..c55159f35 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -557,7 +557,6 @@ pub(super) async fn fetch_and_handle_auth_events( &discovered_event_id, room_id, pdu_json, - true, )) .await { diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 2e95f39ab..d8e77cf7c 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -104,7 +104,7 @@ pub(super) async fn fetch_prevs( let obj = mapped.get(event_id).cloned().unwrap(); let persist_start = Instant::now(); match self - .handle_outlier_pdu(origin, create_event, event_id, room_id, obj, false) + .handle_outlier_pdu(origin, create_event, event_id, room_id, obj) .await { | Ok((pdu, val)) if pdu.origin_server_ts() >= first_ts_in_room => { diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 22f8c6acd..dbd0b75d6 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -331,7 +331,6 @@ pub(super) async fn fetch_full_state( &incoming_event_id, &incoming_room_id, incoming_event_json, - true, ) .await .inspect_err(|e| { @@ -368,7 +367,6 @@ pub(super) async fn fetch_full_state( &incoming_event_id, &incoming_room_id, incoming_event_json, - true, ) .await .inspect_err(|e| { diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 019ef5f89..e0b1d383e 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -222,7 +222,7 @@ pub async fn handle_incoming_pdu<'a>( }} let (incoming_pdu, val) = self - .handle_outlier_pdu(origin, create_event, event_id, room_id, value, false) + .handle_outlier_pdu(origin, create_event, event_id, room_id, value) .await?; // 8. if not timeline event: stop diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index b9f8f3c10..4d3bbc4d0 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -24,7 +24,6 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( event_id: &'a EventId, room_id: &'a RoomId, mut value: CanonicalJsonObject, - _auth_events_known: bool, ) -> Result<(PduEvent, BTreeMap)> where Pdu: Event + Send + Sync, From 3b6858e9366119420bfeada9457cfced9682ba37 Mon Sep 17 00:00:00 2001 From: timedout Date: Tue, 23 Jun 2026 15:26:28 +0100 Subject: [PATCH 71/81] perf: Throttle dummy events to prevent stampeding --- .../event_handler/handle_incoming_pdu.rs | 98 +++++++++++-------- src/service/rooms/event_handler/mod.rs | 7 ++ 2 files changed, 64 insertions(+), 41 deletions(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index e0b1d383e..329dcd7cc 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -1,4 +1,7 @@ -use std::{collections::BTreeMap, time::Instant}; +use std::{ + collections::BTreeMap, + time::{Duration, Instant}, +}; use conduwuit::{ Err, Event, PduEvent, Result, debug, debug_error, debug_info, debug_warn, defer, err, error, @@ -265,47 +268,60 @@ pub async fn handle_incoming_pdu<'a>( .await; if extremities_count >= self.services.server.config.dummy_event_threshold.into() { - debug_warn!( - count=%extremities_count, - threshold=%self.services.server.config.dummy_event_threshold, - "Attempting to squash extremities after upgrading pdu" - ); - // Try to send a dummy event to squash extremities. See issue #1844 - let power_levels = self - .services - .state_accessor - .get_room_power_levels(room_id) - .await; - let mut local_users = self.services.state_cache.local_users_in_room(room_id); - while let Some(user_id) = local_users.next().await { - if !power_levels.user_can_send_message(&user_id, "org.matrix.dummy_event".into()) { - trace!(%user_id, "user does not have power level to send dummy event, skipping"); - continue; - } - let state_lock = self.services.state.mutex.lock(room_id).await; - if self - .services - .timeline - .build_and_append_pdu( - PartialPdu { - event_type: "org.matrix.dummy_event".into(), - ..PartialPdu::default() - }, - &user_id, - Some(room_id), - &state_lock, - ) - .await - .inspect(|_| debug!(sender=%user_id, "Successfully sent a dummy event")) - .inspect_err( - |e| debug!(sender=%user_id, ?e, "Failed to send a dummy event via user"), - ) - .is_ok() - { - break; - } - } + self.squash_extremities(room_id, extremities_count).await; } Ok(pdu_id) } + +#[implement(super::Service)] +async fn squash_extremities(&self, room_id: &RoomId, count: usize) { + let last_squash = { + let squash_timings = self.last_extremity_squash.read(); + squash_timings.get(room_id).copied() + }; + if last_squash.is_some_and(|s| s.elapsed() < Duration::from_mins(1)) { + // Avoid sending more than one squash per minute to avoid flooding rooms. + return; + } + debug_warn!( + %count, + threshold=%self.services.server.config.dummy_event_threshold, + "Attempting to squash extremities after upgrading pdu" + ); + // Try to send a dummy event to squash extremities. See issue #1844 + let power_levels = self + .services + .state_accessor + .get_room_power_levels(room_id) + .await; + let mut local_users = self.services.state_cache.local_users_in_room(room_id); + while let Some(user_id) = local_users.next().await { + if !power_levels.user_can_send_message(&user_id, "org.matrix.dummy_event".into()) { + trace!(%user_id, "user does not have power level to send dummy event, skipping"); + continue; + } + let state_lock = self.services.state.mutex.lock(room_id).await; + if self + .services + .timeline + .build_and_append_pdu( + PartialPdu { + event_type: "org.matrix.dummy_event".into(), + ..PartialPdu::default() + }, + &user_id, + Some(room_id), + &state_lock, + ) + .await + .inspect(|_| debug!(sender=%user_id, "Successfully sent a dummy event")) + .inspect_err(|e| debug!(sender=%user_id, ?e, "Failed to send a dummy event via user")) + .is_ok() + { + break; + } + } + let mut squash_timings = self.last_extremity_squash.write(); + squash_timings.insert(room_id.to_owned(), Instant::now()); +} diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index c11853ecd..7235143ac 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -25,6 +25,7 @@ pub struct Service { pub mutex_federation: RoomMutexMap, pub federation_handletime: SyncRwLock, + pub last_extremity_squash: SyncRwLock>, services: Services, server_shutdown: Notify, } @@ -55,6 +56,7 @@ fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { mutex_federation: RoomMutexMap::new(), federation_handletime: HandleTimeMap::new().into(), + last_extremity_squash: SyncRwLock::new(HashMap::new()), services: Services { globals: args.depend::("globals"), sending: args.depend::("sending"), @@ -90,6 +92,11 @@ async fn memory_usage(&self, out: &mut (dyn Write + Send)) -> Result { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } fn interrupt(&self) { self.server_shutdown.notify_waiters(); } + + async fn clear_cache(&self) { + let mut squashes = self.last_extremity_squash.write(); + squashes.clear(); + } } impl Service { From 945ea5a78af4633513524c6f15c23871bc29e822 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 27 Jun 2026 20:39:50 +0100 Subject: [PATCH 72/81] chore: Box large futures --- src/service/rooms/event_handler/fetch_prev.rs | 44 +++++++++++-------- .../event_handler/handle_incoming_pdu.rs | 27 ++++++++---- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index d8e77cf7c..8f27d1be3 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -108,25 +108,31 @@ pub(super) async fn fetch_prevs( .await { | Ok((pdu, val)) if pdu.origin_server_ts() >= first_ts_in_room => { - self.upgrade_outlier_to_timeline_pdu(pdu, val, create_event, origin, room_id) - .await - .inspect_err(|e| { - debug_warn!( - total_elapsed=?start.elapsed(), - job_elapsed=?job_start.elapsed(), - task_elapsed=?persist_start.elapsed(), - "Failed to upgrade prev event {event_id}: {e}", - ); - }) - .inspect(|_| { - debug_info!( - total_elapsed=?start.elapsed(), - job_elapsed=?job_start.elapsed(), - task_elapsed=?persist_start.elapsed(), - "Upgraded prev event {event_id}", - ); - }) - .ok(); + Box::pin(self.upgrade_outlier_to_timeline_pdu( + pdu, + val, + create_event, + origin, + room_id, + )) + .await + .inspect_err(|e| { + debug_warn!( + total_elapsed=?start.elapsed(), + job_elapsed=?job_start.elapsed(), + task_elapsed=?persist_start.elapsed(), + "Failed to upgrade prev event {event_id}: {e}", + ); + }) + .inspect(|_| { + debug_info!( + total_elapsed=?start.elapsed(), + job_elapsed=?job_start.elapsed(), + task_elapsed=?persist_start.elapsed(), + "Upgraded prev event {event_id}", + ); + }) + .ok(); }, | Err(e) => debug_warn!( total_elapsed=?start.elapsed(), diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 645d4ae9f..0c9974ee8 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -255,16 +255,27 @@ pub async fn handle_incoming_pdu<'a>( // These are timeline events debug!("Fetching and persisting any missing prev events"); - self.fetch_prevs(room_id, create_event, &incoming_pdu, origin, first_ts_in_room) - .await - .debug_inspect_err(|e| { - error!("Failed to fetch and persist incoming event's prev_events: {e:?}"); - })?; + Box::pin(self.fetch_prevs( + room_id, + create_event, + &incoming_pdu, + origin, + first_ts_in_room, + )) + .await + .debug_inspect_err(|e| { + error!("Failed to fetch and persist incoming event's prev_events: {e:?}"); + })?; // Done with prev events, now handling the incoming event - let pdu_id = self - .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) - .await?; + let pdu_id = Box::pin(self.upgrade_outlier_to_timeline_pdu( + incoming_pdu, + val, + create_event, + origin, + room_id, + )) + .await?; let extremities_count = self .services From b6bc7dfc16d1b305466b2d1dd7e4e3af112cb081 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 28 Jun 2026 00:20:15 +0100 Subject: [PATCH 73/81] feat: Debounce extremity squashing Additionaly circuit-breaks it if the squash would have only b triggered by dummy events / other squashes --- .../event_handler/handle_incoming_pdu.rs | 111 ++++++++++++++++-- src/service/rooms/event_handler/mod.rs | 15 ++- 2 files changed, 105 insertions(+), 21 deletions(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 0c9974ee8..ae7fcb5e4 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -18,6 +18,7 @@ room::member::{MembershipState, RoomMemberEventContent}, }, }; +use tokio::sync::mpsc; use crate::rooms::timeline::{RawPduId, pdu_fits}; @@ -267,6 +268,8 @@ pub async fn handle_incoming_pdu<'a>( error!("Failed to fetch and persist incoming event's prev_events: {e:?}"); })?; + let is_dummy_event = incoming_pdu.event_type().to_string() == "org.matrix.dummy_event"; + // Done with prev events, now handling the incoming event let pdu_id = Box::pin(self.upgrade_outlier_to_timeline_pdu( incoming_pdu, @@ -284,24 +287,108 @@ pub async fn handle_incoming_pdu<'a>( .count() .await; - if extremities_count >= self.services.server.config.dummy_event_threshold.into() { - self.squash_extremities(room_id, extremities_count).await; - } + self.maybe_squash_extremities(room_id, extremities_count, is_dummy_event) + .await; Ok(pdu_id) } - async fn squash_extremities(&self, room_id: &RoomId, count: usize) { - let last_squash = { - let squash_timings = self.last_extremity_squash.read(); - squash_timings.get(room_id).copied() + async fn maybe_squash_extremities( + &self, + room_id: &RoomId, + extremities_count: usize, + is_dummy_event: bool, + ) { + let (tx, fut) = { + if let Some(tx) = self.extremity_squashers.read().get(room_id) + && !tx.is_closed() + { + (tx.clone(), None) + } else { + let mut map = self.extremity_squashers.upgradable_read(); + + if let Some(tx) = map.get(room_id) + && !tx.is_closed() + { + (tx.clone(), None) + } else { + let (tx, rx) = mpsc::channel(100); + map.with_upgraded(|map| map.insert(room_id.to_owned(), tx.clone())); + + (tx, Some(self.spawn_squasher(room_id, rx))) + } + } }; - if last_squash.is_some_and(|s| s.elapsed() < Duration::from_mins(1)) { - // Avoid sending more than one squash per minute to avoid flooding rooms. - return; + + if let Some(fut) = fut { + fut.await; } + let _ = tx.try_send((extremities_count, is_dummy_event)); + } + + async fn spawn_squasher(&self, room_id: &RoomId, mut rx: mpsc::Receiver<(usize, bool)>) { + let Some(service) = self.me.upgrade() else { + return; + }; + let room_id = room_id.to_owned(); + + self.services.server.runtime().spawn(async move { + let mut latest_extremity_count = None; + let mut non_dummy_event = false; + + let mut closing = false; + + let waker = tokio::time::sleep(Duration::from_mins(2)); + tokio::pin!(waker); + + loop { + tokio::select! { + msg = rx.recv() => { + if let Some((extremities_count, is_dummy_event)) = msg { + latest_extremity_count = Some(extremities_count); + non_dummy_event = non_dummy_event || !is_dummy_event; + #[allow(clippy::arithmetic_side_effects)] + waker.as_mut().reset(tokio::time::Instant::now() + Duration::from_mins(1)); + } else { + {let mut map = service.extremity_squashers.write(); + if let Some(tx) = map.get(&room_id) && tx.is_closed() { + map.remove(&room_id); + }} + + if let Some(count) = latest_extremity_count { + if non_dummy_event && count >= service.services.server.config.dummy_event_threshold.into() { + Self::squash_extremities(&service, &room_id, count).await; + } + } + break; + } + } + () = &mut waker, if !closing => { + if let Some(count) = latest_extremity_count { + if non_dummy_event && count >= service.services.server.config.dummy_event_threshold.into() { + Self::squash_extremities(&service, &room_id, count).await; + } + latest_extremity_count = None; + non_dummy_event = false; + #[allow(clippy::arithmetic_side_effects)] + waker.as_mut().reset(tokio::time::Instant::now() + Duration::from_mins(2)); + } else { + rx.close(); + closing = true; + } + } + () = service.server_shutdown.notified(), if !closing => { + rx.close(); + closing = true; + } + } + } + }); + } + + async fn squash_extremities(&self, room_id: &RoomId, extremities_count: usize) { debug_warn!( - %count, + %extremities_count, threshold=%self.services.server.config.dummy_event_threshold, "Attempting to squash extremities after upgrading pdu" ); @@ -340,7 +427,5 @@ async fn squash_extremities(&self, room_id: &RoomId, count: usize) { break; } } - let mut squash_timings = self.last_extremity_squash.write(); - squash_timings.insert(room_id.to_owned(), Instant::now()); } } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 616dae3a4..3eae2f477 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -19,15 +19,16 @@ OwnedEventId, OwnedRoomId, RoomId, events::room::create::RoomCreateEventContent, room_version_rules::RoomVersionRules, }; -use tokio::sync::Notify; +use tokio::sync::{Notify, mpsc}; use crate::{Dep, globals, rooms, sending, server_keys}; pub struct Service { pub mutex_federation: RoomMutexMap, pub federation_handletime: SyncRwLock, - pub last_extremity_squash: SyncRwLock>, + pub extremity_squashers: SyncRwLock>>, services: Services, server_shutdown: Notify, + me: std::sync::Weak, } struct Services { @@ -53,10 +54,11 @@ struct Services { #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { - Ok(Arc::new(Self { + Ok(Arc::new_cyclic(|s| Self { + me: s.clone(), mutex_federation: RoomMutexMap::new(), federation_handletime: HandleTimeMap::new().into(), - last_extremity_squash: SyncRwLock::new(HashMap::new()), + extremity_squashers: SyncRwLock::new(HashMap::new()), services: Services { globals: args.depend::("globals"), sending: args.depend::("sending"), @@ -93,10 +95,7 @@ fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } fn interrupt(&self) { self.server_shutdown.notify_waiters(); } - async fn clear_cache(&self) { - let mut squashes = self.last_extremity_squash.write(); - squashes.clear(); - } + async fn clear_cache(&self) {} } impl Service { From 5199cde870b4aaf43c1e5b1f3547128d8fc491a5 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 28 Jun 2026 03:44:23 +0100 Subject: [PATCH 74/81] feat: Allow sending dummy events to clients --- conduwuit-example.toml | 5 +++++ src/api/client/message.rs | 4 +++- src/core/config/mod.rs | 5 +++++ .../rooms/event_handler/handle_outlier_pdu.rs | 20 +++++++++++++------ .../event_handler/upgrade_outlier_pdu.rs | 14 +++++++------ src/service/rooms/timeline/backfill.rs | 2 +- 6 files changed, 36 insertions(+), 14 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 1742db047..893400ec1 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1436,6 +1436,11 @@ # #send_messages_from_ignored_users_to_client = false +# Send "org.matrix.dummy_event" events to the client. This is a debugging +# option. +# +#send_dummy_events_to_clients = false + # Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you # do not want continuwuity to send outbound requests to. Defaults to # RFC1918, unroutable, loopback, multicast, and testnet addresses for diff --git a/src/api/client/message.rs b/src/api/client/message.rs index a2da36e48..2c3ccc782 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -291,7 +291,9 @@ pub(crate) async fn is_ignored_pdu( { // exclude Synapse's dummy events from bloating up response bodies. clients // don't need to see this. - if event.kind().to_string() == "org.matrix.dummy_event" { + if !services.config.send_dummy_events_to_clients + && event.kind().to_string() == "org.matrix.dummy_event" + { return Ok(true); } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index f8a0846d0..661098485 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1661,6 +1661,11 @@ pub struct Config { #[serde(default)] pub send_messages_from_ignored_users_to_client: bool, + /// Send "org.matrix.dummy_event" events to the client. This is a debugging + /// option. + #[serde(default)] + pub send_dummy_events_to_clients: bool, + /// Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you /// do not want continuwuity to send outbound requests to. Defaults to /// RFC1918, unroutable, loopback, multicast, and testnet addresses for diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 5c7ef0bfa..470cd6643 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -1,8 +1,8 @@ use std::collections::{BTreeMap, HashMap, hash_map}; use conduwuit::{ - Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, info, state_res, - trace, warn, + Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, info, state_res, trace, + warn, }; use futures::future::ready; use ruma::{ @@ -104,14 +104,21 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( let mut auth_events: HashMap = HashMap::new(); for auth_event_id in pdu_event.auth_events() { - if self.services.pdu_metadata.is_event_rejected(auth_event_id).await { + if self + .services + .pdu_metadata + .is_event_rejected(auth_event_id) + .await + { debug_warn!( "Rejecting incoming event {} which depends on rejected auth event \ - {auth_event_id}", + {auth_event_id}", event_id, ); self.services.pdu_metadata.mark_event_rejected(event_id); - return Err!(Request(Forbidden("Event has rejected auth event: {auth_event_id}"))); + return Err!(Request(Forbidden( + "Event has rejected auth event: {auth_event_id}" + ))); } if let Ok(auth_event) = self.services.timeline.get_pdu(auth_event_id).await { @@ -120,7 +127,8 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>( auth_events.insert(auth_event_id.to_owned(), auth_event); } else { debug_warn!( - "Could not find auth event {auth_event_id} for outlier event {event_id} locally" + "Could not find auth event {auth_event_id} for outlier event {event_id} \ + locally" ); } } diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 845c62455..c6099ccf8 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -26,16 +26,16 @@ impl super::Service { #[tracing::instrument(name="upgrade_outlier", skip_all, fields(event_id=%incoming_pdu.event_id()))] pub(super) async fn upgrade_outlier_to_timeline_pdu( - &self, - incoming_pdu: PduEvent, - mut val: CanonicalJsonObject, - create_event: &PduEvent, + &self, + incoming_pdu: PduEvent, + mut val: CanonicalJsonObject, + create_event: &PduEvent, origin: &ServerName, room_id: &RoomId, ) -> Result> { let (pduid, rejected, soft_failed) = join!( self.services.timeline.get_pdu_id(incoming_pdu.event_id()), - self.services + self.services .pdu_metadata .is_event_rejected(incoming_pdu.event_id()), self.services @@ -85,7 +85,9 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( trace!("Could not calculate incoming state, asking remote {origin} for it"); self.fetch_state(origin, create_event, room_id, incoming_pdu.event_id()) .await - .debug_inspect_err(|e| debug_error!("Could not fetch state from {origin}: {e}"))? + .debug_inspect_err(|e| { + debug_error!("Could not fetch state from {origin}: {e}"); + })? }, }; diff --git a/src/service/rooms/timeline/backfill.rs b/src/service/rooms/timeline/backfill.rs index 3bbda9d80..36b01c8e8 100644 --- a/src/service/rooms/timeline/backfill.rs +++ b/src/service/rooms/timeline/backfill.rs @@ -228,7 +228,7 @@ pub(crate) async fn candidate_backfill_servers( &self, room_id: &RoomId, ) -> HashSet { - let mut candidate_backfill_servers = HashSet::new(); + let mut candidate_backfill_servers = HashSet::new(); let power_levels = self .services From 90797fa3cdb8f2023bc4e3a632c6ebe33703b6f9 Mon Sep 17 00:00:00 2001 From: timedout Date: Mon, 29 Jun 2026 17:44:13 +0100 Subject: [PATCH 75/81] fix: Avoid string roundtrip when fetching timestamps in `build_local_dag` --- .../fetch_and_handle_outliers.rs | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index c55159f35..53c7b4da6 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -8,6 +8,7 @@ use conduwuit::error; use conduwuit::{ Err, Event, PduEvent, debug, debug_error, debug_info, debug_warn, err, + result::FlatOk, state_res::lexicographical_topological_sort, trace, utils::{IterStream, math::Expected, stream::BroadbandExt}, @@ -52,11 +53,11 @@ pub async fn build_local_dag( // matter. let prev_events = value .get("prev_events") - .unwrap() - .as_array() - .unwrap() + .and_then(CanonicalJsonValue::as_array) + .expect("prev_events must be present in PDU JSON object") .iter() - .map(|v| EventId::parse(v.as_str().unwrap()).unwrap()) + .map(|v| v.as_str().expect("prev_events entries must be strings")) + .map(|v| EventId::parse(v).expect("prev_events entries must be valid event IDs")) .filter(|id| pdu_map.contains_key(id)) .collect(); @@ -64,6 +65,9 @@ pub async fn build_local_dag( let origin_server_ts = value .get("origin_server_ts") .and_then(CanonicalJsonValue::as_integer) + .map(i64::from) + .map(UInt::try_from) + .flat_ok() .unwrap_or_default(); id_origin_ts.insert(event_id.clone(), origin_server_ts); } @@ -73,15 +77,7 @@ pub async fn build_local_dag( // Note: we don't bother fetching power levels because that would massively slow // this function down. This is a best-effort attempt to order events correctly // for processing, however ultimately that should be the sender's job. - let ts = id_origin_ts - .get(&node_id) - .copied() - .unwrap_or_else(|| int!(0)) - .to_string() - .parse::() - .ok() - .and_then(UInt::new) - .unwrap_or_default(); + let ts = id_origin_ts.get(&node_id).copied().unwrap_or_default(); Ok((int!(0), MilliSecondsSinceUnixEpoch(ts))) }) .await From e936d183244ca1dd8d2e01cf9240935d2de55360 Mon Sep 17 00:00:00 2001 From: timedout Date: Mon, 29 Jun 2026 18:12:16 +0100 Subject: [PATCH 76/81] feat: Don't panic in `build_local_dag` Previously the function assumed the caller had performed proper validation on the inputs (and all current callers do), but this is a poor reason to panic when sane error handling is available. Events with no prev events now return an error, and prev events which are illegal are simply skipped. --- .../rooms/event_handler/fetch_and_handle_outliers.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 53c7b4da6..41104378f 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -54,11 +54,13 @@ pub async fn build_local_dag( let prev_events = value .get("prev_events") .and_then(CanonicalJsonValue::as_array) - .expect("prev_events must be present in PDU JSON object") + .ok_or_else(|| { + err!(Request(BadJson("event JSON for {event_id} is missing prev_events"))) + })? .iter() - .map(|v| v.as_str().expect("prev_events entries must be strings")) - .map(|v| EventId::parse(v).expect("prev_events entries must be valid event IDs")) - .filter(|id| pdu_map.contains_key(id)) + .map(|v| v.as_str().and_then(|s| EventId::parse(s).ok())) + .filter(|id| id.as_ref().is_some_and(|id| pdu_map.contains_key(id))) + .map(Option::unwrap) .collect(); dag.insert(event_id.clone(), prev_events); From 2bfd678c712450adcf08bfa98d7af9291288af65 Mon Sep 17 00:00:00 2001 From: timedout Date: Mon, 29 Jun 2026 18:29:04 +0100 Subject: [PATCH 77/81] feat: Add jitter to extremity squasher --- src/core/utils/time.rs | 3 +++ src/core/utils/time/jitter.rs | 16 ++++++++++++++++ src/service/announcements/mod.rs | 11 ++++++----- .../rooms/event_handler/handle_incoming_pdu.rs | 6 +++--- 4 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 src/core/utils/time/jitter.rs diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index b8fad41d8..2904d87e8 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,7 +1,10 @@ pub mod exponential_backoff; +pub mod jitter; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +pub use jitter::jitter; + use crate::{Result, err}; #[inline] diff --git a/src/core/utils/time/jitter.rs b/src/core/utils/time/jitter.rs new file mode 100644 index 000000000..cc90d89d3 --- /dev/null +++ b/src/core/utils/time/jitter.rs @@ -0,0 +1,16 @@ +use std::{ops::RangeInclusive, time::Duration}; + +/// Returns a `Duration` that is jittered by a random percentage of the base +/// duration. The jitter is applied as a random percentage in the range of +/// `-jitter_range` to `jitter_range`. +/// +/// # Example +/// ``` +/// use conduwuit_core::utils::time::jitter; +/// let sleep_duration = jitter(Duration::from_secs(1), -10..=10); +/// // Adds a jitter of between -10% and 10% to the duration. +/// ``` +#[must_use] +pub fn jitter(base: Duration, jitter_range: RangeInclusive) -> Duration { + base.mul_f64(1.0 + (rand::random_range(jitter_range)) / 100.0) +} diff --git a/src/service/announcements/mod.rs b/src/service/announcements/mod.rs index edee89b6e..fd0bced63 100644 --- a/src/service/announcements/mod.rs +++ b/src/service/announcements/mod.rs @@ -18,7 +18,11 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduwuit::{Result, Server, debug, error, utils::response::LimitReadExt, warn}; +use conduwuit::{ + Result, Server, debug, error, + utils::{response::LimitReadExt, time::jitter}, + warn, +}; use database::{Deserialized, Map}; use ruma::events::{Mentions, room::message::RoomMessageEventContent}; use serde::Deserialize; @@ -98,10 +102,7 @@ async fn worker(self: Arc) -> Result<()> { .ok(); } - let first_check_jitter = { - let jitter_percent = rand::random_range(-50.0..=10.0); - self.interval.mul_f64(1.0 + jitter_percent / 100.0) - }; + let first_check_jitter = jitter(self.interval, -50.0..=10.0); let mut i = interval(self.interval); i.set_missed_tick_behavior(MissedTickBehavior::Delay); diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index ae7fcb5e4..282608d34 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -5,7 +5,7 @@ use conduwuit::{ Err, Event, PduEvent, Result, debug, debug_error, debug_info, debug_warn, defer, err, error, - info, matrix::PartialPdu, result::DebugInspect, trace, warn, + info, matrix::PartialPdu, result::DebugInspect, trace, utils::time::jitter, warn, }; use futures::{ FutureExt, StreamExt, @@ -338,7 +338,7 @@ async fn spawn_squasher(&self, room_id: &RoomId, mut rx: mpsc::Receiver<(usize, let mut closing = false; - let waker = tokio::time::sleep(Duration::from_mins(2)); + let waker = tokio::time::sleep(jitter(Duration::from_mins(2), -25.0..=25.0)); tokio::pin!(waker); loop { @@ -348,7 +348,7 @@ async fn spawn_squasher(&self, room_id: &RoomId, mut rx: mpsc::Receiver<(usize, latest_extremity_count = Some(extremities_count); non_dummy_event = non_dummy_event || !is_dummy_event; #[allow(clippy::arithmetic_side_effects)] - waker.as_mut().reset(tokio::time::Instant::now() + Duration::from_mins(1)); + waker.as_mut().reset(tokio::time::Instant::now() + jitter(Duration::from_mins(1), -50.0..=50.0)); } else { {let mut map = service.extremity_squashers.write(); if let Some(tx) = map.get(&room_id) && tx.is_closed() { From 19a83c689191532e7f23a1ec4cab9813f523fd60 Mon Sep 17 00:00:00 2001 From: timedout Date: Mon, 29 Jun 2026 18:37:04 +0100 Subject: [PATCH 78/81] style: Add docstrings to squasher functions --- .../rooms/event_handler/handle_incoming_pdu.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 282608d34..63b7f71d8 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -293,6 +293,9 @@ pub async fn handle_incoming_pdu<'a>( Ok(pdu_id) } + /// Conditionally starts an extremity squasher. If there is no waiting + /// extremity squasher, a new one is created. Otherwise, the existing one is + /// pinged. async fn maybe_squash_extremities( &self, room_id: &RoomId, @@ -326,6 +329,7 @@ async fn maybe_squash_extremities( let _ = tx.try_send((extremities_count, is_dummy_event)); } + /// Spawns an extremity squasher with the given room and receiver channel. async fn spawn_squasher(&self, room_id: &RoomId, mut rx: mpsc::Receiver<(usize, bool)>) { let Some(service) = self.me.upgrade() else { return; @@ -386,6 +390,14 @@ async fn spawn_squasher(&self, room_id: &RoomId, mut rx: mpsc::Receiver<(usize, }); } + /// Squashes extremities in a room by sending dummy events (empty events + /// that are hidden from clients) to the room. It will only send ONE dummy + /// event to squash. If there are more than 20 extremities, multiple calls + /// to `squash_extremities` will be required. + /// Sending the dummy event will be attempted by iterating over each local + /// user currently joined to the room (including deactivated users) until + /// either one of them successfully builds and appends a dummy event PDU, or + /// there are no more users to try. async fn squash_extremities(&self, room_id: &RoomId, extremities_count: usize) { debug_warn!( %extremities_count, @@ -424,8 +436,9 @@ async fn squash_extremities(&self, room_id: &RoomId, extremities_count: usize) { ) .is_ok() { - break; + return; } } + debug_warn!("Unable to squash extremities using any local user"); } } From 9b80a99aa75a9fd8fa9e40a52d9237d215395be0 Mon Sep 17 00:00:00 2001 From: timedout Date: Mon, 29 Jun 2026 18:46:39 +0100 Subject: [PATCH 79/81] style: De-duplicate build_local_dag definitions, replace outdated comment on unwrap safety --- src/api/server/send.rs | 95 +++---------------- .../fetch_and_handle_outliers.rs | 14 +-- 2 files changed, 19 insertions(+), 90 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index bba4fb93e..47438ae85 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashMap}, net::IpAddr, time::{Duration, Instant}, }; @@ -9,7 +9,6 @@ use conduwuit::{ Err, Error, Result, debug, debug_error, debug_warn, err, error, result::LogErr, - state_res::lexicographical_topological_sort, trace, utils::{ IterStream, ReadyExt, millis_since_unix_epoch, @@ -25,8 +24,7 @@ use http::StatusCode; use itertools::Itertools; use ruma::{ - CanonicalJsonObject, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, - OwnedUserId, RoomId, ServerName, UInt, UserId, + CanonicalJsonObject, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, ServerName, UserId, api::{ error::{ErrorKind, LimitExceededErrorData}, federation::transactions::{ @@ -39,12 +37,12 @@ }, }, events::receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType}, - int, serde::Raw, to_device::DeviceIdOrAllDevices, }; -use service::transactions::{ - FederationTxnState, TransactionError, TxnKey, WrappedTransactionResponse, +use service::{ + rooms::event_handler::build_local_dag, + transactions::{FederationTxnState, TransactionError, TxnKey, WrappedTransactionResponse}, }; use tokio::sync::watch::{Receiver, Sender}; use tracing::instrument; @@ -276,74 +274,6 @@ async fn handle( Ok(results) } -/// Attempts to build a localised directed acyclic graph out of the given PDUs, -/// returning them in a topologically sorted order. -/// -/// This is used to attempt to process PDUs in an order that respects their -/// dependencies, however it is ultimately the sender's responsibility to send -/// them in a processable order, so this is just a best effort attempt. It does -/// not account for power levels or other tie breaks. -async fn build_local_dag( - pdu_map: &HashMap, -) -> Result> { - debug_assert!(pdu_map.len() >= 2, "needless call to build_local_dag with less than 2 PDUs"); - let mut dag: HashMap> = - HashMap::with_capacity(pdu_map.len()); - let mut id_origin_ts: HashMap = HashMap::with_capacity(pdu_map.len()); - - for (event_id, value) in pdu_map { - // We already checked that these properties are correct in parse_incoming_pdu, - // so it's safe to unwrap here. - // We also filter to remove any prev_events that are not in this pdu_map, as we - // need to have at least one event with zero out degrees for the lexico-topo - // sort below. If there are multiple events with omitted prevs, they will be - // ordered by timestamp, then event ID. At that point though, it's unlikely to - // matter. - let prev_events = value - .get("prev_events") - .unwrap() - .as_array() - .unwrap() - .iter() - .map(|v| EventId::parse(v.as_str().unwrap()).unwrap()) - .filter(|id| pdu_map.contains_key(id)) - .collect(); - - dag.insert(event_id.clone(), prev_events); - let origin_server_ts = value - .get("origin_server_ts") - .and_then(ruma::CanonicalJsonValue::as_integer) - .unwrap_or_default(); - id_origin_ts.insert(event_id.clone(), origin_server_ts); - } - - debug!(count = dag.len(), "Sorting incoming events with partial graph"); - lexicographical_topological_sort(&dag, &async |node_id| { - // Note: we don't bother fetching power levels because that would massively slow - // this function down. This is a best-effort attempt to order events correctly - // for processing, however ultimately that should be the sender's job. - let ts = id_origin_ts - .get(&node_id) - .copied() - .unwrap_or_else(|| int!(0)) - .to_string() - .parse::() - .ok() - .and_then(UInt::new) - .unwrap_or_default(); - Ok((int!(0), MilliSecondsSinceUnixEpoch(ts))) - }) - .await - .inspect(|sorted| { - debug_assert_eq!( - sorted.len(), - pdu_map.len(), - "Sorted graph was not the same size as the input graph" - ); - }) - .map_err(|e| err!("failed to resolve local graph: {e}")) -} - async fn handle_room( services: &Services, _client: &IpAddr, @@ -359,7 +289,7 @@ async fn handle_room( .await; let room_id = &room_id; - let pdu_map: HashMap = pdus + let mut pdu_map: HashMap = pdus .into_iter() .map(|(_, event_id, value)| (event_id, value)) .collect(); @@ -367,7 +297,11 @@ async fn handle_room( // failure (e.g., cycles). This is best-effort; proper ordering is the sender's // responsibility. let sorted_event_ids = if pdu_map.len() >= 2 { - build_local_dag(&pdu_map).await.unwrap_or_else(|e| { + let refmap = pdu_map + .iter() + .map(|(event_id, obj)| (event_id.clone(), obj)) + .collect(); + build_local_dag(&refmap).await.unwrap_or_else(|e| { debug_warn!("Failed to build local DAG for room {room_id}: {e}"); pdu_map.keys().cloned().collect() }) @@ -377,9 +311,8 @@ async fn handle_room( let mut results = Vec::with_capacity(sorted_event_ids.len()); for event_id in sorted_event_ids { let value = pdu_map - .get(&event_id) - .expect("sorted event IDs must be from the original map") - .clone(); + .remove(&event_id) + .expect("sorted event IDs must be from the original map"); services .server .check_running() @@ -387,7 +320,7 @@ async fn handle_room( let result = services .rooms .event_handler - .handle_incoming_pdu(origin, room_id, &event_id, value, true) + .handle_incoming_pdu(origin, room_id, &event_id, value.clone(), true) .boxed() .await .map(|_| ()); diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 41104378f..5b766f416 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -35,8 +35,9 @@ /// dependencies, however it is ultimately the sender's responsibility to send /// them in a processable order, so this is just a best effort attempt. It does /// not account for power levels or other tie breaks. -pub async fn build_local_dag( - pdu_map: &HashMap, +#[allow(clippy::implicit_hasher)] +pub async fn build_local_dag( + pdu_map: &HashMap, ) -> conduwuit::Result> { debug_assert!(pdu_map.len() >= 2, "needless call to build_local_dag with less than 2 PDUs"); let mut dag: HashMap> = @@ -44,13 +45,8 @@ pub async fn build_local_dag( let mut id_origin_ts: HashMap = HashMap::with_capacity(pdu_map.len()); for (event_id, value) in pdu_map { - // We already checked that these properties are correct in parse_incoming_pdu, - // so it's safe to unwrap here. - // We also filter to remove any prev_events that are not in this pdu_map, as we - // need to have at least one event with zero out degrees for the lexico-topo - // sort below. If there are multiple events with omitted prevs, they will be - // ordered by timestamp, then event ID. At that point though, it's unlikely to - // matter. + // Parse all prev events as event IDs - if they are missing, return an error (we + // can't sanely continue in this case), otherwise skip invalid prev events. let prev_events = value .get("prev_events") .and_then(CanonicalJsonValue::as_array) From 5fa3d5f6c81274721e07c09c1857f8786916a455 Mon Sep 17 00:00:00 2001 From: timedout Date: Mon, 29 Jun 2026 18:51:06 +0100 Subject: [PATCH 80/81] fix: Don't create fake objects when defaulting a partial PDU build --- src/core/matrix/pdu/partial.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/matrix/pdu/partial.rs b/src/core/matrix/pdu/partial.rs index 88db067fa..059726873 100644 --- a/src/core/matrix/pdu/partial.rs +++ b/src/core/matrix/pdu/partial.rs @@ -5,7 +5,10 @@ events::{MessageLikeEventContent, StateEventContent, TimelineEventType}, }; use serde::Deserialize; -use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; +use serde_json::{ + json, + value::{RawValue as RawJsonValue, to_raw_value}, +}; use super::StateKey; @@ -62,7 +65,7 @@ impl Default for PartialPdu { fn default() -> Self { Self { event_type: "m.room.message".into(), - content: to_raw_value("{}").unwrap(), + content: to_raw_value(&json!({})).unwrap(), unsigned: None, state_key: None, redacts: None, From 62e0b53f5255534b346287bcff2e2f3e21ef2e94 Mon Sep 17 00:00:00 2001 From: timedout Date: Mon, 29 Jun 2026 19:07:28 +0100 Subject: [PATCH 81/81] feat: Prioritise sending dummy events when the extremity count reaches 20 or more --- src/service/rooms/event_handler/handle_incoming_pdu.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 63b7f71d8..f9a497f3c 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -351,8 +351,14 @@ async fn spawn_squasher(&self, room_id: &RoomId, mut rx: mpsc::Receiver<(usize, if let Some((extremities_count, is_dummy_event)) = msg { latest_extremity_count = Some(extremities_count); non_dummy_event = non_dummy_event || !is_dummy_event; + let sleep_duration = if extremities_count >= 20 { + // Skip the original sleep duration and send in the next 3-7 seconds as the number of extremities has grown beyond what one squash can reasonably reduce. We still jitter here in case we receive more events in that time that reduce the number anyway, and to account for other servers sending the same squashes. + jitter(Duration::from_secs(5), -50.0..=50.0) + } else { + jitter(Duration::from_mins(1), -50.0..=50.0) + }; #[allow(clippy::arithmetic_side_effects)] - waker.as_mut().reset(tokio::time::Instant::now() + jitter(Duration::from_mins(1), -50.0..=50.0)); + waker.as_mut().reset(tokio::time::Instant::now() + sleep_duration); } else { {let mut map = service.extremity_squashers.write(); if let Some(tx) = map.get(&room_id) && tx.is_closed() {