diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs
index 938321887..286cce0b8 100644
--- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs
+++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs
@@ -424,7 +424,12 @@ pub(super) async fn fetch_and_handle_missing_events<'a, Pdu>(
 				}
 				let attempts = seen.get(&*next_id).copied().unwrap_or_default();
 				if attempts >= 5 {
-					debug_error!(elapsed=?start.elapsed(),%attempts, %next_id, "Could not fetch missing event after 5 attempts, giving up");
+					debug_error!(
+						elapsed=?start.elapsed(),
+						%attempts,
+						%next_id,
+						"Could not fetch missing event after 5 attempts, giving up"
+					);
 					continue;
 				}
 
diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs
index 40db2b831..0a46129c9 100644
--- a/src/service/rooms/event_handler/fetch_state.rs
+++ b/src/service/rooms/event_handler/fetch_state.rs
@@ -145,7 +145,7 @@ pub(super) async fn fetch_state(
 							);
 							self.fetch_and_handle_missing_events(
 								origin,
-								to_fetch,
+								res.pdu_ids.clone(),
 								create_event,
 								room_id,
 							)
@@ -161,7 +161,7 @@ pub(super) async fn fetch_state(
 						);
 						self.fetch_and_handle_missing_events(
 							origin,
-							to_fetch,
+							res.pdu_ids.clone(),
 							create_event,
 							room_id,
 						)
@@ -192,6 +192,11 @@ pub(super) async fn fetch_state(
 					})
 					.collect()
 					.await;
+				assert!(
+					!state_events.is_empty(),
+					"Only missing {} events but read-ahead state vec was empty",
+					to_fetch.len()
+				);
 				debug!(
 					elapsed=?start.elapsed(),
 					to_fetch = to_fetch.len(),
@@ -200,13 +205,12 @@ pub(super) async fn fetch_state(
 				let fetched_state = self
 					.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id)
 					.await;
-				assert!(
-					!fetched_state.is_empty(),
-					"fetch_and_handle_missing_events returned empty state map"
-				);
 				state_events.extend(fetched_state);
 			}
 		}
+		if state_events.is_empty() {
+			return Ok(Some(HashMap::new()));
+		}
 
 		let mut state: HashMap<ShortStateKey, OwnedEventId> =
 			HashMap::with_capacity(state_events.len());
diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs
index 182169755..24bc81f6a 100644
--- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs
+++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs
@@ -93,7 +93,11 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu(
 	if state_at_incoming_event.is_empty()
 		&& *incoming_pdu.event_type() != StateEventType::RoomCreate.into()
 	{
-		return Err!(Request(Forbidden("Incoming event has empty incoming state at")));
+		// This can happen if the remote sends an event but cannot be reached to fetch
+		// the state at it, and all other servers in the room (which might just be the
+		// unreachable server) are unable to provide required info.
+		// returning an error here allows the upgrade to be attempted at another time.
+		return Err!(Request(Forbidden("Could not resolve incoming state at event")));
 	}
 	trace!(state_events = state_at_incoming_event.len(), "Calculated incoming state");