diff --git a/src/service/rooms/timeline/append.rs b/src/service/rooms/timeline/append.rs
index 37e1ffa78..f4b53a785 100644
--- a/src/service/rooms/timeline/append.rs
+++ b/src/service/rooms/timeline/append.rs
@@ -207,8 +207,10 @@ async fn progress_state_and_extremities(
 		// status.
 		let mut ancestors_todo: VecDeque<OwnedEventId> =
 			VecDeque::from_iter(incoming_pdu.prev_events.clone());
+		let room_id = incoming_pdu.room_id_or_hash();
 		while let Some(prev_event_id) = ancestors_todo.pop_front() {
 			new_extremities.remove(&prev_event_id);
+			self.services.pdu_metadata.mark_as_referenced(&room_id, once(prev_event_id.as_ref()));
 			let Ok(prev_event) = self.get_pdu(&prev_event_id).await else {
 				continue;
 			};
@@ -233,12 +235,6 @@ async fn progress_state_and_extremities(
 				})
 				.collect::<Vec<_>>()
 				.await;
-			for ancestor_id in &ancestors_todo {
-				self.services.pdu_metadata.mark_as_referenced(
-					&incoming_pdu.room_id_or_hash(),
-					once(ancestor_id.as_ref()),
-				);
-			}
 			ancestors_todo.extend(outlier_prevs);
 		}