diff --git a/changelog.d/19592.misc b/changelog.d/19592.misc
new file mode 100644
index 0000000000..e3fa3a5416
--- /dev/null
+++ b/changelog.d/19592.misc
@@ -0,0 +1 @@
+Ensure old Complement test files are removed when downloading a Complement checkout via `./scripts-dev/complement.sh`.
\ No newline at end of file
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index a8a361fd4a..cca87d42a9 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -160,8 +160,18 @@ main() {
   if [[ -z "$COMPLEMENT_DIR" ]]; then
     COMPLEMENT_REF=${COMPLEMENT_REF:-main}
     echo "COMPLEMENT_DIR not set. Fetching Complement checkout from ${COMPLEMENT_REF}..."
-    wget -Nq https://github.com/matrix-org/complement/archive/${COMPLEMENT_REF}.tar.gz
+    
+    # Download the Complement checkout at the specified ref.
+    wget -q https://github.com/matrix-org/complement/archive/${COMPLEMENT_REF}.tar.gz
+
+    # Delete the existing complement checkout. Otherwise we'll end up with stale
+    # test files after they're deleted server-side, and `tar` will not delete
+    # old files.
+    rm -rf complement-${COMPLEMENT_REF}
+
+    # Extract the checkout.
     tar -xzf ${COMPLEMENT_REF}.tar.gz
+
     COMPLEMENT_DIR=complement-${COMPLEMENT_REF}
     echo "Checkout available at 'complement-${COMPLEMENT_REF}'"
   fi