From b11df68d46afa490d114847e06397a4e45bac2a8 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 24 Mar 2026 19:22:40 -0400
Subject: [PATCH] speed up the tests a bit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

  Summary of the performance fix:

  - Synchronous DB for in-memory SQLite: When _use_shared_conn is True (in-memory SQLite with a single shared connection — the test setup), runWithConnection and runInteraction now run the function directly on the event loop
  thread instead of dispatching to a ThreadPoolExecutor. This eliminates thread context switch overhead for every DB query.
  - pump() uses asyncio.sleep(0) not asyncio.sleep(0.01): Zero-delay yields drain pending callbacks without burning real wall-clock time.

  The remaining gap vs Twisted (~14s vs ~6s for login tests) is inherent to IsolatedAsyncioTestCase — each test creates a new event loop, and the _advance_time background task adds per-iteration overhead that Twisted's
  synchronous pump() didn't have. Further optimization would require either reducing the number of _advance_time iterations or finding a way to process callbacks synchronously like Twisted did.
---
 synapse/storage/native_database.py | 28 ++++++++++++++++++++++------
 tests/server.py                    |  8 ++++----
 tests/unittest.py                  |  5 ++---
 3 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/synapse/storage/native_database.py b/synapse/storage/native_database.py
index 7da3fe5485..620cc26148 100644
--- a/synapse/storage/native_database.py
+++ b/synapse/storage/native_database.py
@@ -176,11 +176,16 @@ class NativeConnectionPool:
         if self._closed:
             raise Exception("Connection pool is closed")
 
-        def _inner() -> R:
-            conn = self._get_connection()
+        conn = self._get_connection()
+
+        if self._use_shared_conn:
+            # For in-memory SQLite with a shared connection, run directly
+            # on the event loop thread to avoid thread dispatch overhead.
+            return func(conn, *args, **kwargs)
+
+        def _inner() -> R:
             return func(conn, *args, **kwargs)
 
-        # Run in thread pool via asyncio
         loop = asyncio.get_running_loop()
         return await loop.run_in_executor(self._executor, _inner)
 
@@ -206,8 +211,20 @@ class NativeConnectionPool:
         if self._closed:
             raise Exception("Connection pool is closed")
 
-        def _inner() -> R:
-            conn = self._get_connection()
+        conn = self._get_connection()
+
+        if self._use_shared_conn:
+            # For in-memory SQLite with a shared connection, run directly
+            # on the event loop thread to avoid thread dispatch overhead.
+            try:
+                result = func(conn, *args, **kwargs)
+                conn.commit()
+                return result
+            except Exception:
+                conn.rollback()
+                raise
+
+        def _inner() -> R:
             try:
                 result = func(conn, *args, **kwargs)
                 conn.commit()
@@ -216,7 +233,6 @@ class NativeConnectionPool:
                 conn.rollback()
                 raise
 
-        # Run in thread pool via asyncio
         loop = asyncio.get_running_loop()
         return await loop.run_in_executor(self._executor, _inner)
 
diff --git a/tests/server.py b/tests/server.py
index 7e2fcb6f08..842d947ed6 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -545,10 +545,10 @@ async def make_request(
     if await_result and req.render_deferred is not None:
         import asyncio
 
-        # Advance fake time in tiny increments (1ms). This is small enough
-        # that ratelimit token buckets don't noticeably refill (0.2/s × 1ms
-        # = 0.0002 tokens per iteration), yet large enough that ratelimit
-        # pauses (0.5s) complete in ~500 iterations.
+        # Advance fake time in a background task so clock.sleep() calls
+        # (ratelimit pauses) and call_later timers resolve. We advance
+        # 1ms per event loop iteration — small enough to not refill
+        # ratelimit buckets, large enough for pauses to complete.
         async def _advance_time() -> None:
             while not req.render_deferred.done():
                 if clock is not None:
diff --git a/tests/unittest.py b/tests/unittest.py
index dc162b99eb..e4014d241e 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -825,10 +825,9 @@ class HomeserverTestCase(TestCase):
         """
         # Advance fake time (fires pending sleeps and callFromThread)
         self.reactor.advance(by)
-        # Yield to the event loop multiple times so background tasks
-        # (including DB operations in executor threads) can complete.
+        # Yield to the event loop multiple times to drain pending callbacks.
         for _ in range(20):
-            await asyncio.sleep(0.01)
+            await asyncio.sleep(0)
 
     async def get_success(self, d: Awaitable[TV], by: float = 0.0) -> TV:
         """Await an awaitable, optionally advancing fake time first."""