dandri/synapse
1
0
Fork 0
mirror of https://github.com/element-hq/synapse.git synced 2026-03-31 01:55:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

⏺ Summary of remaining Twisted defer usage:

Browse Source 
- 0 @defer.inlineCallbacks — all converted to async def
  - 12 defer.ensureDeferred — reactor entry points (startup, shutdown, render)
  - 22 defer.Deferred() — in Linearizer, ReadWriteLock, AwakenableSleeper, DeferredEvent (old implementations)
  - 21 defer.gatherResults — in fallback paths and old implementations
  - 11 defer.succeed/fail — immediate value wrapping in old implementations
  - 3 defer.FirstError — in fallback paths
  - 13 defer.TimeoutError — in timeout_deferred and its callers

  The majority (22 + 21 + 11 + 13 = 67) are in the old Deferred-based utility implementations (Linearizer, ReadWriteLock, ObservableDeferred, timeout_deferred, etc.) that already have native replacements (NativeLinearizer,
  NativeReadWriteLock, ObservableFuture, native_timeout, etc.). These will be removed when callers switch to the native versions.

  The 12 defer.ensureDeferred are in reactor entry points that will be removed when reactor.run() → asyncio.run().

  The codebase is now in a clean transitional state where:
  1. All Twisted imports are conditional (try/except ImportError)
  2. ContextVar is the primary logcontext storage
  3. Test base class is stdlib (unittest.TestCase)
  4. CancelledError is asyncio.CancelledError in production code
  5. @defer.inlineCallbacks is eliminated (0 remaining)
  6. yieldable_gather_results uses asyncio.gather (with Twisted fallback)
  7. Module API is fully async (no more Deferred return types)
  8. Twisted is optional in pyproject.toml
This commit is contained in:
Matthew Hodgson
2026-03-21 20:54:28 +00:00
parent a5928e6839
commit 8e1c26067b
25 changed files with 160 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
synapse/_scripts/synapse_port_db.py
View File

@@ -1580,11 +1580,10 @@ def main() -> None:
hs=hs,
)
@defer.inlineCallbacks
def run() -> Generator["defer.Deferred[Any]", Any, None]:
yield defer.ensureDeferred(porter.run())
async def run() -> None:
await porter.run()
hs.get_clock().call_when_running(run)
hs.get_clock().call_when_running(lambda: defer.ensureDeferred(run()))
reactor.run()

5
synapse/events/presence_router.py
View File

@@ -30,10 +30,7 @@ from typing import (
from typing_extensions import ParamSpec
try:
from twisted.internet.defer import CancelledError
except ImportError:
pass
from asyncio import CancelledError
from synapse.api.presence import UserPresenceState
from synapse.util.async_helpers import delay_cancellation, maybe_awaitable

1
synapse/handlers/auth.py
View File

@@ -20,6 +20,7 @@
# [This file includes modifications made by New Vector Limited]
#
#
from asyncio import CancelledError
import logging
import time
import unicodedata

5
synapse/handlers/profile.py
View File

@@ -23,10 +23,7 @@ import random
from bisect import bisect_right
from typing import TYPE_CHECKING
try:
from twisted.internet.defer import CancelledError
except ImportError:
pass
from asyncio import CancelledError
from synapse.api.constants import ProfileFields
from synapse.api.errors import (
AuthError,

16
synapse/http/federation/matrix_federation_agent.py
View File

@@ -173,14 +173,13 @@ class MatrixFederationAgent:
self._well_known_resolver = _well_known_resolver
@defer.inlineCallbacks
def request(
async def request(
self,
method: bytes,
uri: bytes,
headers: Headers | None = None,
bodyProducer: Optional[IBodyProducer] = None,
) -> Generator[defer.Deferred, Any, IResponse]:
) -> IResponse:
"""
Args:
method: HTTP method: GET/POST/etc
@@ -193,10 +192,7 @@ class MatrixFederationAgent:
a file for a file upload). Or None if the request is to have
no body.
Returns:
A deferred which fires when the header of the response has been received
(regardless of the response status code). Fails if there is any problem
which prevents that response from being received (including problems that
prevent the request from being sent).
The response (once headers are received).
"""
# We use urlparse as that will set `port` to None if there is no
# explicit port.
@@ -216,8 +212,8 @@ class MatrixFederationAgent:
and not _is_ip_literal(parsed_uri.hostname)
and not parsed_uri.port
):
well_known_result = yield defer.ensureDeferred(
self._well_known_resolver.get_well_known(parsed_uri.hostname)
well_known_result = await self._well_known_resolver.get_well_known(
parsed_uri.hostname
)
delegated_server = well_known_result.delegated_server
@@ -249,7 +245,7 @@ class MatrixFederationAgent:
if not request_headers.hasHeader(b"user-agent"):
request_headers.addRawHeader(b"user-agent", self.user_agent)
res = yield make_deferred_yieldable(
res = await make_deferred_yieldable(
self._agent.request(method, uri, request_headers, bodyProducer)
)

5
synapse/http/federation/well_known_resolver.py
View File

@@ -43,10 +43,7 @@ from synapse.util.clock import Clock
from synapse.util.duration import Duration
from synapse.util.json import json_decoder
from synapse.util.metrics import Measure
try:
from twisted.internet.defer import CancelledError
except ImportError:
pass
from asyncio import CancelledError
# period to cache .well-known results for by default
WELL_KNOWN_DEFAULT_CACHE_PERIOD = 24 * 3600

18
synapse/http/server.py
View File

@@ -45,15 +45,17 @@ import jinja2
from canonicaljson import encode_canonical_json
from zope.interface import implementer
import asyncio as _asyncio
from asyncio import CancelledError
from twisted.internet import defer, interfaces, reactor
from twisted.internet.defer import CancelledError
from twisted.python import failure
# Tuple of CancelledError types for f.check() during transition
_CancelledErrors = (CancelledError, _asyncio.CancelledError)
from twisted.web import resource
try:
from twisted.internet import defer, interfaces, reactor
from twisted.internet.defer import CancelledError as TwistedCancelledError
from twisted.python import failure
from twisted.web import resource
# Catch both CancelledError types during transition
_CancelledErrors = (CancelledError, TwistedCancelledError)
except ImportError:
_CancelledErrors = (CancelledError,) # type: ignore[assignment]
from synapse.types import ISynapseThreadlessReactor

1
synapse/logging/_remote.py
View File

@@ -19,6 +19,7 @@
#
#
from asyncio import CancelledError
import logging
import sys
import traceback

98
synapse/module_api/__init__.py
View File

@@ -827,7 +827,7 @@ class ModuleApi:
"""
return [attr.asdict(t) for t in await self._store.user_get_threepids(user_id)]
def check_user_exists(self, user_id: str) -> "defer.Deferred[str | None]":
async def check_user_exists(self, user_id: str) -> str | None:
"""Check if user exists.
Added in Synapse v0.25.0.
@@ -839,15 +839,14 @@ class ModuleApi:
Canonical (case-corrected) user_id, or None
if the user is not registered.
"""
return defer.ensureDeferred(self._auth_handler.check_user_exists(user_id))
return await self._auth_handler.check_user_exists(user_id)
@defer.inlineCallbacks
def register(
async def register(
self,
localpart: str,
displayname: str | None = None,
emails: list[str] | None = None,
) -> Generator["defer.Deferred[Any]", Any, tuple[str, str]]:
) -> tuple[str, str]:
"""Registers a new user with given localpart and optional displayname, emails.
Also returns an access token for the new user.
@@ -869,17 +868,17 @@ class ModuleApi:
logger.warning(
"Using deprecated ModuleApi.register which creates a dummy user device."
)
user_id = yield self.register_user(localpart, displayname, emails or [])
_, access_token, _, _ = yield self.register_device(user_id)
user_id = await self.register_user(localpart, displayname, emails or [])
_, access_token, _, _ = await self.register_device(user_id)
return user_id, access_token
def register_user(
async def register_user(
self,
localpart: str,
displayname: str | None = None,
emails: list[str] | None = None,
admin: bool = False,
) -> "defer.Deferred[str]":
) -> str:
"""Registers a new user with given localpart and optional displayname, emails.
Added in Synapse v1.2.0.
@@ -898,21 +897,19 @@ class ModuleApi:
Returns:
user_id
"""
return defer.ensureDeferred(
self._hs.get_registration_handler().register_user(
localpart=localpart,
default_display_name=displayname,
bind_emails=emails or [],
admin=admin,
)
return await self._hs.get_registration_handler().register_user(
localpart=localpart,
default_display_name=displayname,
bind_emails=emails or [],
admin=admin,
)
def register_device(
async def register_device(
self,
user_id: str,
device_id: str | None = None,
initial_display_name: str | None = None,
) -> "defer.Deferred[tuple[str, str, int | None, str | None]]":
) -> tuple[str, str, int | None, str | None]:
"""Register a device for a user and generate an access token.
Added in Synapse v1.2.0.
@@ -927,17 +924,15 @@ class ModuleApi:
Returns:
Tuple of device ID, access token, access token expiration time and refresh token
"""
return defer.ensureDeferred(
self._hs.get_registration_handler().register_device(
user_id=user_id,
device_id=device_id,
initial_display_name=initial_display_name,
)
return await self._hs.get_registration_handler().register_device(
user_id=user_id,
device_id=device_id,
initial_display_name=initial_display_name,
)
def record_user_external_id(
async def record_user_external_id(
self, auth_provider_id: str, remote_user_id: str, registered_user_id: str
) -> defer.Deferred:
) -> None:
"""Record a mapping between an external user id from a single sign-on provider
and a mxid.
@@ -952,10 +947,8 @@ class ModuleApi:
external_id: id on that system
user_id: complete mxid that it is mapped to
"""
return defer.ensureDeferred(
self._store.record_user_external_id(
auth_provider_id, remote_user_id, registered_user_id
)
await self._store.record_user_external_id(
auth_provider_id, remote_user_id, registered_user_id
)
async def create_login_token(
@@ -988,10 +981,9 @@ class ModuleApi:
auth_provider_session_id,
)
@defer.inlineCallbacks
def invalidate_access_token(
async def invalidate_access_token(
self, access_token: str
) -> Generator["defer.Deferred[Any]", Any, None]:
) -> None:
"""Invalidate an access token for a user
Added in Synapse v0.25.0.
@@ -999,37 +991,27 @@ class ModuleApi:
Args:
access_token: access token
Returns:
twisted.internet.defer.Deferred - resolves once the access token
has been removed.
Raises:
synapse.api.errors.AuthError: the access token is invalid
"""
# see if the access token corresponds to a device
user_info = yield defer.ensureDeferred(
self._auth.get_user_by_access_token(access_token)
)
user_info = await self._auth.get_user_by_access_token(access_token)
device_id = user_info.get("device_id")
user_id = user_info["user"].to_string()
if device_id:
# delete the device, which will also delete its access tokens
yield defer.ensureDeferred(
self._device_handler.delete_devices(user_id, [device_id])
)
await self._device_handler.delete_devices(user_id, [device_id])
else:
# no associated device. Just delete the access token.
yield defer.ensureDeferred(
self._auth_handler.delete_access_token(access_token)
)
await self._auth_handler.delete_access_token(access_token)
def run_db_interaction(
async def run_db_interaction(
self,
desc: str,
func: Callable[Concatenate[LoggingTransaction, P], T],
*args: P.args,
**kwargs: P.kwargs,
) -> "defer.Deferred[T]":
) -> T:
"""Run a function with a database connection
Added in Synapse v0.25.0.
@@ -1045,9 +1027,7 @@ class ModuleApi:
Result of func
"""
# type-ignore: See https://github.com/python/mypy/issues/8862
return defer.ensureDeferred(
self._store.db_pool.runInteraction(desc, func, *args, **kwargs) # type: ignore[arg-type]
)
return await self._store.db_pool.runInteraction(desc, func, *args, **kwargs) # type: ignore[arg-type]
def register_cached_function(self, cached_func: CachedFunction) -> None:
"""Register a cached function that should be invalidated across workers.
@@ -1117,15 +1097,11 @@ class ModuleApi:
new_user=new_user,
)
@defer.inlineCallbacks
def get_state_events_in_room(
async def get_state_events_in_room(
self, room_id: str, types: Iterable[tuple[str, str | None]]
) -> Generator[defer.Deferred, Any, Iterable[EventBase]]:
) -> Iterable[EventBase]:
"""Gets current state events for the given room.
(This is exposed for compatibility with the old SpamCheckerApi. We should
probably deprecate it and replace it with an async method in a subclass.)
Added in Synapse v1.22.0.
Args:
@@ -1136,12 +1112,10 @@ class ModuleApi:
Returns:
The filtered state events in the room.
"""
state_ids = yield defer.ensureDeferred(
self._storage_controllers.state.get_current_state_ids(
room_id=room_id, state_filter=StateFilter.from_types(types)
)
state_ids = await self._storage_controllers.state.get_current_state_ids(
room_id=room_id, state_filter=StateFilter.from_types(types)
)
state = yield defer.ensureDeferred(self._store.get_events(state_ids.values()))
state = await self._store.get_events(state_ids.values())
return state.values()
async def update_room_membership(

5
synapse/module_api/callbacks/third_party_event_rules_callbacks.py
View File

@@ -21,10 +21,7 @@
import logging
from typing import TYPE_CHECKING, Any, Awaitable, Callable
try:
from twisted.internet.defer import CancelledError
except ImportError:
pass
from asyncio import CancelledError
from synapse.api.errors import ModuleFailedException, SynapseError
from synapse.events import EventBase
from synapse.events.snapshot import UnpersistedEventContextBase

5
synapse/notifier.py
View File

@@ -67,10 +67,7 @@ from synapse.util.async_helpers import (
from synapse.util.duration import Duration
from synapse.util.stringutils import shortstr
from synapse.visibility import filter_and_transform_events_for_client
try:
from twisted.internet.defer import CancelledError
except ImportError:
pass
from asyncio import CancelledError
if TYPE_CHECKING:
from synapse.server import HomeServer

1
synapse/types/__init__.py
View File

@@ -21,6 +21,7 @@
#
import abc
import asyncio
from asyncio import CancelledError
import logging
import re
import string

126
synapse/util/async_helpers.py
View File

@@ -24,6 +24,7 @@ import asyncio
import collections
import inspect
import itertools
from asyncio import CancelledError
import logging
from collections import OrderedDict
from contextlib import asynccontextmanager
@@ -47,12 +48,19 @@ from typing import (
import attr
from typing_extensions import Concatenate, ParamSpec, Unpack
from asyncio import CancelledError
try:
from twisted.internet import defer
from twisted.internet.defer import CancelledError
from twisted.internet.defer import CancelledError as TwistedCancelledError
from twisted.python.failure import Failure
# Tuple for catching both CancelledError types during transition
AnyCancelledError = (CancelledError, TwistedCancelledError)
except ImportError:
pass
defer = None # type: ignore[assignment]
Failure = BaseException # type: ignore[misc,assignment]
AnyCancelledError = (CancelledError,) # type: ignore[assignment]
from synapse.logging.context import (
PreserveLoggingContext,
@@ -317,7 +325,7 @@ async def yieldable_gather_results(
"""Executes the function with each argument concurrently.
Args:
func: Function to execute that returns a Deferred
func: Function to execute that returns an awaitable
iter: An iterable that yields items that get passed as the first
argument to the function
*args: Arguments to be passed to each call to func
@@ -326,28 +334,31 @@ async def yieldable_gather_results(
Returns
A list containing the results of the function
"""
async def _run(item: T) -> R:
return await func(item, *args, **kwargs)
try:
return await make_deferred_yieldable(
defer.gatherResults(
[run_in_background(func, item, *args, **kwargs) for item in iter],
consumeErrors=True,
)
asyncio.get_running_loop()
results = await asyncio.gather(
*[_run(item) for item in iter],
return_exceptions=True,
)
except defer.FirstError as dfe:
# unwrap the error from defer.gatherResults.
# The raised exception's traceback only includes func() etc if
# the 'await' happens before the exception is thrown - ie if the failure
# happens *asynchronously* - otherwise Twisted throws away the traceback as it
# could be large.
#
# We could maybe reconstruct a fake traceback from Failure.frames. Or maybe
# we could throw Twisted into the fires of Mordor.
# suppress exception chaining, because the FirstError doesn't tell us anything
# very interesting.
assert isinstance(dfe.subFailure.value, BaseException)
raise dfe.subFailure.value from None
for r in results:
if isinstance(r, BaseException):
raise r
return results # type: ignore[return-value]
except RuntimeError:
# No asyncio loop — use Twisted fallback
try:
return await make_deferred_yieldable(
defer.gatherResults(
[run_in_background(func, item, *args, **kwargs) for item in iter],
consumeErrors=True,
)
)
except defer.FirstError as dfe:
assert isinstance(dfe.subFailure.value, BaseException)
raise dfe.subFailure.value from None
async def yieldable_gather_results_delaying_cancellation(
@@ -362,7 +373,7 @@ async def yieldable_gather_results_delaying_cancellation(
See `yieldable_gather_results`.
Args:
func: Function to execute that returns a Deferred
func: Function to execute that returns an awaitable
iter: An iterable that yields items that get passed as the first
argument to the function
*args: Arguments to be passed to each call to func
@@ -371,18 +382,22 @@ async def yieldable_gather_results_delaying_cancellation(
Returns
A list containing the results of the function
"""
try:
return await make_deferred_yieldable(
delay_cancellation(
defer.gatherResults(
[run_in_background(func, item, *args, **kwargs) for item in iter],
consumeErrors=True,
)
)
# Use asyncio.shield to delay cancellation
async def _run(item: T) -> R:
return await func(item, *args, **kwargs)
results = await asyncio.shield(
asyncio.gather(
*[_run(item) for item in iter],
return_exceptions=True,
)
except defer.FirstError as dfe:
assert isinstance(dfe.subFailure.value, BaseException)
raise dfe.subFailure.value from None
)
for r in results:
if isinstance(r, BaseException):
raise r
return results # type: ignore[return-value]
T1 = TypeVar("T1")
@@ -536,7 +551,21 @@ async def gather_optional_coroutines(
overload above.
"""
# Use asyncio.gather if an event loop is running, otherwise fall back to
# Twisted's defer.gatherResults during transition
try:
asyncio.get_running_loop()
tasks = [
asyncio.ensure_future(coroutine)
for coroutine in coroutines
if coroutine is not None
]
results = await asyncio.gather(*tasks, return_exceptions=True)
for r in results:
if isinstance(r, BaseException):
raise r
except RuntimeError:
# No asyncio loop — use Twisted
results = await make_deferred_yieldable(
defer.gatherResults(
[
@@ -548,26 +577,11 @@ async def gather_optional_coroutines(
)
)
results_iter = iter(results)
return tuple(
next(results_iter) if coroutine is not None else None
for coroutine in coroutines
)
except defer.FirstError as dfe:
# unwrap the error from defer.gatherResults.
# The raised exception's traceback only includes func() etc if
# the 'await' happens before the exception is thrown - ie if the failure
# happens *asynchronously* - otherwise Twisted throws away the traceback as it
# could be large.
#
# We could maybe reconstruct a fake traceback from Failure.frames. Or maybe
# we could throw Twisted into the fires of Mordor.
# suppress exception chaining, because the FirstError doesn't tell us anything
# very interesting.
assert isinstance(dfe.subFailure.value, BaseException)
raise dfe.subFailure.value from None
results_iter = iter(results)
return tuple(
next(results_iter) if coroutine is not None else None
for coroutine in coroutines
)
@attr.s(slots=True, auto_attribs=True)
@@ -692,7 +706,7 @@ class Linearizer:
# exit path, but that would slow down the uncontended case.
try:
await self._clock.sleep(Duration(seconds=0))
except CancelledError:
except AnyCancelledError:
self._release_lock(key, entry)
raise

5
synapse/util/background_queue.py
View File

@@ -31,10 +31,7 @@ except ImportError:
from synapse.util.async_helpers import DeferredEvent
from synapse.util.duration import Duration
try:
from twisted.internet.defer import CancelledError
except ImportError:
pass
from asyncio import CancelledError
if TYPE_CHECKING:
from synapse.server import HomeServer

5
synapse/util/caches/response_cache.py
View File

@@ -51,10 +51,7 @@ from synapse.util.cancellation import cancellable, is_function_cancellable
from synapse.util.clock import Clock
from synapse.util.duration import Duration
from synapse.util.wheel_timer import WheelTimer
try:
from twisted.internet.defer import CancelledError
except ImportError:
pass
from asyncio import CancelledError
logger = logging.getLogger(__name__)

5
synapse/util/task_scheduler.py
View File

@@ -41,10 +41,7 @@ from synapse.metrics.background_process_metrics import (
from synapse.types import JsonMapping, ScheduledTask, TaskStatus
from synapse.util.duration import Duration
from synapse.util.stringutils import random_string
try:
from twisted.internet.defer import CancelledError
except ImportError:
pass
from asyncio import CancelledError
if TYPE_CHECKING:
from synapse.server import HomeServer

2
tests/storage/databases/main/test_events_worker.py
View File

@@ -25,7 +25,7 @@ from unittest import mock
try:
from twisted.enterprise.adbapi import ConnectionPool
from twisted.internet.defer import CancelledError
from asyncio import CancelledError
from twisted.internet.defer import Deferred, ensureDeferred
except ImportError:
pass

2
tests/storage/test_database.py
View File

@@ -26,7 +26,7 @@ import attr
try:
from twisted.internet import defer
from twisted.internet.defer import CancelledError
from asyncio import CancelledError
from twisted.internet.defer import Deferred
except ImportError:
pass

2
tests/storage/util/test_partial_state_events_tracker.py
View File

@@ -23,7 +23,7 @@ from typing import Collection
from unittest import mock
try:
from twisted.internet.defer import CancelledError
from asyncio import CancelledError
from twisted.internet.defer import ensureDeferred
except ImportError:
pass

20
tests/unittest.py
View File

@@ -393,10 +393,22 @@ class TestCase(_stdlib_unittest.TestCase):
result = results[0]
if not isinstance(result, Failure):
self.fail(f"Deferred {d!r} succeeded with {result!r}, expected failure")
if expected_types and not result.check(*expected_types):
self.fail(
f"Expected {expected_types}, got {result.type}: {result.value}"
)
if expected_types:
# During transition, check both asyncio and Twisted CancelledError
expanded_types = list(expected_types)
from asyncio import CancelledError as AsyncCE
try:
from twisted.internet.defer import CancelledError as TwistedCE
if AsyncCE in expanded_types and TwistedCE not in expanded_types:
expanded_types.append(TwistedCE)
elif TwistedCE in expanded_types and AsyncCE not in expanded_types:
expanded_types.append(AsyncCE)
except ImportError:
pass
if not result.check(*expanded_types):
self.fail(
f"Expected {expected_types}, got {result.type}: {result.value}"
)
return result
def assertObjectHasAttributes(self, attrs: dict[str, object], obj: object) -> None:

2
tests/util/caches/test_descriptors.py
View File

@@ -31,7 +31,7 @@ from unittest import mock
try:
from twisted.internet import defer, reactor
from twisted.internet.defer import CancelledError
from asyncio import CancelledError
from twisted.internet.defer import Deferred
from twisted.internet.interfaces import IReactorTime
except ImportError:

2
tests/util/caches/test_response_cache.py
View File

@@ -36,7 +36,7 @@ from synapse.util.duration import Duration
from tests.server import get_clock
from tests.unittest import TestCase
try:
from twisted.internet.defer import CancelledError
from asyncio import CancelledError
except ImportError:
pass

2
tests/util/test_async_helpers.py
View File

@@ -25,7 +25,7 @@ from parameterized import parameterized_class
try:
from twisted.internet import defer
from twisted.internet.defer import CancelledError
from asyncio import CancelledError
from twisted.internet.defer import Deferred, ensureDeferred
from twisted.python.failure import Failure
except ImportError:

2
tests/util/test_linearizer.py
View File

@@ -23,7 +23,7 @@ from typing import Hashable, Protocol
try:
from twisted.internet import defer
from twisted.internet.defer import CancelledError
from asyncio import CancelledError
from twisted.internet.defer import Deferred
except ImportError:
pass

2
tests/util/test_rwlock.py
View File

@@ -23,7 +23,7 @@ from typing import AsyncContextManager, Callable, Sequence
try:
from twisted.internet import defer
from twisted.internet.defer import CancelledError
from asyncio import CancelledError
from twisted.internet.defer import Deferred
except ImportError:
pass