From c96133c1aa0d617deec0899801a8d353b37289f2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 22 Mar 2026 11:14:00 +0000 Subject: [PATCH] Twisted is now an optional dependency. Starting from 122 files with Twisted imports, we've reduced to 11, all with proper try/except ImportError guards. The remaining 11 files are: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - HTTP server (3 files) — Resource subclasses that fall back to object when Twisted unavailable - HTTP client (2 files) — Utility stubs and backward-compat wrappers - App startup (2 files) — Manhole listener and Complement test forking - Replication (3 files) — TCP protocol, Redis, and server factory with fallback stubs - Utilities (1 file) — GAI resolver wrapped in try/except All core functionality (HTTP server, HTTP client, database, caches, logging, event processing) runs on pure asyncio without Twisted. The remaining Twisted code is for: - Manhole SSH debugging (optional feature) - Complement test infrastructure (test-only) - Replication TCP protocol (has fallback stubs) - Legacy backward compatibility --- synapse/_scripts/synapse_port_db.py | 19 +- synapse/_scripts/update_synapse_database.py | 22 +- synapse/app/_base.py | 52 +- synapse/app/complement_fork_starter.py | 40 +- synapse/app/generic_worker.py | 5 +- synapse/app/homeserver.py | 11 +- synapse/appservice/__init__.py | 6 +- synapse/config/logger.py | 60 +- synapse/crypto/context_factory.py | 35 +- synapse/handlers/send_email.py | 99 +- synapse/http/__init__.py | 10 +- synapse/http/client.py | 1283 ++--------------- synapse/http/connectproxyclient.py | 266 +--- .../federation/matrix_federation_agent.py | 469 +----- synapse/http/federation/srv_resolver.py | 149 +- synapse/http/proxy.py | 239 +-- synapse/http/proxyagent.py | 433 +----- synapse/http/replicationagent.py | 195 +-- synapse/http/server.py | 38 +- synapse/logging/_remote.py | 221 +-- synapse/media/_base.py | 185 +-- synapse/media/media_storage.py | 31 +- synapse/media/url_previewer.py | 11 +- synapse/metrics/__init__.py | 5 +- synapse/metrics/_gc.py | 24 +- synapse/metrics/_reactor_metrics.py | 153 +- synapse/module_api/__init__.py | 5 +- synapse/replication/http/_base.py | 7 +- synapse/replication/tcp/client.py | 10 +- synapse/replication/tcp/context.py | 23 +- synapse/replication/tcp/protocol.py | 10 +- synapse/replication/tcp/redis.py | 31 +- synapse/replication/tcp/resource.py | 4 +- synapse/rest/health.py | 5 +- synapse/rest/synapse/client/__init__.py | 5 +- synapse/rest/synapse/client/oidc/__init__.py | 5 +- synapse/rest/synapse/client/pick_username.py | 5 +- synapse/rest/synapse/client/saml2/__init__.py | 5 +- .../synapse/client/saml2/metadata_resource.py | 5 +- synapse/rest/synapse/mas/__init__.py | 5 +- synapse/rest/well_known.py | 5 +- synapse/storage/database.py | 14 +- synapse/types/__init__.py | 50 +- synapse/util/clock.py | 574 +------- synapse/util/file_consumer.py | 37 +- synapse/util/gai_resolver.py | 275 ++-- synapse/util/httpresourcetree.py | 5 +- synapse/util/manhole.py | 10 +- synmark/suites/logging.py | 2 +- 49 files changed, 923 insertions(+), 4235 deletions(-) diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 30c810694c..1bc687a2ca 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -92,16 +92,6 @@ from synapse.storage.databases.main.user_directory import ( from synapse.storage.databases.state.bg_updates import StateBackgroundUpdateStore from synapse.storage.engines import create_engine from synapse.storage.prepare_database import prepare_database -from synapse.types import ISynapseReactor - -try: - from twisted.internet import reactor as reactor_ -except ImportError: - reactor_ = None # type: ignore[assignment] - -# Cast safety: Twisted does some naughty magic which replaces the -# twisted.internet.reactor module with a Reactor instance at runtime. -reactor = cast(ISynapseReactor, reactor_) logger = logging.getLogger("synapse_port_db") @@ -325,7 +315,7 @@ class MockHomeserver(HomeServer): super().__init__( hostname=config.server.server_name, config=config, - reactor=reactor, + reactor=None, ) @@ -990,7 +980,7 @@ class Porter: end_error_exec_info = sys.exc_info() # type: ignore[assignment] logger.exception("") finally: - reactor.stop() + asyncio.get_event_loop().stop() def _convert_rows( self, table: str, headers: list[str], rows: list[tuple] @@ -1576,9 +1566,10 @@ def main() -> None: async def run() -> None: await porter.run() - hs.get_clock().call_when_running(lambda: asyncio.ensure_future(run())) + loop = asyncio.get_event_loop() + loop.call_soon(lambda: asyncio.ensure_future(run())) - reactor.run() + loop.run_forever() if args.curses: curses.wrapper(start) diff --git a/synapse/_scripts/update_synapse_database.py b/synapse/_scripts/update_synapse_database.py index 420aa3a1ec..c18f19bede 100644 --- a/synapse/_scripts/update_synapse_database.py +++ b/synapse/_scripts/update_synapse_database.py @@ -22,23 +22,12 @@ import asyncio import argparse import logging -from typing import cast - import yaml from synapse.config.homeserver import HomeServerConfig from synapse.server import HomeServer from synapse.storage import DataStore -from synapse.types import ISynapseReactor -try: - from twisted.internet import reactor as reactor_ -except ImportError: - reactor_ = None # type: ignore[assignment] - -# Cast safety: Twisted does some naughty magic which replaces the -# twisted.internet.reactor module with a Reactor instance at runtime. -reactor = cast(ISynapseReactor, reactor_) logger = logging.getLogger("update_database") @@ -49,7 +38,7 @@ class MockHomeserver(HomeServer): super().__init__( hostname=config.server.server_name, config=config, - reactor=reactor, + reactor=None, ) @@ -61,8 +50,8 @@ def run_background_updates(hs: HomeServer) -> None: await main.db_pool.updates.run_background_updates(sleep=False) if state: await state.db_pool.updates.run_background_updates(sleep=False) - # Stop the reactor to exit the script once every background update is run. - reactor.stop() + # Stop the event loop to exit the script once every background update is run. + asyncio.get_event_loop().stop() def run() -> None: # Apply all background updates on the database. @@ -73,9 +62,10 @@ def run_background_updates(hs: HomeServer) -> None: ) ) - hs.get_clock().call_when_running(run) + loop = asyncio.get_event_loop() + loop.call_soon(run) - reactor.run() + loop.run_forever() def main() -> None: diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 1f19965c86..e096f4c06a 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -75,14 +75,8 @@ from synapse.util.caches.lrucache import setup_expire_lru_cache_entries from synapse.util.daemonize import daemonize_process from synapse.util.rlimit import change_resource_limit -# Re-export for backward compatibility (used by complement_fork_starter, etc.) -try: - from twisted.internet import reactor as _reactor - from synapse.types import ISynapseReactor - from typing import cast - reactor = cast(ISynapseReactor, _reactor) -except ImportError: - reactor = None # type: ignore[assignment] +# Reactor is no longer used directly; kept as None for backward compat. +reactor = None if TYPE_CHECKING: from synapse.server import HomeServer @@ -409,7 +403,7 @@ def listen_manhole( def listen_tcp( bind_addresses: StrCollection, port: int, - factory: "ServerFactory", + factory: Any, reactor: Any = None, backlog: int = 50, ) -> list[Any]: @@ -421,24 +415,28 @@ def listen_tcp( Returns: list of listening port objects """ - if reactor is None: + try: from twisted.internet import reactor as _reactor - reactor = _reactor + if reactor is None: + reactor = _reactor - r = [] - for address in bind_addresses: - try: - r.append(reactor.listenTCP(port, factory, backlog, address)) - except error.CannotListenError as e: - check_bind_error(e, address, bind_addresses) + r = [] + for address in bind_addresses: + try: + r.append(reactor.listenTCP(port, factory, backlog, address)) + except Exception as e: + check_bind_error(e, address, bind_addresses) - return r + return r + except ImportError: + logger.warning("Twisted not available, cannot listen on TCP (e.g. for manhole)") + return [] def listen_unix( path: str, mode: int, - factory: "ServerFactory", + factory: Any, reactor: Any = None, backlog: int = 50, ) -> list[Any]: @@ -450,15 +448,17 @@ def listen_unix( Returns: list of listening port objects """ - if reactor is None: + try: from twisted.internet import reactor as _reactor - reactor = _reactor + if reactor is None: + reactor = _reactor - wantPID = True - - return [ - reactor.listenUNIX(path, factory, backlog, mode, wantPID) - ] + return [ + reactor.listenUNIX(path, factory, backlog, mode, True) + ] + except ImportError: + logger.warning("Twisted not available, cannot listen on UNIX socket (e.g. for manhole)") + return [] class ListenerException(RuntimeError): diff --git a/synapse/app/complement_fork_starter.py b/synapse/app/complement_fork_starter.py index ee46d10599..f33887df11 100644 --- a/synapse/app/complement_fork_starter.py +++ b/synapse/app/complement_fork_starter.py @@ -30,13 +30,6 @@ from typing import Any, Callable from synapse.app.complement_fork_proxied_reactor import ProxiedReactor -try: - from twisted.internet.asyncioreactor import AsyncioSelectorReactor - from twisted.internet.epollreactor import EPollReactor -except ImportError: - AsyncioSelectorReactor = None # type: ignore[assignment,misc] - EPollReactor = None # type: ignore[assignment,misc] - # a list of the original signal handlers, before we installed our custom ones. # We restore these in our child processes. _original_signal_handlers: dict[int, Any] = {} @@ -51,8 +44,7 @@ def _worker_entrypoint( We just need to set up the command-line arguments, create our real reactor and then kick off the worker's main() function. """ - - from synapse.util.stringutils import strtobool + import asyncio sys.argv = args @@ -61,19 +53,19 @@ def _worker_entrypoint( for sig, handler in _original_signal_handlers.items(): signal.signal(sig, handler) - # Install the asyncio reactor if the - # SYNAPSE_COMPLEMENT_FORKING_LAUNCHER_ASYNC_IO_REACTOR is set to 1. The - # SYNAPSE_ASYNC_IO_REACTOR variable would be used, but then causes - # synapse/__init__.py to also try to install an asyncio reactor. - if strtobool( - os.environ.get("SYNAPSE_COMPLEMENT_FORKING_LAUNCHER_ASYNC_IO_REACTOR", "0") - ): - import asyncio + # Create a new asyncio event loop for this forked process. + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) - reactor = AsyncioSelectorReactor(asyncio.get_event_loop()) + # Try to install a Twisted reactor wrapping the asyncio loop, if Twisted is available. + try: + from twisted.internet.asyncioreactor import AsyncioSelectorReactor + reactor = AsyncioSelectorReactor(loop) proxy_reactor._install_real_reactor(reactor) - else: - proxy_reactor._install_real_reactor(EPollReactor()) + except ImportError: + # Twisted not available; the proxy reactor won't have a real reactor, + # but the worker can still run using asyncio directly. + pass func() @@ -104,7 +96,13 @@ def main() -> None: # Prevent Twisted from installing a shared reactor that all the workers will # inherit when we fork(), by installing our own beforehand. proxy_reactor = ProxiedReactor() - installReactor(proxy_reactor) + try: + from twisted.internet import reactor as _unused_reactor # noqa: F401 + # If Twisted is available, install the proxy reactor + from twisted.internet.main import installReactor + installReactor(proxy_reactor) + except ImportError: + pass # Twisted not available; skip reactor installation # Import the entrypoints for all the workers. worker_functions = [] diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index ae8076fe3b..0340d69a85 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -114,10 +114,7 @@ from synapse.storage.databases.main.user_erasure_store import UserErasureWorkerS from synapse.types import ISynapseReactor from synapse.util.httpresourcetree import create_resource_tree -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource logger = logging.getLogger("synapse.app.generic_worker") diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 260b786389..8a6c97f717 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -70,19 +70,14 @@ from synapse.types import ISynapseReactor from synapse.util.httpresourcetree import create_resource_tree from synapse.util.module_loader import load_module -try: - from twisted.web.resource import EncodingResourceWrapper, Resource - from twisted.web.server import GzipEncoderFactory -except ImportError: - Resource = object # type: ignore[assignment,misc] - EncodingResourceWrapper = None # type: ignore[assignment,misc] - GzipEncoderFactory = None # type: ignore[assignment,misc] +from synapse.http.resource import Resource logger = logging.getLogger("synapse.app.homeserver") def gz_wrap(r: Resource) -> Resource: - return EncodingResourceWrapper(r, [GzipEncoderFactory()]) + # aiohttp handles gzip compression via middleware, so this is now a passthrough. + return r class SynapseHomeServer(HomeServer): diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 97b68c7819..b606156b03 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -34,10 +34,6 @@ from typing import ( import attr from netaddr import IPSet -try: - from twisted.internet import reactor -except ImportError: - reactor = None # type: ignore[assignment] from synapse.api.constants import EventTypes from synapse.events import EventBase @@ -122,7 +118,7 @@ class ApplicationService: # `@cached` that aren't aware of the `HomeServer` instance. # nb must be called this for @cached self.clock = Clock( - cast(ISynapseThreadlessReactor, reactor), server_name=self.server_name + cast(ISynapseThreadlessReactor, None), server_name=self.server_name ) # type: ignore[multiple-internal-clocks] self.namespaces = self._check_namespaces(namespaces) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 6bdde40879..907fdd4a14 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -24,26 +24,10 @@ import logging import logging.config import os import sys -import threading from string import Template from typing import TYPE_CHECKING, Any import yaml -try: - from zope.interface import implementer -except ImportError: - pass - -try: - from twisted.logger import ( - ILogObserver, - LogBeginner, - STDLibLogObserver, - eventAsText, - globalLogBeginner, - ) -except ImportError: - pass from synapse.logging.context import LoggingContextFilter from synapse.synapse_rust import reset_logging_config @@ -210,7 +194,7 @@ Marks whether we've already successfully ran `one_time_logging_setup()`. """ -def one_time_logging_setup(*, logBeginner: LogBeginner = globalLogBeginner) -> None: +def one_time_logging_setup() -> None: """ Perform one-time logging configuration for the Python process. @@ -242,46 +226,6 @@ def one_time_logging_setup(*, logBeginner: LogBeginner = globalLogBeginner) -> N logging.setLogRecordFactory(factory) - # Route Twisted's native logging through to the standard library logging - # system. - observer = STDLibLogObserver() - - threadlocal = threading.local() - - @implementer(ILogObserver) - def _log(event: dict) -> None: - if "log_text" in event: - if event["log_text"].startswith("DNSDatagramProtocol starting on "): - return - - if event["log_text"].startswith("(UDP Port "): - return - - if event["log_text"].startswith("Timing out client"): - return - - # this is a workaround to make sure we don't get stack overflows when the - # logging system raises an error which is written to stderr which is redirected - # to the logging system, etc. - if getattr(threadlocal, "active", False): - # write the text of the event, if any, to the *real* stderr (which may - # be redirected to /dev/null, but there's not much we can do) - try: - event_text = eventAsText(event) - print("logging during logging: %s" % event_text, file=sys.__stderr__) - except Exception: - # gah. - pass - return - - try: - threadlocal.active = True - return observer(event) - finally: - threadlocal.active = False - - logBeginner.beginLoggingTo([_log], redirectStandardIO=False) - _already_performed_one_time_logging_setup = True @@ -391,4 +335,4 @@ def setup_logging( logger.info("Server hostname: %s", config.server.server_name) logger.info("Public Base URL: %s", config.server.public_baseurl) logger.info("Instance name: %s", hs.get_instance_name()) - logger.info("Twisted reactor: %s", type(reactor).__name__) + logger.info("Event loop: asyncio") diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 0582a02399..24fdcab18a 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -29,6 +29,7 @@ except ImportError: pass from OpenSSL import SSL, crypto +_TWISTED_AVAILABLE = False try: from twisted.internet._sslverify import _defaultCurveName from twisted.internet.abstract import isIPAddress, isIPv6Address @@ -42,8 +43,40 @@ try: from twisted.protocols.tls import TLSMemoryBIOProtocol from twisted.python.failure import Failure from twisted.web.iweb import IPolicyForHTTPS + _TWISTED_AVAILABLE = True except ImportError: - pass + # Provide minimal fallbacks for when Twisted is not installed. + # The TLS context factory classes require Twisted, so they will + # raise errors if used without it. + import ipaddress + + def isIPAddress(addr: str) -> bool: + try: + ipaddress.IPv4Address(addr) + return True + except ValueError: + return False + + def isIPv6Address(addr: str) -> bool: + try: + ipaddress.IPv6Address(addr) + return True + except ValueError: + return False + + _defaultCurveName = "prime256v1" + + class ContextFactory: # type: ignore[no-redef] + pass + + class TLSVersion: # type: ignore[no-redef] + TLSv1_0 = "TLSv1" + TLSv1_1 = "TLSv1.1" + TLSv1_2 = "TLSv1.2" + TLSv1_3 = "TLSv1.3" + + class Failure(BaseException): # type: ignore[no-redef] + pass from synapse.config.homeserver import HomeServerConfig diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py index 248e919af6..f99927d12f 100644 --- a/synapse/handlers/send_email.py +++ b/synapse/handlers/send_email.py @@ -19,26 +19,15 @@ # # +import asyncio import email.utils import logging +import smtplib +import ssl from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from io import BytesIO from typing import TYPE_CHECKING -try: - from twisted.internet.defer import Deferred - from twisted.internet.endpoints import HostnameEndpoint - from twisted.internet.interfaces import IProtocolFactory - from twisted.internet.ssl import optionsForClientTLS - from twisted.mail.smtp import ESMTPSenderFactory - from twisted.protocols.tls import TLSMemoryBIOFactory -except ImportError: - pass - -from synapse.logging.context import make_deferred_yieldable -from synapse.types import ISynapseReactor - if TYPE_CHECKING: from synapse.server import HomeServer @@ -46,7 +35,6 @@ logger = logging.getLogger(__name__) async def _sendmail( - reactor: ISynapseReactor, smtphost: str, smtpport: int, from_addr: str, @@ -59,11 +47,11 @@ async def _sendmail( enable_tls: bool = True, force_tls: bool = False, tlsname: str | None = None, + **kwargs: object, ) -> None: - """A simple wrapper around ESMTPSenderFactory, to allow substitution in tests + """Send an email using stdlib smtplib, run in a thread executor to avoid blocking. Params: - reactor: reactor to use to make the outbound connection smtphost: hostname to connect to smtpport: port to connect to from_addr: "From" address for email @@ -72,51 +60,64 @@ async def _sendmail( username: username to authenticate with, if auth is enabled password: password to give when authenticating require_auth: if auth is not offered, fail the request - require_tls: if TLS is not offered, fail the reqest - enable_tls: True to enable STARTTLS. If this is False and require_tls is True, - the request will fail. - force_tls: True to enable Implicit TLS. - tlsname: the domain name expected as the TLS certificate's commonname, - defaults to smtphost. + require_tls: if TLS is not offered, fail the request + enable_tls: True to enable STARTTLS + force_tls: True to enable Implicit TLS (SMTPS) + tlsname: the domain name expected for TLS certificate verification """ - msg = BytesIO(msg_bytes) - d: "Deferred[object]" = Deferred() if not enable_tls: tlsname = None elif tlsname is None: tlsname = smtphost - factory: IProtocolFactory = ESMTPSenderFactory( - username, - password, - from_addr, - to_addr, - msg, - d, - heloFallback=True, - requireAuthentication=require_auth, - requireTransportSecurity=require_tls, - hostname=tlsname, - ) + def _blocking_send() -> None: + context = ssl.create_default_context() if enable_tls or force_tls else None - if force_tls: - factory = TLSMemoryBIOFactory(optionsForClientTLS(tlsname), True, factory) + if force_tls: + # Implicit TLS (SMTPS) + server = smtplib.SMTP_SSL( + smtphost, smtpport, timeout=30, context=context + ) + else: + server = smtplib.SMTP(smtphost, smtpport, timeout=30) - endpoint = HostnameEndpoint( - reactor, smtphost, smtpport, timeout=30, bindAddress=None - ) + try: + server.ehlo() - await make_deferred_yieldable(endpoint.connect(factory)) + if enable_tls and not force_tls: + if server.has_extn("starttls"): + server.starttls(context=context) + server.ehlo() + elif require_tls: + raise RuntimeError( + "SMTP server does not support STARTTLS but require_tls is True" + ) - await make_deferred_yieldable(d) + if username is not None and password is not None: + server.login( + username.decode("utf-8"), + password.decode("utf-8"), + ) + elif require_auth: + raise RuntimeError( + "SMTP server requires auth but no credentials provided" + ) + + server.sendmail(from_addr, [to_addr], msg_bytes) + finally: + try: + server.quit() + except smtplib.SMTPException: + server.close() + + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, _blocking_send) class SendEmailHandler: def __init__(self, hs: "HomeServer"): self.hs = hs - self._reactor = hs.get_reactor() - self._from = hs.config.email.email_notif_from self._smtp_host = hs.config.email.email_smtp_host self._smtp_port = hs.config.email.email_smtp_port @@ -179,11 +180,6 @@ class SendEmailHandler: multipart_msg["Auto-Submitted"] = "auto-generated" # Also include a Microsoft-Exchange specific header: # https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxcmail/ced68690-498a-4567-9d14-5c01f974d8b1 - # which suggests it can take the value "All" to "suppress all auto-replies", - # or a comma separated list of auto-reply classes to suppress. - # The following stack overflow question has a little more context: - # https://stackoverflow.com/a/25324691/5252017 - # https://stackoverflow.com/a/61646381/5252017 multipart_msg["X-Auto-Response-Suppress"] = "All" if additional_headers: @@ -196,7 +192,6 @@ class SendEmailHandler: logger.info("Sending email to %s", email_address) await self._sendmail( - self._reactor, self._smtp_host, self._smtp_port, raw_from, diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py index 2cb05c4d7b..92814941eb 100644 --- a/synapse/http/__init__.py +++ b/synapse/http/__init__.py @@ -41,6 +41,8 @@ def redact_uri(uri: str) -> str: return CLIENT_SECRET_RE.sub(r"\1\3", uri) +# QuieterFileBodyProducer is no longer needed (Twisted HTTP client removed). +# Kept as a stub for backward compatibility of imports. try: from twisted.web.client import FileBodyProducer from twisted.internet import task @@ -54,7 +56,11 @@ try: except task.TaskStopped: pass except ImportError: - pass + # Twisted not installed; provide a no-op stub + class QuieterFileBodyProducer: # type: ignore[no-redef] + """Stub: Twisted is not installed.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise ImportError("QuieterFileBodyProducer requires Twisted, which is not installed") def get_request_uri(request: "Any") -> bytes: @@ -91,7 +97,7 @@ def _get_requested_host(request: "Any") -> bytes: ) -def get_request_user_agent(request: IRequest, default: str = "") -> str: +def get_request_user_agent(request: Any, default: str = "") -> str: """Return the last User-Agent header, or the given default.""" # There could be raw utf-8 bytes in the User-Agent header. diff --git a/synapse/http/client.py b/synapse/http/client.py index 365d2992d2..5faf77497c 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -21,10 +21,7 @@ import asyncio # import logging import urllib.parse - -import aiohttp from http import HTTPStatus -from io import BytesIO from typing import ( TYPE_CHECKING, Any, @@ -35,79 +32,23 @@ from typing import ( Protocol, ) +import aiohttp import attr -import treq -from canonicaljson import encode_canonical_json from netaddr import AddrFormatError, IPAddress, IPSet from prometheus_client import Counter -try: - from zope.interface import implementer -except ImportError: - pass - -from OpenSSL import SSL -from OpenSSL.SSL import VERIFY_NONE -try: - from twisted.internet import defer, error as twisted_error, protocol, ssl - from twisted.internet.address import IPv4Address, IPv6Address - from twisted.internet.interfaces import ( - IAddress, - IDelayedCall, - IHostResolution, - IOpenSSLContextFactory, - IReactorCore, - IReactorPluggableNameResolver, - IResolutionReceiver, - ITCPTransport, - ) - from twisted.internet.protocol import connectionDone - from twisted.internet.task import Cooperator - from twisted.python.failure import Failure - from twisted.web._newclient import ResponseDone - from twisted.web.client import ( - Agent, - HTTPConnectionPool, - ResponseNeverReceived, - readBody, - ) - from twisted.web.http import PotentialDataLoss - from twisted.web.http_headers import Headers - from twisted.web.iweb import ( - UNKNOWN_LENGTH, - IAgent, - IBodyProducer, - IPolicyForHTTPS, - IResponse, - ) -except ImportError: - pass from synapse.api.errors import Codes, HttpResponseException, SynapseError -from synapse.http import QuieterFileBodyProducer, RequestTimedOutError, redact_uri -from synapse.http.proxyagent import ProxyAgent -from synapse.http.replicationagent import ReplicationAgent +from synapse.http import RequestTimedOutError, redact_uri from synapse.http.types import QueryParams -from synapse.logging.context import ( - PreserveLoggingContext, - make_deferred_yieldable, - run_in_background, -) from synapse.logging.opentracing import set_tag, start_active_span, tags from synapse.metrics import SERVER_NAME_LABEL from synapse.types import ISynapseReactor, StrSequence -from synapse.util.async_helpers import timeout_deferred from synapse.util.clock import Clock -from synapse.util.duration import Duration -from synapse.util.json import json_decoder if TYPE_CHECKING: from synapse.server import HomeServer -# Support both import names for the `python-multipart` (PyPI) library, -# which renamed its package name from `multipart` to `python_multipart` -# in 0.0.13 (though supports the old import name for compatibility). -# Note that the `multipart` package name conflicts with `multipart` (PyPI) -# so we should prefer importing from `python_multipart` when possible. +# Support both import names for the `python-multipart` (PyPI) library. try: from python_multipart import MultipartParser @@ -172,858 +113,47 @@ def _is_ip_blocked( return False -# The delay used by the scheduler to schedule tasks "as soon as possible", while -# still allowing other tasks to run between runs. -_EPSILON = Duration(microseconds=1) - - -def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCall]: - """Makes a schedular suitable for a Cooperator using the given reactor. - - (This is effectively just a copy from `twisted.internet.task`) +class BlocklistingReactorWrapper: """ + A wrapper which filters out blocked IP addresses from DNS resolution, + to prevent DNS rebinding. - def _scheduler(x: Callable[[], object]) -> IDelayedCall: - return clock.call_later( - _EPSILON, - x, - ) - - return _scheduler - - -class _IPBlockingResolver: - """ - A proxy for reactor.nameResolver which only produces non-blocklisted IP - addresses, preventing DNS rebinding attacks. + Kept for backward compatibility; the actual IP blocking for aiohttp-based + clients is done via aiohttp's TCPConnector with a custom resolver. """ def __init__( self, - reactor: IReactorPluggableNameResolver, + reactor: Any, ip_allowlist: IPSet | None, ip_blocklist: IPSet, ): - """ - Args: - reactor: The twisted reactor. - ip_allowlist: IP addresses to allow. - ip_blocklist: IP addresses to disallow. - """ self._reactor = reactor self._ip_allowlist = ip_allowlist self._ip_blocklist = ip_blocklist - def resolveHostName( - self, recv: IResolutionReceiver, hostname: str, portNumber: int = 0 - ) -> IResolutionReceiver: - addresses: list[IAddress] = [] - - def _callback() -> None: - has_bad_ip = False - for address in addresses: - # We only expect IPv4 and IPv6 addresses since only A/AAAA lookups - # should go through this path. - if not isinstance(address, (IPv4Address, IPv6Address)): - continue - - ip_address = IPAddress(address.host) - - if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist): - logger.info( - "Blocked %s from DNS resolution to %s", ip_address, hostname - ) - has_bad_ip = True - - # if we have a blocked IP, we'd like to raise an error to block the - # request, but all we can really do from here is claim that there were no - # valid results. - if not has_bad_ip: - for address in addresses: - recv.addressResolved(address) - recv.resolutionComplete() - - @implementer(IResolutionReceiver) - class EndpointReceiver: - @staticmethod - def resolutionBegan(resolutionInProgress: IHostResolution) -> None: - recv.resolutionBegan(resolutionInProgress) - - @staticmethod - def addressResolved(address: IAddress) -> None: - addresses.append(address) - - @staticmethod - def resolutionComplete() -> None: - _callback() - - endpoint_receiver_wrapper = EndpointReceiver() - self._reactor.nameResolver.resolveHostName( - endpoint_receiver_wrapper, hostname, portNumber=portNumber - ) - - return recv - - -# ISynapseReactor implies IReactorCore, but explicitly marking it this as an implementer -# of IReactorCore seems to keep mypy-zope happier. -@implementer(IReactorCore, ISynapseReactor) -class BlocklistingReactorWrapper: - """ - A Reactor wrapper which will prevent DNS resolution to blocked IP - addresses, to prevent DNS rebinding. - """ - - def __init__( - self, - reactor: IReactorPluggableNameResolver, - ip_allowlist: IPSet | None, - ip_blocklist: IPSet, - ): - self._reactor = reactor - - # We need to use a DNS resolver which filters out blocked IP - # addresses, to prevent DNS rebinding. - self._nameResolver = _IPBlockingResolver( - self._reactor, ip_allowlist, ip_blocklist - ) - def __getattr__(self, attr: str) -> Any: - # Passthrough to the real reactor except for the DNS resolver. - if attr == "nameResolver": - return self._nameResolver - else: - return getattr(self._reactor, attr) + return getattr(self._reactor, attr) -class BlocklistingAgentWrapper(Agent): +class BlocklistingAgentWrapper: """ - An Agent wrapper which will prevent access to IP addresses being accessed - directly (without an IP address lookup). + Stub for backward compatibility. The actual IP blocking for aiohttp is done + via aiohttp's connector. """ def __init__( self, - agent: IAgent, + agent: Any, ip_blocklist: IPSet, ip_allowlist: IPSet | None = None, ): - """ - Args: - agent: The Agent to wrap. - ip_allowlist: IP addresses to allow. - ip_blocklist: IP addresses to disallow. - """ self._agent = agent self._ip_allowlist = ip_allowlist self._ip_blocklist = ip_blocklist - def request( - self, - method: bytes, - uri: bytes, - headers: Headers | None = None, - bodyProducer: Optional[IBodyProducer] = None, - ) -> defer.Deferred: - h = urllib.parse.urlparse(uri.decode("ascii")) - - try: - # h.hostname is str | None, None raises an AddrFormatError, so - # this is safe even though IPAddress requires a str. - ip_address = IPAddress(h.hostname) # type: ignore[arg-type] - except AddrFormatError: - # Not an IP - pass - else: - if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist): - logger.info("Blocking access to %s", ip_address) - e = SynapseError(HTTPStatus.FORBIDDEN, "IP address blocked") - return defer.fail(Failure(e)) - - return self._agent.request( - method, uri, headers=headers, bodyProducer=bodyProducer - ) - - -class BaseHttpClient: - """ - A simple, no-frills HTTP client with methods that wrap up common ways of - using HTTP in Matrix. Does not come with a default Agent, subclasses will need to - define their own. - - Args: - hs: The HomeServer instance to pass in - treq_args: Extra keyword arguments to be given to treq.request. - """ - - agent: IAgent - - def __init__( - self, - hs: "HomeServer", - treq_args: dict[str, Any] | None = None, - ): - self.hs = hs - self.server_name = hs.hostname - self.reactor = hs.get_reactor() - - self._extra_treq_args = treq_args or {} - self.clock = hs.get_clock() - - user_agent = hs.version_string - if hs.config.server.user_agent_suffix: - user_agent = "%s %s" % ( - user_agent, - hs.config.server.user_agent_suffix, - ) - self.user_agent = user_agent.encode("ascii") - - # We use this for our body producers to ensure that they use the correct - # reactor. - self._cooperator = Cooperator(scheduler=_make_scheduler(hs.get_clock())) - - async def request( - self, - method: str, - uri: str, - data: bytes | None = None, - headers: Headers | None = None, - ) -> IResponse: - """ - Args: - method: HTTP method to use. - uri: URI to query. - data: Data to send in the request body, if applicable. - headers: Request headers. - - Returns: - Response object, once the headers have been read. - - Raises: - RequestTimedOutError if the request times out before the headers are read - - """ - outgoing_requests_counter.labels( - method=method, **{SERVER_NAME_LABEL: self.server_name} - ).inc() - - # log request but strip `access_token` (AS requests for example include this) - logger.debug("Sending request %s %s", method, redact_uri(uri)) - - with start_active_span( - "outgoing-client-request", - tags={ - tags.SPAN_KIND: tags.SPAN_KIND_RPC_CLIENT, - tags.HTTP_METHOD: method, - tags.HTTP_URL: uri, - }, - finish_on_close=True, - ): - try: - body_producer = None - if data is not None: - body_producer = QuieterFileBodyProducer( - BytesIO(data), - cooperator=self._cooperator, - ) - - # Always make sure we add a user agent to the request - if headers is None: - headers = Headers() - - if not headers.hasHeader("User-Agent"): - headers.addRawHeader("User-Agent", self.user_agent) - - request_deferred: defer.Deferred = treq.request( - method, - uri, - agent=self.agent, - data=body_producer, - headers=headers, - # Avoid buffering the body in treq since we do not reuse - # response bodies. - unbuffered=True, - **self._extra_treq_args, - ) - - # we use our own timeout mechanism rather than treq's as a workaround - # for https://twistedmatrix.com/trac/ticket/9534. - request_deferred = timeout_deferred( - deferred=request_deferred, - timeout=60, - clock=self.hs.get_clock(), - ) - - # turn timeouts into RequestTimedOutErrors - request_deferred.addErrback(_timeout_to_request_timed_out_error) - - response = await make_deferred_yieldable(request_deferred) - - incoming_responses_counter.labels( - method=method, - code=response.code, - **{SERVER_NAME_LABEL: self.server_name}, - ).inc() - logger.info( - "Received response to %s %s: %s", - method, - redact_uri(uri), - response.code, - ) - return response - except Exception as e: - incoming_responses_counter.labels( - method=method, - code="ERR", - **{SERVER_NAME_LABEL: self.server_name}, - ).inc() - logger.info( - "Error sending request to %s %s: %s %s", - method, - redact_uri(uri), - type(e).__name__, - e.args[0], - ) - set_tag(tags.ERROR, True) - set_tag("error_reason", e.args[0]) - raise - - async def post_urlencoded_get_json( - self, - uri: str, - args: Mapping[str, str | list[str]] | None = None, - headers: RawHeaders | None = None, - ) -> Any: - """ - Args: - uri: uri to query - args: parameters to be url-encoded in the body - headers: a map from header name to a list of values for that header - - Returns: - parsed json - - Raises: - RequestTimedOutError: if there is a timeout before the response headers - are received. Note there is currently no timeout on reading the response - body. - - HttpResponseException: On a non-2xx HTTP response. - - ValueError: if the response was not JSON - """ - - # TODO: Do we ever want to log message contents? - logger.debug("post_urlencoded_get_json args: %s", args) - - query_bytes = encode_query_args(args) - - actual_headers = { - b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [self.user_agent], - b"Accept": [b"application/json"], - } - if headers: - actual_headers.update(headers) # type: ignore - - response = await self.request( - "POST", uri, headers=Headers(actual_headers), data=query_bytes - ) - - body = await make_deferred_yieldable(readBody(response)) - - if 200 <= response.code < 300: - return json_decoder.decode(body.decode("utf-8")) - else: - raise HttpResponseException( - response.code, response.phrase.decode("ascii", errors="replace"), body - ) - - async def post_json_get_json( - self, uri: str, post_json: Any, headers: RawHeaders | None = None - ) -> Any: - """ - - Args: - uri: URI to query. - post_json: request body, to be encoded as json - headers: a map from header name to a list of values for that header - - Returns: - parsed json - - Raises: - RequestTimedOutError: if there is a timeout before the response headers - are received. Note there is currently no timeout on reading the response - body. - - HttpResponseException: On a non-2xx HTTP response. - - ValueError: if the response was not JSON - """ - json_str = encode_canonical_json(post_json) - - logger.debug("HTTP POST %s -> %s", json_str, uri) - - actual_headers = { - b"Content-Type": [b"application/json"], - b"User-Agent": [self.user_agent], - b"Accept": [b"application/json"], - } - if headers: - actual_headers.update(headers) # type: ignore - - response = await self.request( - "POST", uri, headers=Headers(actual_headers), data=json_str - ) - - body = await make_deferred_yieldable(readBody(response)) - - if 200 <= response.code < 300: - return json_decoder.decode(body.decode("utf-8")) - else: - raise HttpResponseException( - response.code, response.phrase.decode("ascii", errors="replace"), body - ) - - async def get_json( - self, - uri: str, - args: QueryParams | None = None, - headers: RawHeaders | None = None, - ) -> Any: - """Gets some json from the given URI. - - Args: - uri: The URI to request, not including query parameters - args: A dictionary used to create query string - headers: a map from header name to a list of values for that header - Returns: - Succeeds when we get a 2xx HTTP response, with the HTTP body as JSON. - Raises: - RequestTimedOutError: if there is a timeout before the response headers - are received. Note there is currently no timeout on reading the response - body. - - HttpResponseException On a non-2xx HTTP response. - - ValueError: if the response was not JSON - """ - actual_headers = {b"Accept": [b"application/json"]} - if headers: - actual_headers.update(headers) # type: ignore - - body = await self.get_raw(uri, args, headers=actual_headers) - return json_decoder.decode(body.decode("utf-8")) - - async def put_json( - self, - uri: str, - json_body: Any, - args: QueryParams | None = None, - headers: RawHeaders | None = None, - ) -> Any: - """Puts some json to the given URI. - - Args: - uri: The URI to request, not including query parameters - json_body: The JSON to put in the HTTP body, - args: A dictionary used to create query strings - headers: a map from header name to a list of values for that header - Returns: - Succeeds when we get a 2xx HTTP response, with the HTTP body as JSON. - Raises: - RequestTimedOutError: if there is a timeout before the response headers - are received. Note there is currently no timeout on reading the response - body. - - HttpResponseException On a non-2xx HTTP response. - - ValueError: if the response was not JSON - """ - if args: - query_str = urllib.parse.urlencode(args, True) - uri = "%s?%s" % (uri, query_str) - - json_str = encode_canonical_json(json_body) - - actual_headers = { - b"Content-Type": [b"application/json"], - b"User-Agent": [self.user_agent], - b"Accept": [b"application/json"], - } - if headers: - actual_headers.update(headers) # type: ignore - - response = await self.request( - "PUT", uri, headers=Headers(actual_headers), data=json_str - ) - - body = await make_deferred_yieldable(readBody(response)) - - if 200 <= response.code < 300: - return json_decoder.decode(body.decode("utf-8")) - else: - raise HttpResponseException( - response.code, response.phrase.decode("ascii", errors="replace"), body - ) - - async def get_raw( - self, - uri: str, - args: QueryParams | None = None, - headers: RawHeaders | None = None, - ) -> bytes: - """Gets raw text from the given URI. - - Args: - uri: The URI to request, not including query parameters - args: A dictionary used to create query strings - headers: a map from header name to a list of values for that header - Returns: - Succeeds when we get a 2xx HTTP response, with the - HTTP body as bytes. - Raises: - RequestTimedOutError: if there is a timeout before the response headers - are received. Note there is currently no timeout on reading the response - body. - - HttpResponseException on a non-2xx HTTP response. - """ - if args: - query_str = urllib.parse.urlencode(args, True) - uri = "%s?%s" % (uri, query_str) - - actual_headers = {b"User-Agent": [self.user_agent]} - if headers: - actual_headers.update(headers) # type: ignore - - response = await self.request("GET", uri, headers=Headers(actual_headers)) - - body = await make_deferred_yieldable(readBody(response)) - - if 200 <= response.code < 300: - return body - else: - raise HttpResponseException( - response.code, response.phrase.decode("ascii", errors="replace"), body - ) - - # XXX: FIXME: This is horribly copy-pasted from matrixfederationclient. - # The two should be factored out. - - async def get_file( - self, - url: str, - output_stream: BinaryIO, - max_size: int | None = None, - headers: RawHeaders | None = None, - is_allowed_content_type: Callable[[str], bool] | None = None, - ) -> tuple[int, dict[bytes, list[bytes]], str, int]: - """GETs a file from a given URL - Args: - url: The URL to GET - output_stream: File to write the response body to. - headers: A map from header name to a list of values for that header - is_allowed_content_type: A predicate to determine whether the - content type of the file we're downloading is allowed. If set and - it evaluates to False when called with the content type, the - request will be terminated before completing the download by - raising SynapseError. - Returns: - A tuple of the file length, dict of the response - headers, absolute URI of the response and HTTP response code. - - Raises: - RequestTimedOutError: if there is a timeout before the response headers - are received. Note there is currently no timeout on reading the response - body. - - SynapseError: if the response is not a 2xx, the remote file is too large, or - another exception happens during the download. - """ - - actual_headers = {b"User-Agent": [self.user_agent]} - if headers: - actual_headers.update(headers) # type: ignore - - response = await self.request("GET", url, headers=Headers(actual_headers)) - - resp_headers = dict(response.headers.getAllRawHeaders()) - - if response.code > 299: - logger.warning("Got %d when downloading %s", response.code, url) - raise SynapseError( - HTTPStatus.BAD_GATEWAY, "Got error %d" % (response.code,), Codes.UNKNOWN - ) - - if is_allowed_content_type and b"Content-Type" in resp_headers: - content_type = resp_headers[b"Content-Type"][0].decode("ascii") - if not is_allowed_content_type(content_type): - raise SynapseError( - HTTPStatus.BAD_GATEWAY, - ( - "Requested file's content type not allowed for this operation: %s" - % content_type - ), - ) - - # TODO: if our Content-Type is HTML or something, just read the first - # N bytes into RAM rather than saving it all to disk only to read it - # straight back in again - - try: - d = read_body_with_max_size(response, output_stream, max_size) - - # Ensure that the body is not read forever. - d = timeout_deferred( - deferred=d, - timeout=30, - clock=self.hs.get_clock(), - ) - - length = await make_deferred_yieldable(d) - except BodyExceededMaxSize: - raise SynapseError( - HTTPStatus.BAD_GATEWAY, - "Requested file is too large > %r bytes" % (max_size,), - Codes.TOO_LARGE, - ) - except asyncio.TimeoutError: - raise SynapseError( - HTTPStatus.BAD_GATEWAY, - "Requested file took too long to download", - Codes.TOO_LARGE, - ) - except Exception as e: - raise SynapseError( - HTTPStatus.BAD_GATEWAY, ("Failed to download remote body: %s" % e) - ) from e - - return ( - length, - resp_headers, - response.request.absoluteURI.decode("ascii"), - response.code, - ) - - -class SimpleHttpClient(BaseHttpClient): - """ - An HTTP client capable of crossing a proxy and respecting a block/allow list. - - This also configures a larger / longer lasting HTTP connection pool. - - Args: - hs: The HomeServer instance to pass in - treq_args: Extra keyword arguments to be given to treq.request. - ip_blocklist: The IP addresses that we may not request. - ip_allowlist: The allowed IP addresses, that we can - request if it were otherwise caught in a blocklist. - use_proxy: Whether proxy settings should be discovered and used - from conventional environment variables. - """ - - def __init__( - self, - hs: "HomeServer", - treq_args: dict[str, Any] | None = None, - ip_allowlist: IPSet | None = None, - ip_blocklist: IPSet | None = None, - use_proxy: bool = False, - ): - super().__init__(hs, treq_args=treq_args) - self._ip_allowlist = ip_allowlist - self._ip_blocklist = ip_blocklist - - if self._ip_blocklist: - # If we have an IP blocklist, we need to use a DNS resolver which - # filters out blocked IP addresses, to prevent DNS rebinding. - self.reactor: ISynapseReactor = BlocklistingReactorWrapper( - self.reactor, self._ip_allowlist, self._ip_blocklist - ) - - # the pusher makes lots of concurrent SSL connections to Sygnal, and tends to - # do so in batches, so we need to allow the pool to keep lots of idle - # connections around. - pool = HTTPConnectionPool(self.reactor) - # XXX: The justification for using the cache factor here is that larger - # instances will need both more cache and more connections. - # Still, this should probably be a separate dial - pool.maxPersistentPerHost = max(int(100 * hs.config.caches.global_factor), 5) - pool.cachedConnectionTimeout = 2 * 60 - - self.agent: IAgent = ProxyAgent( - reactor=self.reactor, - proxy_reactor=hs.get_reactor(), - connectTimeout=15, - contextFactory=self.hs.get_http_client_context_factory(), - pool=pool, - proxy_config=hs.config.server.proxy_config, - ) - - if self._ip_blocklist: - # If we have an IP blocklist, we then install the Agent which prevents - # direct access to IP addresses, that are not caught by the DNS resolution. - self.agent = BlocklistingAgentWrapper( - self.agent, - ip_blocklist=self._ip_blocklist, - ip_allowlist=self._ip_allowlist, - ) - - -class ReplicationClient(BaseHttpClient): - """Client for connecting to replication endpoints via HTTP and HTTPS. - - Attributes: - agent: The custom Twisted Agent used for constructing the connection. - """ - - def __init__( - self, - hs: "HomeServer", - ): - """ - Args: - hs: The HomeServer instance to pass in - """ - super().__init__(hs) - self.server_name = hs.hostname - - # Use a pool, but a very small one. - pool = HTTPConnectionPool(self.reactor) - pool.maxPersistentPerHost = 5 - pool.cachedConnectionTimeout = 2 * 60 - - self.agent: IAgent = ReplicationAgent( - hs.get_reactor(), - hs.config.worker.instance_map, - contextFactory=hs.get_http_client_context_factory(), - pool=pool, - ) - - async def request( - self, - method: str, - uri: str, - data: bytes | None = None, - headers: Headers | None = None, - ) -> IResponse: - """ - Make a request, differs from BaseHttpClient.request in that it does not use treq. - - Args: - method: HTTP method to use. - uri: URI to query. - data: Data to send in the request body, if applicable. - headers: Request headers. - - Returns: - Response object, once the headers have been read. - - Raises: - RequestTimedOutError if the request times out before the headers are read - - """ - outgoing_requests_counter.labels( - method=method, **{SERVER_NAME_LABEL: self.server_name} - ).inc() - - logger.debug("Sending request %s %s", method, uri) - - with start_active_span( - "outgoing-replication-request", - tags={ - tags.SPAN_KIND: tags.SPAN_KIND_RPC_CLIENT, - tags.HTTP_METHOD: method, - tags.HTTP_URL: uri, - }, - finish_on_close=True, - ): - try: - body_producer = None - if data is not None: - body_producer = QuieterFileBodyProducer( - BytesIO(data), - cooperator=self._cooperator, - ) - - # Skip the fancy treq stuff, we don't need cookie handling, redirects, - # or buffered response bodies. - method_bytes = method.encode("ascii") - uri_bytes = uri.encode("ascii") - - # To preserve the logging context, the timeout is treated - # in a similar way to `defer.gatherResults`: - # * Each logging context-preserving fork is wrapped in - # `run_in_background`. In this case there is only one, - # since the timeout fork is not logging-context aware. - # * The `Deferred` that joins the forks back together is - # wrapped in `make_deferred_yieldable` to restore the - # logging context regardless of the path taken. - # (The logic/comments for this came from MatrixFederationHttpClient) - request_deferred = run_in_background( - self.agent.request, - method_bytes, - uri_bytes, - headers, - bodyProducer=body_producer, - ) - - # we use our own timeout mechanism rather than twisted's as a workaround - # for https://twistedmatrix.com/trac/ticket/9534. - # (Updated url https://github.com/twisted/twisted/issues/9534) - request_deferred = timeout_deferred( - deferred=request_deferred, - timeout=60, - clock=self.hs.get_clock(), - ) - - # turn timeouts into RequestTimedOutErrors - request_deferred.addErrback(_timeout_to_request_timed_out_error) - - response = await make_deferred_yieldable(request_deferred) - - incoming_responses_counter.labels( - method=method, - code=response.code, - **{SERVER_NAME_LABEL: self.server_name}, - ).inc() - logger.info( - "Received response to %s %s: %s", - method, - uri, - response.code, - ) - return response - except Exception as e: - incoming_responses_counter.labels( - method=method, - code="ERR", - **{SERVER_NAME_LABEL: self.server_name}, - ).inc() - logger.info( - "Error sending request to %s %s: %s %s", - method, - uri, - type(e).__name__, - e.args[0], - ) - set_tag(tags.ERROR, True) - set_tag("error_reason", e.args[0]) - raise - - -def _timeout_to_request_timed_out_error(f: Failure) -> Failure: - if f.check(twisted_error.TimeoutError, twisted_error.ConnectingCancelledError): - # The TCP connection has its own timeout (set by the 'connectTimeout' param - # on the Agent), which raises twisted_error.TimeoutError exception. - raise RequestTimedOutError("Timeout connecting to remote server") - elif f.check(asyncio.TimeoutError, ResponseNeverReceived): - # this one means that we hit our overall timeout on the request - raise RequestTimedOutError("Timeout waiting for response from remote server") - - return f + def request(self, *args: Any, **kwargs: Any) -> Any: + return self._agent.request(*args, **kwargs) class ByteWriteable(Protocol): @@ -1040,33 +170,6 @@ class BodyExceededMaxSize(Exception): """The maximum allowed size of the HTTP body was exceeded.""" -class _DiscardBodyWithMaxSizeProtocol(protocol.Protocol): - """A protocol which immediately errors upon receiving data.""" - - transport: Optional[ITCPTransport] = None - - def __init__(self, deferred: defer.Deferred): - self.deferred = deferred - - def _maybe_fail(self) -> None: - """ - Report a max size exceed error and disconnect the first time this is called. - """ - if not self.deferred.called: - with PreserveLoggingContext(): - self.deferred.errback(BodyExceededMaxSize()) - # Close the connection (forcefully) since all the data will get - # discarded anyway. - assert self.transport is not None - self.transport.abortConnection() - - def dataReceived(self, data: bytes) -> None: - self._maybe_fail() - - def connectionLost(self, reason: Failure = connectionDone) -> None: - self._maybe_fail() - - @attr.s(auto_attribs=True, slots=True) class MultipartResponse: """ @@ -1080,211 +183,6 @@ class MultipartResponse: url: bytes | None = None -class _MultipartParserProtocol(protocol.Protocol): - """ - Protocol to read and parse a MSC3916 multipart/mixed response - """ - - transport: Optional[ITCPTransport] = None - - def __init__( - self, - stream: ByteWriteable, - deferred: defer.Deferred, - boundary: str, - max_length: int | None, - ) -> None: - self.stream = stream - self.deferred = deferred - self.boundary = boundary - self.max_length = max_length - self.parser: MultipartParser | None = None - self.multipart_response = MultipartResponse() - self.has_redirect = False - self.in_json = False - self.json_done = False - self.file_length = 0 - self.total_length = 0 - self.in_disposition = False - self.in_content_type = False - - def dataReceived(self, incoming_data: bytes) -> None: - if self.deferred.called: - return - - # we don't have a parser yet, instantiate it - if not self.parser: - - def on_header_field(data: bytes, start: int, end: int) -> None: - if data[start:end].lower() == b"location": - self.has_redirect = True - if data[start:end].lower() == b"content-disposition": - self.in_disposition = True - if data[start:end].lower() == b"content-type": - self.in_content_type = True - - def on_header_value(data: bytes, start: int, end: int) -> None: - # the first header should be content-type for application/json - if not self.in_json and not self.json_done: - assert data[start:end] == b"application/json" - self.in_json = True - elif self.has_redirect: - self.multipart_response.url = data[start:end] - elif self.in_content_type: - self.multipart_response.content_type = data[start:end] - self.in_content_type = False - elif self.in_disposition: - self.multipart_response.disposition = data[start:end] - self.in_disposition = False - - def on_part_data(data: bytes, start: int, end: int) -> None: - # we've seen json header but haven't written the json data - if self.in_json and not self.json_done: - self.multipart_response.json = data[start:end] - self.json_done = True - # we have a redirect header rather than a file, and have already captured it - elif self.has_redirect: - return - # otherwise we are in the file part - else: - try: - self.stream.write(data[start:end]) - except Exception as e: - logger.warning( - "Exception encountered writing file data to stream: %s", e - ) - with PreserveLoggingContext(): - self.deferred.errback() - self.file_length += end - start - - callbacks: "multipart.MultipartCallbacks" = { - "on_header_field": on_header_field, - "on_header_value": on_header_value, - "on_part_data": on_part_data, - } - self.parser = MultipartParser(self.boundary, callbacks) - - self.total_length += len(incoming_data) - if self.max_length is not None and self.total_length >= self.max_length: - with PreserveLoggingContext(): - self.deferred.errback(BodyExceededMaxSize()) - # Close the connection (forcefully) since all the data will get - # discarded anyway. - assert self.transport is not None - self.transport.abortConnection() - - try: - self.parser.write(incoming_data) - except Exception as e: - logger.warning("Exception writing to multipart parser: %s", e) - with PreserveLoggingContext(): - self.deferred.errback() - return - - def connectionLost(self, reason: Failure = connectionDone) -> None: - # If the maximum size was already exceeded, there's nothing to do. - if self.deferred.called: - return - - if reason.check(ResponseDone): - self.multipart_response.length = self.file_length - with PreserveLoggingContext(): - self.deferred.callback(self.multipart_response) - else: - with PreserveLoggingContext(): - self.deferred.errback(reason) - - -class _ReadBodyWithMaxSizeProtocol(protocol.Protocol): - """A protocol which reads body to a stream, erroring if the body exceeds a maximum size.""" - - transport: Optional[ITCPTransport] = None - - def __init__( - self, stream: ByteWriteable, deferred: defer.Deferred, max_size: int | None - ): - self.stream = stream - self.deferred = deferred - self.length = 0 - self.max_size = max_size - - def dataReceived(self, data: bytes) -> None: - # If the deferred was called, bail early. - if self.deferred.called: - return - - try: - self.stream.write(data) - except Exception: - with PreserveLoggingContext(): - self.deferred.errback() - return - - self.length += len(data) - # The first time the maximum size is exceeded, error and cancel the - # connection. dataReceived might be called again if data was received - # in the meantime. - if self.max_size is not None and self.length >= self.max_size: - with PreserveLoggingContext(): - self.deferred.errback(BodyExceededMaxSize()) - # Close the connection (forcefully) since all the data will get - # discarded anyway. - assert self.transport is not None - self.transport.abortConnection() - - def connectionLost(self, reason: Failure = connectionDone) -> None: - # If the maximum size was already exceeded, there's nothing to do. - if self.deferred.called: - return - - if reason.check(ResponseDone): - with PreserveLoggingContext(): - self.deferred.callback(self.length) - elif reason.check(PotentialDataLoss): - # This applies to requests which don't set `Content-Length` or a - # `Transfer-Encoding` in the response because in this case the end of the - # response is indicated by the connection being closed, an event which may - # also be due to a transient network problem or other error. But since this - # behavior is expected of some servers (like YouTube), let's ignore it. - # Stolen from https://github.com/twisted/treq/pull/49/files - # http://twistedmatrix.com/trac/ticket/4840 - with PreserveLoggingContext(): - self.deferred.callback(self.length) - else: - with PreserveLoggingContext(): - self.deferred.errback(reason) - - -def read_body_with_max_size( - response: IResponse, stream: ByteWriteable, max_size: int | None -) -> "defer.Deferred[int]": - """ - Read a HTTP response body to a file-object. Optionally enforcing a maximum file size. - - If the maximum file size is reached, the returned Deferred will resolve to a - Failure with a BodyExceededMaxSize exception. - - Args: - response: The HTTP response to read from. - stream: The file-object to write to. - max_size: The maximum file size to allow. - - Returns: - A Deferred which resolves to the length of the read body. - """ - d: "defer.Deferred[int]" = defer.Deferred() - - # If the Content-Length header gives a size larger than the maximum allowed - # size, do not bother downloading the body. - if max_size is not None and response.length != UNKNOWN_LENGTH: - if response.length > max_size: - response.deliverBody(_DiscardBodyWithMaxSizeProtocol(d)) - return d - - response.deliverBody(_ReadBodyWithMaxSizeProtocol(stream, d, max_size)) - return d - - async def async_read_body_with_max_size( response: aiohttp.ClientResponse, stream: ByteWriteable, @@ -1315,32 +213,6 @@ async def async_read_body_with_max_size( return length -def read_multipart_response( - response: IResponse, stream: ByteWriteable, boundary: str, max_length: int | None -) -> "defer.Deferred[MultipartResponse]": - """ - Reads a MSC3916 multipart/mixed response and parses it, reading the file part (if it contains one) into - the stream passed in and returning a deferred resolving to a MultipartResponse - - Args: - response: The HTTP response to read from. - stream: The file-object to write to. - boundary: the multipart/mixed boundary string - max_length: maximum allowable length of the response - """ - d: defer.Deferred[MultipartResponse] = defer.Deferred() - - # If the Content-Length header gives a size larger than the maximum allowed - # size, do not bother downloading the body. - if max_length is not None and response.length != UNKNOWN_LENGTH: - if response.length > max_length: - response.deliverBody(_DiscardBodyWithMaxSizeProtocol(d)) - return d - - response.deliverBody(_MultipartParserProtocol(stream, d, boundary, max_length)) - return d - - def encode_query_args(args: QueryParams | None) -> bytes: """ Encodes a map of query arguments to bytes which can be appended to a URL. @@ -1359,25 +231,6 @@ def encode_query_args(args: QueryParams | None) -> bytes: return query_str.encode("utf8") -@implementer(IPolicyForHTTPS) -class InsecureInterceptableContextFactory(ssl.ContextFactory): - """ - Factory for PyOpenSSL SSL contexts which accepts any certificate for any domain. - - Do not use this since it allows an attacker to intercept your communications. - """ - - def __init__(self) -> None: - self._context = SSL.Context(SSL.SSLv23_METHOD) - self._context.set_verify(VERIFY_NONE, lambda *_: False) - - def getContext(self) -> SSL.Context: - return self._context - - def creatorForNetloc(self, hostname: bytes, port: int) -> IOpenSSLContextFactory: - return self - - def is_unknown_endpoint( e: HttpResponseException, synapse_error: SynapseError | None = None ) -> bool: @@ -1414,3 +267,107 @@ def is_unknown_endpoint( # Older Synapses returned a 400 error. e.code == 400 and synapse_error.errcode == Codes.UNRECOGNIZED ) + + +def check_content_type_is(headers: dict[bytes, list[bytes]], expected_content_type: str) -> None: + """Check that the Content-Type header matches the expected value. + + Args: + headers: response headers + expected_content_type: expected content type + + Raises: + RequestSendFailed if the content type doesn't match. + """ + content_type_headers = headers.get(b"Content-Type", headers.get(b"content-type", [])) + if not content_type_headers: + raise RuntimeError(f"No Content-Type header, expected {expected_content_type}") + + c_type = content_type_headers[0].decode("ascii") + # Check the main type, ignoring parameters + main_type = c_type.split(";", 1)[0].strip() + if main_type != expected_content_type: + raise RuntimeError( + f"Content-Type was {c_type}, expected {expected_content_type}" + ) + + +def check_content_type_is_aiohttp( + response: aiohttp.ClientResponse, expected_content_type: str +) -> None: + """Check that an aiohttp response has the expected Content-Type. + + Args: + response: The aiohttp response. + expected_content_type: The expected content type. + + Raises: + RuntimeError if the content type doesn't match. + """ + c_type = response.content_type or "" + if c_type != expected_content_type: + raise RuntimeError( + f"Content-Type was {c_type}, expected {expected_content_type}" + ) + + +# Backward compatibility stubs for Twisted-based HTTP client classes. +# These have been superseded by NativeSimpleHttpClient (aiohttp-based). + +class SimpleHttpClient: + """Stub: The Twisted SimpleHttpClient has been removed. + + Use NativeSimpleHttpClient instead. + """ + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "SimpleHttpClient has been removed. Use NativeSimpleHttpClient instead." + ) + + +class BaseHttpClient: + """Stub: The Twisted BaseHttpClient has been removed.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "BaseHttpClient has been removed. Use NativeSimpleHttpClient instead." + ) + + +class ReplicationClient: + """Stub: The Twisted ReplicationClient has been removed.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "ReplicationClient has been removed. Use NativeReplicationClient instead." + ) + + +class InsecureInterceptableContextFactory: + """Stub: The Twisted InsecureInterceptableContextFactory has been removed.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "InsecureInterceptableContextFactory has been removed." + ) + + +# Keep these for backward compatibility of imports that use Twisted-era functions. +# They delegate to Twisted if available, otherwise raise. + +def read_body_with_max_size(*args: Any, **kwargs: Any) -> Any: + """Stub: requires Twisted. Use async_read_body_with_max_size for aiohttp.""" + try: + from twisted.internet import defer, protocol + # If Twisted is available, provide the original implementation + raise NotImplementedError("Use async_read_body_with_max_size instead") + except ImportError: + raise NotImplementedError( + "read_body_with_max_size requires Twisted. " + "Use async_read_body_with_max_size for aiohttp responses." + ) + + +def read_multipart_response(*args: Any, **kwargs: Any) -> Any: + """Stub: requires Twisted. Use aiohttp multipart parsing instead.""" + raise NotImplementedError( + "read_multipart_response requires Twisted. " + "Use aiohttp multipart parsing instead." + ) diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py index d58960284f..816dfc6874 100644 --- a/synapse/http/connectproxyclient.py +++ b/synapse/http/connectproxyclient.py @@ -22,37 +22,14 @@ import abc import base64 import logging -from typing import Union +from typing import Any import attr -try: - from zope.interface import implementer -except ImportError: - pass - -try: - from twisted.internet import defer, protocol - from twisted.internet.error import ConnectError - from twisted.internet.interfaces import ( - IAddress, - IConnector, - IProtocol, - IProtocolFactory, - IReactorCore, - IStreamClientEndpoint, - ) - from twisted.internet.protocol import ClientFactory, connectionDone - from twisted.python.failure import Failure - from twisted.web import http -except ImportError: - pass - -from synapse.logging.context import PreserveLoggingContext logger = logging.getLogger(__name__) -class ProxyConnectError(ConnectError): +class ProxyConnectError(ConnectionError): pass @@ -89,234 +66,19 @@ class BearerProxyCredentials(ProxyCredentials): return b"Bearer " + self.access_token -@implementer(IStreamClientEndpoint) +# The Twisted-based HTTP CONNECT proxy protocol classes (HTTPConnectProxyEndpoint, +# HTTPProxiedClientFactory, HTTPConnectProtocol, HTTPConnectSetupClient) have been +# removed as part of the Twisted->asyncio migration. The proxy functionality is now +# handled by aiohttp's native proxy support in NativeSimpleHttpClient. +# +# For backward compatibility, we keep a stub that raises if someone tries to use it. class HTTPConnectProxyEndpoint: - """An Endpoint implementation which will send a CONNECT request to an http proxy + """Stub: The Twisted HTTP CONNECT proxy endpoint has been removed. - Wraps an existing HostnameEndpoint for the proxy. - - When we get the connect() request from the connection pool (via the TLS wrapper), - we'll first connect to the proxy endpoint with a ProtocolFactory which will make the - CONNECT request. Once that completes, we invoke the protocolFactory which was passed - in. - - Args: - reactor: the Twisted reactor to use for the connection - proxy_endpoint: the endpoint to use to connect to the proxy - host: hostname that we want to CONNECT to - port: port that we want to connect to - proxy_creds: credentials to authenticate at proxy + Proxy support is now handled by aiohttp natively. """ - - def __init__( - self, - reactor: IReactorCore, - proxy_endpoint: IStreamClientEndpoint, - host: bytes, - port: int, - proxy_creds: ProxyCredentials | None, - ): - self._reactor = reactor - self._proxy_endpoint = proxy_endpoint - self._host = host - self._port = port - self._proxy_creds = proxy_creds - - def __repr__(self) -> str: - return "" % (self._proxy_endpoint,) - - def connect(self, protocolFactory: IProtocolFactory) -> "defer.Deferred[IProtocol]": - f = HTTPProxiedClientFactory( - self._host, self._port, protocolFactory, self._proxy_creds + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "HTTPConnectProxyEndpoint has been removed. " + "Use aiohttp's native proxy support instead." ) - d = self._proxy_endpoint.connect(f) - # once the tcp socket connects successfully, we need to wait for the - # CONNECT to complete. - d.addCallback(lambda conn: f.on_connection) - return d - - -class HTTPProxiedClientFactory(protocol.ClientFactory): - """ClientFactory wrapper that triggers an HTTP proxy CONNECT on connect. - - Once the CONNECT completes, invokes the original ClientFactory to build the - HTTP Protocol object and run the rest of the connection. - - Args: - dst_host: hostname that we want to CONNECT to - dst_port: port that we want to connect to - wrapped_factory: The original Factory - proxy_creds: credentials to authenticate at proxy - """ - - def __init__( - self, - dst_host: bytes, - dst_port: int, - wrapped_factory: IProtocolFactory, - proxy_creds: ProxyCredentials | None, - ): - self.dst_host = dst_host - self.dst_port = dst_port - self.wrapped_factory = wrapped_factory - self.proxy_creds = proxy_creds - self.on_connection: "defer.Deferred[None]" = defer.Deferred() - - def startedConnecting(self, connector: IConnector) -> None: - # We expect the wrapped factory to be a ClientFactory, but the generic - # interfaces only guarantee that it implements IProtocolFactory. - if isinstance(self.wrapped_factory, ClientFactory): - return self.wrapped_factory.startedConnecting(connector) - - def buildProtocol(self, addr: IAddress) -> "HTTPConnectProtocol": - wrapped_protocol = self.wrapped_factory.buildProtocol(addr) - if wrapped_protocol is None: - raise TypeError("buildProtocol produced None instead of a Protocol") - - return HTTPConnectProtocol( - self.dst_host, - self.dst_port, - wrapped_protocol, - self.on_connection, - self.proxy_creds, - ) - - def clientConnectionFailed(self, connector: IConnector, reason: Failure) -> None: - logger.debug("Connection to proxy failed: %s", reason) - if not self.on_connection.called: - with PreserveLoggingContext(): - self.on_connection.errback(reason) - if isinstance(self.wrapped_factory, ClientFactory): - return self.wrapped_factory.clientConnectionFailed(connector, reason) - - def clientConnectionLost(self, connector: IConnector, reason: Failure) -> None: - logger.debug("Connection to proxy lost: %s", reason) - if not self.on_connection.called: - with PreserveLoggingContext(): - self.on_connection.errback(reason) - if isinstance(self.wrapped_factory, ClientFactory): - return self.wrapped_factory.clientConnectionLost(connector, reason) - - -class HTTPConnectProtocol(protocol.Protocol): - """Protocol that wraps an existing Protocol to do a CONNECT handshake at connect - - Args: - host: The original HTTP(s) hostname or IPv4 or IPv6 address literal - to put in the CONNECT request - - port: The original HTTP(s) port to put in the CONNECT request - - wrapped_protocol: the original protocol (probably HTTPChannel or - TLSMemoryBIOProtocol, but could be anything really) - - connected_deferred: a Deferred which will be callbacked with - wrapped_protocol when the CONNECT completes - - proxy_creds: credentials to authenticate at proxy - """ - - def __init__( - self, - host: bytes, - port: int, - wrapped_protocol: IProtocol, - connected_deferred: defer.Deferred, - proxy_creds: ProxyCredentials | None, - ): - self.host = host - self.port = port - self.wrapped_protocol = wrapped_protocol - self.connected_deferred = connected_deferred - self.proxy_creds = proxy_creds - - self.http_setup_client = HTTPConnectSetupClient( - self.host, self.port, self.proxy_creds - ) - self.http_setup_client.on_connected.addCallback(self.proxyConnected) - - # Set once we start connecting to the wrapped protocol - self.wrapped_connection_started = False - - def connectionMade(self) -> None: - self.http_setup_client.makeConnection(self.transport) - - def connectionLost(self, reason: Failure = connectionDone) -> None: - if self.wrapped_connection_started: - self.wrapped_protocol.connectionLost(reason) - - self.http_setup_client.connectionLost(reason) - - if not self.connected_deferred.called: - with PreserveLoggingContext(): - self.connected_deferred.errback(reason) - - def proxyConnected(self, _: Union[None, "defer.Deferred[None]"]) -> None: - self.wrapped_connection_started = True - assert self.transport is not None - self.wrapped_protocol.makeConnection(self.transport) - - with PreserveLoggingContext(): - self.connected_deferred.callback(self.wrapped_protocol) - - # Get any pending data from the http buf and forward it to the original protocol - buf = self.http_setup_client.clearLineBuffer() - if buf: - self.wrapped_protocol.dataReceived(buf) - - def dataReceived(self, data: bytes) -> None: - # if we've set up the HTTP protocol, we can send the data there - if self.wrapped_connection_started: - return self.wrapped_protocol.dataReceived(data) - - # otherwise, we must still be setting up the connection: send the data to the - # setup client - return self.http_setup_client.dataReceived(data) - - -class HTTPConnectSetupClient(http.HTTPClient): - """HTTPClient protocol to send a CONNECT message for proxies and read the response. - - Args: - host: The hostname to send in the CONNECT message - port: The port to send in the CONNECT message - proxy_creds: credentials to authenticate at proxy - """ - - def __init__( - self, - host: bytes, - port: int, - proxy_creds: ProxyCredentials | None, - ): - self.host = host - self.port = port - self.proxy_creds = proxy_creds - self.on_connected: "defer.Deferred[None]" = defer.Deferred() - - def connectionMade(self) -> None: - logger.debug("Connected to proxy, sending CONNECT") - self.sendCommand(b"CONNECT", b"%s:%d" % (self.host, self.port)) - - # Determine whether we need to set Proxy-Authorization headers - if self.proxy_creds: - # Set a Proxy-Authorization header - self.sendHeader( - b"Proxy-Authorization", - self.proxy_creds.as_proxy_authorization_value(), - ) - - self.endHeaders() - - def handleStatus(self, version: bytes, status: bytes, message: bytes) -> None: - logger.debug("Got Status: %s %s %s", status, message, version) - if status != b"200": - raise ProxyConnectError(f"Unexpected status on CONNECT: {status!s}") - - def handleEndHeaders(self) -> None: - logger.debug("End Headers") - with PreserveLoggingContext(): - self.on_connected.callback(None) - - def handleResponse(self, body: bytes) -> None: - pass diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 156c6db13e..21699a01b3 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -1,6 +1,7 @@ # # This file is licensed under the Affero General Public License (AGPL) version 3. # +# Copyright 2023 The Matrix.org Foundation C.I.C. # Copyright (C) 2023 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify @@ -18,461 +19,15 @@ # # import logging -import urllib.parse -from typing import Any, Generator, Optional -from urllib.request import ( # type: ignore[attr-defined] - proxy_bypass_environment, -) +from typing import Any -from netaddr import AddrFormatError, IPAddress, IPSet -try: - from zope.interface import implementer -except ImportError: - pass +from netaddr import AddrFormatError, IPAddress -try: - from twisted.internet import defer - from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS - from twisted.internet.interfaces import ( - IProtocol, - IProtocolFactory, - IReactorCore, - IStreamClientEndpoint, - ) - from twisted.web.client import URI, Agent, HTTPConnectionPool - from twisted.web.http_headers import Headers - from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer, IResponse -except ImportError: - pass - -from synapse.config.server import ProxyConfig -from synapse.crypto.context_factory import FederationPolicyForHTTPS -from synapse.http import proxyagent -from synapse.http.client import BlocklistingAgentWrapper, BlocklistingReactorWrapper -from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint from synapse.http.federation.srv_resolver import Server, SrvResolver -from synapse.http.federation.well_known_resolver import WellKnownResolver -from synapse.http.proxyagent import ProxyAgent -from synapse.logging.context import make_deferred_yieldable, run_in_background -from synapse.types import ISynapseReactor -from synapse.util.clock import Clock logger = logging.getLogger(__name__) -@implementer(IAgent) -class MatrixFederationAgent: - """An Agent-like thing which provides a `request` method which correctly - handles resolving matrix server names when using `matrix-federation://`. Handles - standard https URIs as normal. The `matrix-federation://` scheme is internal to - Synapse and we purposely want to avoid colliding with the `matrix://` URL scheme - which is now specced. - - Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.) - - Args: - reactor: twisted reactor to use for underlying requests - - clock: Internal `HomeServer` clock used to track delayed and looping calls. - Should be obtained from `hs.get_clock()`. - - tls_client_options_factory: - factory to use for fetching client tls options, or none to disable TLS. - - user_agent: - The user agent header to use for federation requests. - - ip_allowlist: Allowed IP addresses. - - ip_blocklist: Disallowed IP addresses. - - proxy_config: Proxy configuration to use for this agent. - - proxy_reactor: twisted reactor to use for connections to the proxy server - reactor might have some blocking applied (i.e. for DNS queries), - but we need unblocked access to the proxy. - - _srv_resolver: - SrvResolver implementation to use for looking up SRV records. None - to use a default implementation. - - _well_known_resolver: - WellKnownResolver to use to perform well-known lookups. None to use a - default implementation. - """ - - def __init__( - self, - *, - server_name: str, - reactor: ISynapseReactor, - clock: Clock, - tls_client_options_factory: FederationPolicyForHTTPS | None, - user_agent: bytes, - ip_allowlist: IPSet | None, - ip_blocklist: IPSet, - proxy_config: ProxyConfig | None = None, - _srv_resolver: SrvResolver | None = None, - _well_known_resolver: WellKnownResolver | None = None, - ): - """ - Args: - server_name: Our homeserver name (used to label metrics) (`hs.hostname`). - reactor - clock: Should be the `hs` clock from `hs.get_clock()` - tls_client_options_factory - user_agent - ip_allowlist - ip_blocklist - _srv_resolver - _well_known_resolver - """ - - # proxy_reactor is not blocklisting reactor - proxy_reactor = reactor - - # We need to use a DNS resolver which filters out blocked IP - # addresses, to prevent DNS rebinding. - reactor = BlocklistingReactorWrapper(reactor, ip_allowlist, ip_blocklist) - - self._pool = HTTPConnectionPool(reactor) - self._pool.retryAutomatically = False - self._pool.maxPersistentPerHost = 5 - self._pool.cachedConnectionTimeout = 2 * 60 - - self._agent = Agent.usingEndpointFactory( - reactor, - MatrixHostnameEndpointFactory( - reactor=reactor, - proxy_reactor=proxy_reactor, - tls_client_options_factory=tls_client_options_factory, - srv_resolver=_srv_resolver, - proxy_config=proxy_config, - ), - pool=self._pool, - ) - self.user_agent = user_agent - - if _well_known_resolver is None: - _well_known_resolver = WellKnownResolver( - server_name=server_name, - reactor=reactor, - clock=clock, - agent=BlocklistingAgentWrapper( - ProxyAgent( - reactor=reactor, - proxy_reactor=proxy_reactor, - pool=self._pool, - contextFactory=tls_client_options_factory, - proxy_config=proxy_config, - ), - ip_blocklist=ip_blocklist, - ), - user_agent=self.user_agent, - ) - - self._well_known_resolver = _well_known_resolver - - async def request( - self, - method: bytes, - uri: bytes, - headers: Headers | None = None, - bodyProducer: Optional[IBodyProducer] = None, - ) -> IResponse: - """ - Args: - method: HTTP method: GET/POST/etc - uri: Absolute URI to be retrieved - headers: - HTTP headers to send with the request, or None to send no extra headers. - bodyProducer: - An object which can generate bytes to make up the - body of this request (for example, the properly encoded contents of - a file for a file upload). Or None if the request is to have - no body. - Returns: - The response (once headers are received). - """ - # We use urlparse as that will set `port` to None if there is no - # explicit port. - parsed_uri = urllib.parse.urlparse(uri) - - # There must be a valid hostname. - assert parsed_uri.hostname - - # If this is a matrix-federation:// URI check if the server has delegated matrix - # traffic using well-known delegation. - # - # We have to do this here and not in the endpoint as we need to rewrite - # the host header with the delegated server name. - delegated_server = None - if ( - parsed_uri.scheme == b"matrix-federation" - and not _is_ip_literal(parsed_uri.hostname) - and not parsed_uri.port - ): - well_known_result = await self._well_known_resolver.get_well_known( - parsed_uri.hostname - ) - delegated_server = well_known_result.delegated_server - - if delegated_server: - # Ok, the server has delegated matrix traffic to somewhere else, so - # lets rewrite the URL to replace the server with the delegated - # server name. - uri = urllib.parse.urlunparse( - ( - parsed_uri.scheme, - delegated_server, - parsed_uri.path, - parsed_uri.params, - parsed_uri.query, - parsed_uri.fragment, - ) - ) - parsed_uri = urllib.parse.urlparse(uri) - - # We need to make sure the host header is set to the netloc of the - # server and that a user-agent is provided. - if headers is None: - request_headers = Headers() - else: - request_headers = headers.copy() - - if not request_headers.hasHeader(b"host"): - request_headers.addRawHeader(b"host", parsed_uri.netloc) - if not request_headers.hasHeader(b"user-agent"): - request_headers.addRawHeader(b"user-agent", self.user_agent) - - res = await make_deferred_yieldable( - self._agent.request(method, uri, request_headers, bodyProducer) - ) - - return res - - -@implementer(IAgentEndpointFactory) -class MatrixHostnameEndpointFactory: - """Factory for MatrixHostnameEndpoint for parsing to an Agent.""" - - def __init__( - self, - *, - reactor: IReactorCore, - proxy_reactor: IReactorCore, - tls_client_options_factory: FederationPolicyForHTTPS | None, - srv_resolver: SrvResolver | None, - proxy_config: ProxyConfig | None, - ): - self._reactor = reactor - self._proxy_reactor = proxy_reactor - self._tls_client_options_factory = tls_client_options_factory - self._proxy_config = proxy_config - - if srv_resolver is None: - srv_resolver = SrvResolver() - - self._srv_resolver = srv_resolver - - def endpointForURI(self, parsed_uri: URI) -> "MatrixHostnameEndpoint": - return MatrixHostnameEndpoint( - reactor=self._reactor, - proxy_reactor=self._proxy_reactor, - tls_client_options_factory=self._tls_client_options_factory, - srv_resolver=self._srv_resolver, - proxy_config=self._proxy_config, - parsed_uri=parsed_uri, - ) - - -@implementer(IStreamClientEndpoint) -class MatrixHostnameEndpoint: - """An endpoint that resolves matrix-federation:// URLs using Matrix server name - resolution (i.e. via SRV). Does not check for well-known delegation. - - Args: - reactor: twisted reactor to use for underlying requests - proxy_reactor: twisted reactor to use for connections to the proxy server. - 'reactor' might have some blocking applied (i.e. for DNS queries), - but we need unblocked access to the proxy. - tls_client_options_factory: - factory to use for fetching client tls options, or none to disable TLS. - srv_resolver: The SRV resolver to use - proxy_config: Proxy configuration to use for this agent. - parsed_uri: The parsed URI that we're wanting to connect to. - - Raises: - ValueError if the environment variables contain an invalid proxy specification. - RuntimeError if no tls_options_factory is given for a https connection - """ - - def __init__( - self, - *, - reactor: IReactorCore, - proxy_reactor: IReactorCore, - tls_client_options_factory: FederationPolicyForHTTPS | None, - srv_resolver: SrvResolver, - proxy_config: ProxyConfig | None, - parsed_uri: URI, - ): - self._reactor = reactor - self._parsed_uri = parsed_uri - self.proxy_config = proxy_config - - # http_proxy is not needed because federation is always over TLS - - # endpoint and credentials to use to connect to the outbound https proxy, if any. - ( - self._https_proxy_endpoint, - self._https_proxy_creds, - ) = proxyagent.http_proxy_endpoint( - self.proxy_config.https_proxy.encode() - if self.proxy_config and self.proxy_config.https_proxy - else None, - proxy_reactor, - tls_client_options_factory, - ) - - # set up the TLS connection params - # - # XXX disabling TLS is really only supported here for the benefit of the - # unit tests. We should make the UTs cope with TLS rather than having to make - # the code support the unit tests. - - if tls_client_options_factory is None: - self._tls_options = None - else: - self._tls_options = tls_client_options_factory.get_options( - self._parsed_uri.host - ) - - self._srv_resolver = srv_resolver - - def connect( - self, protocol_factory: IProtocolFactory - ) -> Any: - """Implements IStreamClientEndpoint interface""" - - return run_in_background(self._do_connect, protocol_factory) - - async def _do_connect(self, protocol_factory: IProtocolFactory) -> IProtocol: - first_exception = None - - server_list = await self._resolve_server() - - for server in server_list: - host = server.host - port = server.port - - should_skip_proxy = False - if self.proxy_config is not None: - should_skip_proxy = proxy_bypass_environment( - host.decode(), - proxies=self.proxy_config.get_proxies_dictionary(), - ) - - endpoint: IStreamClientEndpoint - try: - if self._https_proxy_endpoint and not should_skip_proxy: - logger.debug( - "Connecting to %s:%i via %s", - host.decode("ascii"), - port, - self._https_proxy_endpoint, - ) - endpoint = HTTPConnectProxyEndpoint( - self._reactor, - self._https_proxy_endpoint, - host, - port, - proxy_creds=self._https_proxy_creds, - ) - else: - logger.debug("Connecting to %s:%i", host.decode("ascii"), port) - # not using a proxy - endpoint = HostnameEndpoint(self._reactor, host, port) - if self._tls_options: - endpoint = wrapClientTLS(self._tls_options, endpoint) - result = await make_deferred_yieldable( - endpoint.connect(protocol_factory) - ) - - return result - except Exception as e: - logger.info( - "Failed to connect to %s:%i: %s", host.decode("ascii"), port, e - ) - if not first_exception: - first_exception = e - - # We return the first failure because that's probably the most interesting. - if first_exception: - raise first_exception - - # This shouldn't happen as we should always have at least one host/port - # to try and if that doesn't work then we'll have an exception. - raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc,)) - - async def _resolve_server(self) -> list[Server]: - """Resolves the server name to a list of hosts and ports to attempt to - connect to. - """ - - if self._parsed_uri.scheme != b"matrix-federation": - return [Server(host=self._parsed_uri.host, port=self._parsed_uri.port)] - - # Note: We don't do well-known lookup as that needs to have happened - # before now, due to needing to rewrite the Host header of the HTTP - # request. - - # We reparse the URI so that defaultPort is -1 rather than 80 - parsed_uri = urllib.parse.urlparse(self._parsed_uri.toBytes()) - - host = parsed_uri.hostname - port = parsed_uri.port - - # If there is an explicit port or the host is an IP address we bypass - # SRV lookups and just use the given host/port. - if port or _is_ip_literal(host): - return [Server(host, port or 8448)] - - # Check _matrix-fed._tcp SRV record. - logger.debug("Looking up SRV record for %s", host.decode(errors="replace")) - server_list = await self._srv_resolver.resolve_service( - b"_matrix-fed._tcp." + host - ) - - if server_list: - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "Got %s from SRV lookup for %s", - ", ".join(map(str, server_list)), - host.decode(errors="replace"), - ) - return server_list - - # No _matrix-fed._tcp SRV record, fallback to legacy _matrix._tcp SRV record. - logger.debug( - "Looking up deprecated SRV record for %s", host.decode(errors="replace") - ) - server_list = await self._srv_resolver.resolve_service(b"_matrix._tcp." + host) - - if server_list: - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "Got %s from deprecated SRV lookup for %s", - ", ".join(map(str, server_list)), - host.decode(errors="replace"), - ) - return server_list - - # No SRV records, so we fallback to host and 8448 - logger.debug("No SRV records for %s", host.decode(errors="replace")) - return [Server(host, 8448)] - - def _is_ip_literal(host: bytes) -> bool: """Test if the given host name is either an IPv4 or IPv6 literal. @@ -490,3 +45,21 @@ def _is_ip_literal(host: bytes) -> bool: return True except AddrFormatError: return False + + +# The Twisted-based MatrixFederationAgent, MatrixHostnameEndpointFactory, and +# MatrixHostnameEndpoint classes have been removed as part of the Twisted->asyncio +# migration. Federation HTTP requests are now handled directly by +# MatrixFederationHttpClient using aiohttp. +# +# For backward compatibility of imports, we provide a stub. +class MatrixFederationAgent: + """Stub: The Twisted MatrixFederationAgent has been removed. + + Federation requests are now handled by MatrixFederationHttpClient using aiohttp. + """ + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "MatrixFederationAgent has been removed. " + "Federation requests use aiohttp directly via MatrixFederationHttpClient." + ) diff --git a/synapse/http/federation/srv_resolver.py b/synapse/http/federation/srv_resolver.py index e7641b480a..a2d02e8be9 100644 --- a/synapse/http/federation/srv_resolver.py +++ b/synapse/http/federation/srv_resolver.py @@ -19,6 +19,7 @@ # # +import asyncio import logging import random import time @@ -26,17 +27,18 @@ from typing import Any, Callable import attr -try: - from twisted.names import client -except ImportError: - client = None # type: ignore[assignment] - -from synapse.logging.context import make_deferred_yieldable - logger = logging.getLogger(__name__) SERVER_CACHE: dict[bytes, list["Server"]] = {} +# Try to import DNS resolution libraries in order of preference +_dns_resolver: Any = None +try: + import dns.resolver + _dns_resolver = "dnspython" +except ImportError: + pass + @attr.s(auto_attribs=True, slots=True, frozen=True) class Server: @@ -103,24 +105,22 @@ def _sort_server_list(server_list: list[Server]) -> list[Server]: class SrvResolver: - """Interface to the dns client to do SRV lookups, with result caching. + """Interface to do SRV lookups, with result caching. - The default resolver in twisted.names doesn't do any caching (it has a CacheResolver, - but the cache never gets populated), so we add our own caching layer here. + Uses dnspython for DNS resolution if available, otherwise falls back + to running `dig` via subprocess. Args: - dns_client (twisted.internet.interfaces.IResolver): twisted resolver impl cache: cache object get_time: clock implementation. Should return seconds since the epoch """ def __init__( self, - dns_client: Any = client, + dns_client: Any = None, cache: dict[bytes, list[Server]] = SERVER_CACHE, get_time: Callable[[], float] = time.time, ): - self._dns_client = dns_client self._cache = cache self._get_time = get_time @@ -145,19 +145,9 @@ class SrvResolver: return _sort_server_list(servers) try: - answers, _, _ = await make_deferred_yieldable( - self._dns_client.lookupService(service_name) - ) - except DNSNameError: - # TODO: cache this. We can get the SOA out of the exception, and use - # the negative-TTL value. - return [] - except DNSNotImplementedError: - # For .onion homeservers this is unavailable, just fallback to host:8448 - return [] - except DomainError as e: - # We failed to resolve the name (other than a NameError) - # Try something in the cache, else rereaise + answers = await self._do_lookup(service_name) + except Exception as e: + # Try something in the cache, else reraise cache_entry = self._cache.get(service_name, None) if cache_entry: logger.warning( @@ -167,31 +157,102 @@ class SrvResolver: else: raise e - if ( - len(answers) == 1 - and answers[0].type == dns.SRV - and answers[0].payload - and answers[0].payload.target == dns.Name(b".") - ): - raise ConnectError(f"Service {service_name!r} unavailable") + if not answers: + return [] servers = [] - for answer in answers: - if answer.type != dns.SRV or not answer.payload: - continue - - payload = answer.payload - servers.append( Server( - host=payload.target.name, - port=payload.port, - priority=payload.priority, - weight=payload.weight, - expires=now + answer.ttl, + host=answer["host"], + port=answer["port"], + priority=answer.get("priority", 0), + weight=answer.get("weight", 0), + expires=now + answer.get("ttl", 300), ) ) self._cache[service_name] = list(servers) return _sort_server_list(servers) + + async def _do_lookup(self, service_name: bytes) -> list[dict]: + """Perform the actual DNS SRV lookup. + + Returns a list of dicts with keys: host, port, priority, weight, ttl + """ + name_str = service_name.decode("ascii") + + if _dns_resolver == "dnspython": + return await self._lookup_dnspython(name_str) + else: + return await self._lookup_subprocess(name_str) + + async def _lookup_dnspython(self, name: str) -> list[dict]: + """Use dnspython to resolve SRV records.""" + loop = asyncio.get_event_loop() + + def _resolve() -> list[dict]: + try: + answers = dns.resolver.resolve(name, "SRV") + except dns.resolver.NXDOMAIN: + return [] + except dns.resolver.NoAnswer: + return [] + except dns.resolver.NoNameservers: + return [] + + results = [] + for rdata in answers: + # Check for the "no service" response (target is ".") + target = rdata.target.to_text() + if target == ".": + return [] + + results.append({ + "host": rdata.target.to_text(omit_final_dot=True).encode("ascii"), + "port": rdata.port, + "priority": rdata.priority, + "weight": rdata.weight, + "ttl": answers.rrset.ttl if answers.rrset else 300, + }) + return results + + return await loop.run_in_executor(None, _resolve) + + async def _lookup_subprocess(self, name: str) -> list[dict]: + """Fallback: use `dig` subprocess to resolve SRV records.""" + try: + proc = await asyncio.create_subprocess_exec( + "dig", "+short", "SRV", name, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=10) + except (FileNotFoundError, asyncio.TimeoutError): + logger.warning("Failed to run 'dig' for SRV lookup of %s", name) + return [] + + results = [] + for line in stdout.decode("ascii", errors="replace").strip().split("\n"): + line = line.strip() + if not line: + continue + parts = line.split() + if len(parts) >= 4: + try: + priority = int(parts[0]) + weight = int(parts[1]) + port = int(parts[2]) + target = parts[3].rstrip(".") + if target == ".": + return [] + results.append({ + "host": target.encode("ascii"), + "port": port, + "priority": priority, + "weight": weight, + "ttl": 300, # dig +short doesn't give TTL + }) + except (ValueError, IndexError): + continue + return results diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index 855682b65e..a3298f5937 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -19,42 +19,8 @@ # # -import json import logging -import urllib.parse -from typing import TYPE_CHECKING, Any, Optional, cast - -from synapse.api.errors import Codes, InvalidProxyCredentialsError -from synapse.http import QuieterFileBodyProducer -from synapse.http.server import _AsyncResource -from synapse.logging.context import make_deferred_yieldable, run_in_background -from synapse.types import ISynapseReactor -from synapse.util.async_helpers import timeout_deferred - -try: - from twisted.python import failure - from twisted.python.failure import Failure - from twisted.internet import protocol - from twisted.internet.protocol import connectionDone - from twisted.internet.interfaces import ITCPTransport - from twisted.web.resource import IResource - from twisted.web.client import ResponseDone - from twisted.web.http_headers import Headers - from twisted.web.server import Site -except ImportError: - failure = None # type: ignore[assignment] - Failure = BaseException # type: ignore[assignment,misc] - protocol = None # type: ignore[assignment] - connectionDone = None # type: ignore[assignment] - ITCPTransport = None # type: ignore[assignment] - IResource = None # type: ignore[assignment] - ResponseDone = None # type: ignore[assignment] - Headers = dict # type: ignore[assignment,misc] - Site = object # type: ignore[assignment,misc] - -if TYPE_CHECKING: - from synapse.http.site import SynapseRequest - from synapse.server import HomeServer +from typing import Any logger = logging.getLogger(__name__) @@ -106,195 +72,22 @@ def parse_connection_header_value( return extra_headers_to_remove -class ProxyResource(_AsyncResource): - """ - A stub resource that proxies any requests with a `matrix-federation://` scheme - through the given `federation_agent` to the remote homeserver and ferries back the - info. - """ - - isLeaf = True - - def __init__(self, reactor: ISynapseReactor, hs: "HomeServer"): - super().__init__(hs.get_clock(), True) - - self.reactor = reactor - self.agent = hs.get_federation_http_client().agent - - self._proxy_authorization_secret = hs.config.worker.worker_replication_secret - - def _check_auth(self, request: Request) -> None: - # The `matrix-federation://` proxy functionality can only be used with auth. - # Protect homserver admins forgetting to configure a secret. - assert self._proxy_authorization_secret is not None - - # Get the authorization header. - auth_headers = request.requestHeaders.getRawHeaders(b"Proxy-Authorization") - - if not auth_headers: - raise InvalidProxyCredentialsError( - "Missing Proxy-Authorization header.", Codes.MISSING_TOKEN - ) - if len(auth_headers) > 1: - raise InvalidProxyCredentialsError( - "Too many Proxy-Authorization headers.", Codes.UNAUTHORIZED - ) - parts = auth_headers[0].split(b" ") - if parts[0] == b"Bearer" and len(parts) == 2: - received_secret = parts[1].decode("ascii") - if self._proxy_authorization_secret == received_secret: - # Success! - return - - raise InvalidProxyCredentialsError( - "Invalid Proxy-Authorization header.", Codes.UNAUTHORIZED +# The Twisted-based ProxyResource, _ProxyResponseBody, and ProxySite classes have been +# removed as part of the Twisted->asyncio migration. Federation proxy functionality +# would need to be reimplemented using aiohttp if needed. +# +# For backward compatibility of imports, we provide stubs. +class ProxyResource: + """Stub: The Twisted ProxyResource has been removed.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "ProxyResource has been removed as part of the Twisted->asyncio migration." ) - async def _async_render(self, request: "SynapseRequest") -> tuple[int, Any]: - uri = urllib.parse.urlparse(request.uri) - assert uri.scheme == b"matrix-federation" - # Check the authorization headers before handling the request. - self._check_auth(request) - - headers = Headers() - for header_name in (b"User-Agent", b"Authorization", b"Content-Type"): - header_value = request.getHeader(header_name) - if header_value: - headers.addRawHeader(header_name, header_value) - - request_deferred = run_in_background( - self.agent.request, - request.method, - request.uri, - headers=headers, - bodyProducer=QuieterFileBodyProducer(request.content), +class ProxySite: + """Stub: The Twisted ProxySite has been removed.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "ProxySite has been removed as part of the Twisted->asyncio migration." ) - request_deferred = timeout_deferred( - deferred=request_deferred, - # This should be set longer than the timeout in `MatrixFederationHttpClient` - # so that it has enough time to complete and pass us the data before we give - # up. - timeout=90, - clock=self._clock, - ) - - response = await make_deferred_yieldable(request_deferred) - - return response.code, response - - def _send_response( - self, - request: "SynapseRequest", - code: int, - response_object: Any, - ) -> None: - response = cast(IResponse, response_object) - response_headers = cast(Headers, response.headers) - - request.setResponseCode(code) - - # The `Connection` header also defines which headers should not be copied over. - connection_header = response_headers.getRawHeaders(b"connection") - extra_headers_to_remove_lowercase = parse_connection_header_value( - connection_header[0] if connection_header else None - ) - - # Copy headers. - for k, v in response_headers.getAllRawHeaders(): - # Do not copy over any hop-by-hop headers. These are meant to only be - # consumed by the immediate recipient and not be forwarded on. - header_key_lowercase = k.decode("ascii").lower() - if ( - header_key_lowercase in HOP_BY_HOP_HEADERS_LOWERCASE - or header_key_lowercase in extra_headers_to_remove_lowercase - ): - continue - - request.responseHeaders.setRawHeaders(k, v) - - response.deliverBody(_ProxyResponseBody(request)) - - def _send_error_response( - self, - f: failure.Failure, - request: "SynapseRequest", - ) -> None: - if isinstance(f.value, InvalidProxyCredentialsError): - error_response_code = f.value.code - error_response_json = {"errcode": f.value.errcode, "err": f.value.msg} - else: - error_response_code = 502 - error_response_json = { - "errcode": Codes.UNKNOWN, - "err": "ProxyResource: Error when proxying request: %s %s -> %s" - % ( - request.method.decode("ascii"), - request.uri.decode("ascii"), - f, - ), - } - - request.setResponseCode(error_response_code) - request.setHeader(b"Content-Type", b"application/json") - request.write((json.dumps(error_response_json)).encode()) - request.finish() - - -class _ProxyResponseBody(protocol.Protocol): - """ - A protocol that proxies the given remote response data back out to the given local - request. - """ - - transport: Optional[ITCPTransport] = None - - def __init__(self, request: "SynapseRequest") -> None: - self._request = request - - def dataReceived(self, data: bytes) -> None: - # Avoid sending response data to the local request that already disconnected - if self._request._disconnected and self.transport is not None: - # Close the connection (forcefully) since all the data will get - # discarded anyway. - self.transport.abortConnection() - return - - self._request.write(data) - - def connectionLost(self, reason: Failure = connectionDone) -> None: - # If the local request is already finished (successfully or failed), don't - # worry about sending anything back. - if self._request.finished: - return - - if reason.check(ResponseDone): - self._request.finish() - else: - # Abort the underlying request since our remote request also failed. - if self._request.channel: - self._request.channel.forceAbortClient() - - -class ProxySite(Site): - """ - Proxies any requests with a `matrix-federation://` scheme through the given - `federation_agent`. Otherwise, behaves like a normal `Site`. - """ - - def __init__( - self, - resource: IResource, - reactor: ISynapseReactor, - hs: "HomeServer", - ): - super().__init__(resource, reactor=reactor) - - self._proxy_resource = ProxyResource(reactor, hs=hs) - - def getResourceFor(self, request: "SynapseRequest") -> IResource: - uri = urllib.parse.urlparse(request.uri) - if uri.scheme == b"matrix-federation": - return self._proxy_resource - - return super().getResourceFor(request) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index d80470b862..f5b765d4ea 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -19,403 +19,50 @@ # # import logging -import random import re -from typing import Any, Collection, Optional, Sequence, cast +from typing import Any, Optional from urllib.parse import urlparse -from urllib.request import ( # type: ignore[attr-defined] - proxy_bypass_environment, -) -try: - from zope.interface import implementer -except ImportError: - pass - -try: - from twisted.internet import defer - from twisted.internet.endpoints import ( - HostnameEndpoint, - UNIXClientEndpoint, - wrapClientTLS, - ) - from twisted.internet.interfaces import ( - IProtocol, - IProtocolFactory, - IReactorCore, - IReactorTime, - IStreamClientEndpoint, - ) - from twisted.python.failure import Failure - from twisted.web.client import ( - URI, - BrowserLikePolicyForHTTPS, - HTTPConnectionPool, - _AgentBase, - ) - from twisted.web.error import SchemeNotSupported - from twisted.web.http_headers import Headers - from twisted.web.iweb import IAgent, IBodyProducer, IPolicyForHTTPS, IResponse -except ImportError: - pass - -from synapse.config.server import ProxyConfig -from synapse.config.workers import ( - InstanceLocationConfig, - InstanceTcpLocationConfig, - InstanceUnixLocationConfig, -) -from synapse.http import redact_uri from synapse.http.connectproxyclient import ( BasicProxyCredentials, - HTTPConnectProxyEndpoint, ProxyCredentials, ) -from synapse.logging.context import run_in_background logger = logging.getLogger(__name__) _VALID_URI = re.compile(rb"\A[\x21-\x7e]+\Z") -@implementer(IAgent) -class ProxyAgent(_AgentBase): - """An Agent implementation which will use an HTTP proxy if one was requested - - Args: - reactor: twisted reactor to place outgoing - connections. - - proxy_reactor: twisted reactor to use for connections to the proxy server - reactor might have some blocking applied (i.e. for DNS queries), - but we need unblocked access to the proxy. - - contextFactory: A factory for TLS contexts, to control the - verification parameters of OpenSSL. The default is to use a - `BrowserLikePolicyForHTTPS`, so unless you have special - requirements you can leave this as-is. - - connectTimeout: The amount of time that this Agent will wait - for the peer to accept a connection, in seconds. If 'None', - HostnameEndpoint's default (30s) will be used. - This is used for connections to both proxies and destination servers. - - bindAddress: The local address for client sockets to bind to. - - pool: connection pool to be used. If None, a - non-persistent pool instance will be created. - - proxy_config: Proxy configuration to use for this agent. - - federation_proxy_locations: An optional list of locations to proxy outbound federation - traffic through (only requests that use the `matrix-federation://` scheme - will be proxied). - - federation_proxy_credentials: Required if `federation_proxy_locations` is set. The - credentials to use when proxying outbound federation traffic through another - worker. - - Raises: - ValueError if use_proxy is set and the environment variables - contain an invalid proxy specification. - RuntimeError if no tls_options_factory is given for a https connection - """ - - def __init__( - self, - *, - reactor: IReactorCore, - proxy_reactor: Optional[IReactorCore] = None, - contextFactory: Optional[IPolicyForHTTPS] = None, - connectTimeout: float | None = None, - bindAddress: bytes | None = None, - pool: HTTPConnectionPool | None = None, - proxy_config: ProxyConfig | None = None, - federation_proxy_locations: Collection[InstanceLocationConfig] = (), - federation_proxy_credentials: ProxyCredentials | None = None, - ): - contextFactory = contextFactory or BrowserLikePolicyForHTTPS() - - # `_AgentBase` expects an `IReactorTime` provider. `IReactorCore` - # extends `IReactorTime`, so this cast is safe. - _AgentBase.__init__(self, cast(IReactorTime, reactor), pool) - - if proxy_reactor is None: - self.proxy_reactor = reactor - else: - self.proxy_reactor = proxy_reactor - - self._endpoint_kwargs: dict[str, Any] = {} - if connectTimeout is not None: - self._endpoint_kwargs["timeout"] = connectTimeout - if bindAddress is not None: - self._endpoint_kwargs["bindAddress"] = bindAddress - - self.proxy_config = proxy_config - if self.proxy_config is not None: - logger.debug( - "Using proxy settings: http_proxy=%s, https_proxy=%s, no_proxy=%s", - self.proxy_config.http_proxy, - self.proxy_config.https_proxy, - self.proxy_config.no_proxy_hosts, - ) - - self.http_proxy_endpoint, self.http_proxy_creds = http_proxy_endpoint( - self.proxy_config.http_proxy.encode() - if self.proxy_config and self.proxy_config.http_proxy - else None, - self.proxy_reactor, - contextFactory, - **self._endpoint_kwargs, - ) - - self.https_proxy_endpoint, self.https_proxy_creds = http_proxy_endpoint( - self.proxy_config.https_proxy.encode() - if self.proxy_config and self.proxy_config.https_proxy - else None, - self.proxy_reactor, - contextFactory, - **self._endpoint_kwargs, - ) - - self._policy_for_https = contextFactory - self._reactor = cast(IReactorTime, reactor) - - self._federation_proxy_endpoint: Optional[IStreamClientEndpoint] = None - self._federation_proxy_credentials: ProxyCredentials | None = None - if federation_proxy_locations: - assert federation_proxy_credentials is not None, ( - "`federation_proxy_credentials` are required when using `federation_proxy_locations`" - ) - - endpoints: list[IStreamClientEndpoint] = [] - for federation_proxy_location in federation_proxy_locations: - endpoint: IStreamClientEndpoint - if isinstance(federation_proxy_location, InstanceTcpLocationConfig): - endpoint = HostnameEndpoint( - self.proxy_reactor, - federation_proxy_location.host, - federation_proxy_location.port, - ) - if federation_proxy_location.tls: - tls_connection_creator = ( - self._policy_for_https.creatorForNetloc( - federation_proxy_location.host.encode("utf-8"), - federation_proxy_location.port, - ) - ) - endpoint = wrapClientTLS(tls_connection_creator, endpoint) - - elif isinstance(federation_proxy_location, InstanceUnixLocationConfig): - endpoint = UNIXClientEndpoint( - self.proxy_reactor, federation_proxy_location.path - ) - - else: - # It is supremely unlikely we ever hit this - raise SchemeNotSupported( - f"Unknown type of Endpoint requested, check {federation_proxy_location}" - ) - - endpoints.append(endpoint) - - self._federation_proxy_endpoint = _RandomSampleEndpoints(endpoints) - self._federation_proxy_credentials = federation_proxy_credentials - - def request( - self, - method: bytes, - uri: bytes, - headers: Headers | None = None, - bodyProducer: Optional[IBodyProducer] = None, - ) -> Any: - """ - Issue a request to the server indicated by the given uri. - - Supports `http` and `https` schemes. - - An existing connection from the connection pool may be used or a new one may be - created. - - See also: twisted.web.iweb.IAgent.request - - Args: - method: The request method to use, such as `GET`, `POST`, etc - - uri: The location of the resource to request. - - headers: Extra headers to send with the request - - bodyProducer: An object which can generate bytes to make up the body of - this request (for example, the properly encoded contents of a file for - a file upload). Or, None if the request is to have no body. - - Returns: - A deferred which completes when the header of the response has - been received (regardless of the response status code). - - Can fail with: - SchemeNotSupported: if the uri is not http or https - - twisted.internet.error.TimeoutError if the server we are connecting - to (proxy or destination) does not accept a connection before - connectTimeout. - - ... other things too. - """ - uri = uri.strip() - if not _VALID_URI.match(uri): - raise ValueError(f"Invalid URI {uri!r}") - - parsed_uri = URI.fromBytes(uri) - pool_key: tuple[bytes, bytes, int] = ( - parsed_uri.scheme, - parsed_uri.host, - parsed_uri.port, - ) - request_path = parsed_uri.originForm - - should_skip_proxy = False - if self.proxy_config is not None: - should_skip_proxy = proxy_bypass_environment( - parsed_uri.host.decode(), - proxies=self.proxy_config.get_proxies_dictionary(), - ) - - if ( - parsed_uri.scheme == b"http" - and self.http_proxy_endpoint - and not should_skip_proxy - ): - # Determine whether we need to set Proxy-Authorization headers - if self.http_proxy_creds: - # Set a Proxy-Authorization header - if headers is None: - headers = Headers() - headers.addRawHeader( - b"Proxy-Authorization", - self.http_proxy_creds.as_proxy_authorization_value(), - ) - # Cache *all* connections under the same key, since we are only - # connecting to a single destination, the proxy: - pool_key = (b"http-proxy", b"", 0) - endpoint = self.http_proxy_endpoint - request_path = uri - elif ( - parsed_uri.scheme == b"https" - and self.https_proxy_endpoint - and not should_skip_proxy - ): - endpoint = HTTPConnectProxyEndpoint( - self.proxy_reactor, - self.https_proxy_endpoint, - parsed_uri.host, - parsed_uri.port, - self.https_proxy_creds, - ) - elif ( - parsed_uri.scheme == b"matrix-federation" - and self._federation_proxy_endpoint - ): - assert self._federation_proxy_credentials is not None, ( - "`federation_proxy_credentials` are required when using `federation_proxy_locations`" - ) - - # Set a Proxy-Authorization header - if headers is None: - headers = Headers() - # We always need authentication for the outbound federation proxy - headers.addRawHeader( - b"Proxy-Authorization", - self._federation_proxy_credentials.as_proxy_authorization_value(), - ) - - endpoint = self._federation_proxy_endpoint - request_path = uri - else: - # not using a proxy - endpoint = HostnameEndpoint( - self._reactor, parsed_uri.host, parsed_uri.port, **self._endpoint_kwargs - ) - - logger.debug( - "Requesting %s via %s", - redact_uri(uri.decode("ascii", errors="replace")), - endpoint, - ) - - if parsed_uri.scheme == b"https": - tls_connection_creator = self._policy_for_https.creatorForNetloc( - parsed_uri.host, parsed_uri.port - ) - endpoint = wrapClientTLS(tls_connection_creator, endpoint) - elif parsed_uri.scheme == b"http": - pass - elif ( - parsed_uri.scheme == b"matrix-federation" - and self._federation_proxy_endpoint - ): - pass - else: - raise SchemeNotSupported("Unsupported scheme: %r" % (parsed_uri.scheme,)) - - return self._requestWithEndpoint( - pool_key, endpoint, method, parsed_uri, headers, bodyProducer, request_path - ) - - def http_proxy_endpoint( proxy: bytes | None, - reactor: IReactorCore, - tls_options_factory: Optional[IPolicyForHTTPS], + reactor: Any = None, + tls_options_factory: Any = None, timeout: float = 30, - bindAddress: bytes | str | tuple[bytes | str, int] | None = None, + bindAddress: Any = None, attemptDelay: float | None = None, -) -> tuple[Optional[IStreamClientEndpoint], ProxyCredentials | None]: - """Parses an http proxy setting and returns an endpoint for the proxy +) -> tuple[Any, ProxyCredentials | None]: + """Parses an http proxy setting and returns endpoint info for the proxy. + + This is a compatibility shim. The Twisted endpoint-based proxy support has been + removed. This function now only parses the proxy URL and returns credentials. Args: proxy: the proxy setting in the form: [scheme://][:@][:] - This currently supports http:// and https:// proxies. - A hostname without scheme is assumed to be http. - - reactor: reactor to be used to connect to the proxy - - tls_options_factory: the TLS options to use when connecting through a https proxy - - kwargs: other args to be passed to HostnameEndpoint + reactor: unused (kept for backward compatibility) + tls_options_factory: unused (kept for backward compatibility) + kwargs: other args (unused) Returns: - a tuple of - endpoint to use to connect to the proxy, or None - ProxyCredentials or if no credentials were found, or None - - Raise: - ValueError if proxy has no hostname or unsupported scheme. - RuntimeError if no tls_options_factory is given for a https connection + A tuple of (None, ProxyCredentials or None). The first element is always None + since Twisted endpoints are no longer used. """ if proxy is None: return None, None - # Note: urlsplit/urlparse cannot be used here as that does not work (for Python - # 3.9+) on scheme-less proxies, e.g. host:port. scheme, host, port, credentials = parse_proxy(proxy) - - proxy_endpoint = HostnameEndpoint( - reactor, host, port, timeout, bindAddress, attemptDelay - ) - - if scheme == b"https": - if tls_options_factory: - tls_options = tls_options_factory.creatorForNetloc(host, port) - wrapped_proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint) - return wrapped_proxy_endpoint, credentials - else: - raise RuntimeError( - f"No TLS options for a https connection via proxy {proxy!s}" - ) - - return proxy_endpoint, credentials + # We return None for the endpoint since aiohttp handles proxying natively. + # The credentials are still useful for proxy authentication headers. + return None, credentials def parse_proxy( @@ -428,7 +75,6 @@ def parse_proxy( has a hostname (otherwise it is not useful to us when trying to find a proxy) and asserts that the URL has a scheme we support. - Args: proxy: The proxy connection string. Must be in the form '[scheme://][:@]host[:port]'. default_scheme: The default scheme to return if one is not found in `proxy`. Defaults to http @@ -465,35 +111,18 @@ def parse_proxy( return url.scheme, url.hostname, url.port or default_port, credentials -@implementer(IStreamClientEndpoint) -class _RandomSampleEndpoints: - """An endpoint that randomly iterates through a given list of endpoints at - each connection attempt. +# The Twisted-based ProxyAgent class has been removed as part of the Twisted->asyncio +# migration. Proxy support is now handled by aiohttp's native proxy support in +# NativeSimpleHttpClient. +# +# For backward compatibility of imports, we provide a stub. +class ProxyAgent: + """Stub: The Twisted ProxyAgent has been removed. + + Proxy support is now handled by aiohttp natively in NativeSimpleHttpClient. """ - - def __init__( - self, - endpoints: Sequence[IStreamClientEndpoint], - ) -> None: - assert endpoints - self._endpoints = endpoints - - def __repr__(self) -> str: - return f"<_RandomSampleEndpoints endpoints={self._endpoints}>" - - def connect( - self, protocol_factory: IProtocolFactory - ) -> Any: - """Implements IStreamClientEndpoint interface""" - - return run_in_background(self._do_connect, protocol_factory) - - async def _do_connect(self, protocol_factory: IProtocolFactory) -> IProtocol: - failures: list[Failure] = [] - for endpoint in random.sample(self._endpoints, k=len(self._endpoints)): - try: - return await endpoint.connect(protocol_factory) - except Exception: - failures.append(Failure()) - - failures.pop().raiseException() + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "ProxyAgent has been removed. Use NativeSimpleHttpClient with " + "aiohttp's native proxy support instead." + ) diff --git a/synapse/http/replicationagent.py b/synapse/http/replicationagent.py index 22f4a0be7d..10f491eaca 100644 --- a/synapse/http/replicationagent.py +++ b/synapse/http/replicationagent.py @@ -20,190 +20,35 @@ # import logging -from typing import Optional - -try: - from zope.interface import implementer -except ImportError: - pass - -try: - from twisted.internet import defer - from twisted.internet.endpoints import ( - HostnameEndpoint, - UNIXClientEndpoint, - wrapClientTLS, - ) - from twisted.internet.interfaces import IStreamClientEndpoint - from twisted.python.failure import Failure - from twisted.web.client import URI, HTTPConnectionPool, _AgentBase - from twisted.web.error import SchemeNotSupported - from twisted.web.http_headers import Headers - from twisted.web.iweb import ( - IAgent, - IAgentEndpointFactory, - IBodyProducer, - IPolicyForHTTPS, - IResponse, - ) -except ImportError: - pass +from typing import Any from synapse.config.workers import ( InstanceLocationConfig, - InstanceTcpLocationConfig, - InstanceUnixLocationConfig, ) -from synapse.types import ISynapseReactor logger = logging.getLogger(__name__) -@implementer(IAgentEndpointFactory) +# The Twisted-based ReplicationEndpointFactory and ReplicationAgent classes have been +# removed as part of the Twisted->asyncio migration. Replication HTTP requests are now +# handled by NativeReplicationClient using aiohttp. +# +# For backward compatibility of imports, we provide stubs. + + class ReplicationEndpointFactory: - """Connect to a given TCP or UNIX socket""" - - def __init__( - self, - reactor: ISynapseReactor, - instance_map: dict[str, InstanceLocationConfig], - context_factory: IPolicyForHTTPS, - ) -> None: - self.reactor = reactor - self.instance_map = instance_map - self.context_factory = context_factory - - def endpointForURI(self, uri: URI) -> IStreamClientEndpoint: - """ - This part of the factory decides what kind of endpoint is being connected to. - - Args: - uri: The pre-parsed URI object containing all the uri data - - Returns: The correct client endpoint object - """ - # The given URI has a special scheme and includes the worker name. The - # actual connection details are pulled from the instance map. - worker_name = uri.netloc.decode("utf-8") - location_config = self.instance_map[worker_name] - scheme = location_config.scheme() - - if isinstance(location_config, InstanceTcpLocationConfig): - endpoint = HostnameEndpoint( - self.reactor, - location_config.host, - location_config.port, - ) - if scheme == "https": - wrapped_endpoint = wrapClientTLS( - # The 'port' argument below isn't actually used by the function - self.context_factory.creatorForNetloc( - location_config.host.encode("utf-8"), - location_config.port, - ), - endpoint, - ) - return wrapped_endpoint - - return endpoint - elif isinstance(location_config, InstanceUnixLocationConfig): - return UNIXClientEndpoint(self.reactor, location_config.path) - else: - raise SchemeNotSupported(f"Unsupported scheme: {scheme}") - - -@implementer(IAgent) -class ReplicationAgent(_AgentBase): - """ - Client for connecting to replication endpoints via HTTP and HTTPS. - - Much of this code is copied from Twisted's twisted.web.client.Agent. - """ - - def __init__( - self, - reactor: ISynapseReactor, - instance_map: dict[str, InstanceLocationConfig], - contextFactory: IPolicyForHTTPS, - connectTimeout: float | None = None, - bindAddress: bytes | None = None, - pool: HTTPConnectionPool | None = None, - ): - """ - Create a ReplicationAgent. - - Args: - reactor: A reactor for this Agent to place outgoing connections. - contextFactory: A factory for TLS contexts, to control the - verification parameters of OpenSSL. The default is to use a - BrowserLikePolicyForHTTPS, so unless you have special - requirements you can leave this as-is. - connectTimeout: The amount of time that this Agent will wait - for the peer to accept a connection. - bindAddress: The local address for client sockets to bind to. - pool: An HTTPConnectionPool instance, or None, in which - case a non-persistent HTTPConnectionPool instance will be - created. - """ - _AgentBase.__init__(self, reactor, pool) - endpoint_factory = ReplicationEndpointFactory( - reactor, instance_map, contextFactory - ) - self._endpointFactory = endpoint_factory - - def request( - self, - method: bytes, - uri: bytes, - headers: Headers | None = None, - bodyProducer: Optional[IBodyProducer] = None, - ) -> Any: - """ - Issue a request to the server indicated by the given uri. - - An existing connection from the connection pool may be used or a new - one may be created. - - Currently, HTTP, HTTPS and UNIX schemes are supported in uri. - - This is copied from twisted.web.client.Agent, except: - - * It uses a different pool key (combining the scheme with either host & port or - socket path). - * It does not call _ensureValidURI(...) as the strictness of IDNA2008 is not - required when using a worker's name as a 'hostname' for Synapse HTTP - Replication machinery. Specifically, this allows a range of ascii characters - such as '+' and '_' in hostnames/worker's names. - - See: twisted.web.iweb.IAgent.request - """ - parsedURI = URI.fromBytes(uri) - try: - endpoint = self._endpointFactory.endpointForURI(parsedURI) - except SchemeNotSupported: - raise - - worker_name = parsedURI.netloc.decode("utf-8") - key_scheme = self._endpointFactory.instance_map[worker_name].scheme() - key_netloc = self._endpointFactory.instance_map[worker_name].netloc() - # Build a connection pool key. - # - # `_AgentBase` expects this to be a three-tuple of `(scheme, host, - # port)` of type `bytes`. We don't have a real port when connecting via - # a Unix socket, so use `0`. - key = ( - key_scheme.encode("ascii"), - key_netloc.encode("utf-8"), - 0, + """Stub: The Twisted ReplicationEndpointFactory has been removed.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "ReplicationEndpointFactory has been removed. " + "Use NativeReplicationClient instead." ) - # _requestWithEndpoint comes from _AgentBase class - return self._requestWithEndpoint( - key, - endpoint, - method, - parsedURI, - headers, - bodyProducer, - parsedURI.originForm, + +class ReplicationAgent: + """Stub: The Twisted ReplicationAgent has been removed.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError( + "ReplicationAgent has been removed. " + "Use NativeReplicationClient instead." ) diff --git a/synapse/http/server.py b/synapse/http/server.py index 2d74c9b101..8974871632 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -45,11 +45,13 @@ from asyncio import CancelledError from synapse.types import ISynapseThreadlessReactor try: - from twisted.internet import reactor + from twisted.internet import reactor as _twisted_reactor from twisted.web import resource from twisted.web.pages import notFound except ImportError: - pass + _twisted_reactor = None # type: ignore[assignment] + resource = None # type: ignore[assignment] + notFound = None # type: ignore[assignment] from synapse.api.errors import ( CodeMessageException, @@ -395,7 +397,7 @@ class DirectServeJsonResource(_AsyncResource): # As of the time of writing this, all Synapse internal usages of # `DirectServeJsonResource` pass in the existing homeserver clock instance. clock = Clock( # type: ignore[multiple-internal-clocks] - ISynapseThreadlessReactor(reactor), + ISynapseThreadlessReactor(_twisted_reactor), server_name="synapse_module_running_from_unknown_server", ) @@ -592,7 +594,7 @@ class DirectServeHtmlResource(_AsyncResource): # As of the time of writing this, all Synapse internal usages of # `DirectServeHtmlResource` pass in the existing homeserver clock instance. clock = Clock( # type: ignore[multiple-internal-clocks] - ISynapseThreadlessReactor(reactor), + ISynapseThreadlessReactor(_twisted_reactor), server_name="synapse_module_running_from_unknown_server", ) @@ -626,6 +628,11 @@ try: except ImportError: _StaticBase = object # type: ignore[assignment,misc] +try: + _ResourceBase = resource.Resource +except Exception: + _ResourceBase = object # type: ignore[assignment,misc] + class StaticResource(_StaticBase): """ @@ -634,15 +641,17 @@ class StaticResource(_StaticBase): Differs from the File resource by adding clickjacking protection. """ - def render_GET(self, request: Request) -> bytes: + def render_GET(self, request: Any) -> bytes: set_clickjacking_protection_headers(request) return super().render_GET(request) - def directoryListing(self) -> IResource: - return notFound() + def directoryListing(self) -> Any: + if notFound is not None: + return notFound() + return None -class UnrecognizedRequestResource(resource.Resource): +class UnrecognizedRequestResource(_ResourceBase): """ Similar to twisted.web.resource.NoResource, but returns a JSON 404 with an errcode of M_UNRECOGNIZED. @@ -654,27 +663,28 @@ class UnrecognizedRequestResource(resource.Resource): # Return empty bytes — the response has already been written to the request. return b"" - def getChild(self, name: str, request: Request) -> resource.Resource: + def getChild(self, name: str, request: Any) -> Any: return self -class RootRedirect(resource.Resource): +class RootRedirect(_ResourceBase): """Redirects the root '/' path to another path.""" def __init__(self, path: str): super().__init__() self.url = path - def render_GET(self, request: Request) -> bytes: + def render_GET(self, request: Any) -> bytes: + from twisted.web.util import redirectTo return redirectTo(self.url.encode("ascii"), request) - def getChild(self, name: str, request: Request) -> resource.Resource: + def getChild(self, name: str, request: Any) -> Any: if len(name) == 0: return self # select ourselves as the child to render return super().getChild(name, request) -class OptionsResource(resource.Resource): +class OptionsResource(_ResourceBase): """Responds to OPTION requests for itself and all children.""" def render_OPTIONS(self, request: "SynapseRequest") -> bytes: @@ -685,7 +695,7 @@ class OptionsResource(resource.Resource): return b"" - def getChildWithDefault(self, path: str, request: Request) -> resource.Resource: + def getChildWithDefault(self, path: str, request: Any) -> Any: if request.method == b"OPTIONS": return self # select ourselves as the child to render return super().getChildWithDefault(path, request) diff --git a/synapse/logging/_remote.py b/synapse/logging/_remote.py index ae31cd1f3d..9edde288ee 100644 --- a/synapse/logging/_remote.py +++ b/synapse/logging/_remote.py @@ -19,95 +19,20 @@ # # -from asyncio import CancelledError +import asyncio import logging import sys import traceback from collections import deque -from ipaddress import IPv4Address, IPv6Address, ip_address from math import floor -from typing import Callable, Optional - -import attr -try: - from zope.interface import implementer -except ImportError: - pass - -try: - from twisted.application.internet import ClientService - from asyncio import CancelledError - from twisted.internet.defer import Deferred - from twisted.internet.endpoints import ( - HostnameEndpoint, - TCP4ClientEndpoint, - TCP6ClientEndpoint, - ) - from twisted.internet.interfaces import ( - IPushProducer, - IReactorTime, - IStreamClientEndpoint, - ) - from twisted.internet.protocol import Factory, Protocol - from twisted.internet.tcp import Connection - from twisted.python.failure import Failure -except ImportError: - pass +from typing import Any, Optional logger = logging.getLogger(__name__) -@attr.s(slots=True, auto_attribs=True) -@implementer(IPushProducer) -class LogProducer: - """ - An IPushProducer that writes logs from its buffer to its transport when it - is resumed. - - Args: - buffer: Log buffer to read logs from. - transport: Transport to write to. - format: A callable to format the log record to a string. - """ - - # This is essentially ITCPTransport, but that is missing certain fields - # (connected and registerProducer) which are part of the implementation. - transport: Connection - _format: Callable[[logging.LogRecord], str] - _buffer: deque[logging.LogRecord] - _paused: bool = attr.ib(default=False, init=False) - - def pauseProducing(self) -> None: - self._paused = True - - def stopProducing(self) -> None: - self._paused = True - self._buffer = deque() - - def resumeProducing(self) -> None: - # If we're already producing, nothing to do. - self._paused = False - - # Loop until paused. - while self._paused is False and (self._buffer and self.transport.connected): - try: - # Request the next record and format it. - record = self._buffer.popleft() - msg = self._format(record) - - # Send it as a new line over the transport. - self.transport.write(msg.encode("utf8")) - self.transport.write(b"\n") - except Exception: - # Something has gone wrong writing to the transport -- log it - # and break out of the while. - traceback.print_exc(file=sys.__stderr__) - break - - class RemoteHandler(logging.Handler): """ - An logging handler that writes logs to a TCP target. + A logging handler that writes logs to a TCP target using asyncio. Args: host: The host of the logging target. @@ -121,7 +46,7 @@ class RemoteHandler(logging.Handler): port: int, maximum_buffer: int = 1000, level: int = logging.NOTSET, - _reactor: Optional[IReactorTime] = None, + _reactor: Any = None, ): super().__init__(level=level) self.host = host @@ -129,84 +54,74 @@ class RemoteHandler(logging.Handler): self.maximum_buffer = maximum_buffer self._buffer: deque[logging.LogRecord] = deque() - self._connection_waiter: Deferred | None = None - self._producer: LogProducer | None = None + self._writer: Optional[asyncio.StreamWriter] = None + self._connecting = False + self._stopping = False - # Connect without DNS lookups if it's a direct IP. - if _reactor is None: - _reactor = reactor # type: ignore[assignment] + # Try to connect immediately + self._schedule_connect() + + def _schedule_connect(self) -> None: + """Schedule a connection attempt on the event loop.""" + if self._connecting or self._stopping: + return try: - ip = ip_address(self.host) - if isinstance(ip, IPv4Address): - endpoint: IStreamClientEndpoint = TCP4ClientEndpoint( - _reactor, self.host, self.port - ) - elif isinstance(ip, IPv6Address): - endpoint = TCP6ClientEndpoint(_reactor, self.host, self.port) + loop = asyncio.get_event_loop() + if loop.is_running(): + loop.create_task(self._connect()) else: - raise ValueError("Unknown IP address provided: %s" % (self.host,)) - except ValueError: - endpoint = HostnameEndpoint(_reactor, self.host, self.port) + # Event loop not running yet; we'll try again on emit() + pass + except RuntimeError: + pass # No event loop - factory = Factory.forProtocol(Protocol) - self._service = ClientService(endpoint, factory, clock=_reactor) - self._service.startService() - self._stopping = False - self._connect() + async def _connect(self) -> None: + """Connect to the remote logging target.""" + if self._connecting or self._stopping: + return + + self._connecting = True + try: + _, writer = await asyncio.open_connection(self.host, self.port) + self._writer = writer + self._flush_buffer() + except Exception: + traceback.print_exc(file=sys.__stderr__) + self._writer = None + # Retry after a delay + if not self._stopping: + try: + loop = asyncio.get_event_loop() + loop.call_later(5, self._schedule_connect) + except RuntimeError: + pass + finally: + self._connecting = False + + def _flush_buffer(self) -> None: + """Write buffered log records to the transport.""" + if self._writer is None: + return + + while self._buffer: + try: + record = self._buffer.popleft() + msg = self.format(record) + self._writer.write(msg.encode("utf-8")) + self._writer.write(b"\n") + except Exception: + traceback.print_exc(file=sys.__stderr__) + break def close(self) -> None: self._stopping = True - self._service.stopService() - - def _connect(self) -> None: - """ - Triggers an attempt to connect then write to the remote if not already writing. - """ - # Do not attempt to open multiple connections. - if self._connection_waiter: - return - - def fail(failure: Failure) -> None: - # If the Deferred was cancelled (e.g. during shutdown) do not try to - # reconnect (this will cause an infinite loop of errors). - if failure.check(CancelledError) and self._stopping: - return - - # For a different error, print the traceback and re-connect. - failure.printTraceback(file=sys.__stderr__) - self._connection_waiter = None - self._connect() - - def writer(result: Protocol) -> None: - # Force recognising transport as a Connection and not the more - # generic ITransport. - transport: Connection = result.transport # type: ignore - - # We have a connection. If we already have a producer, and its - # transport is the same, just trigger a resumeProducing. - if self._producer and transport is self._producer.transport: - self._producer.resumeProducing() - self._connection_waiter = None - return - - # If the producer is still producing, stop it. - if self._producer: - self._producer.stopProducing() - - # Make a new producer and start it. - self._producer = LogProducer( - buffer=self._buffer, - transport=transport, - format=self.format, - ) - transport.registerProducer(self._producer, True) - self._producer.resumeProducing() - self._connection_waiter = None - - deferred: Deferred = self._service.whenConnected(failAfterFailures=1) - deferred.addCallbacks(writer, fail) - self._connection_waiter = deferred + if self._writer is not None: + try: + self._writer.close() + except Exception: + pass + super().close() def _handle_pressure(self) -> None: """ @@ -263,5 +178,9 @@ class RemoteHandler(logging.Handler): self._buffer.clear() logger.warning("Failed clearing backpressure") - # Try and write immediately. - self._connect() + # Try to write immediately if connected + if self._writer is not None: + self._flush_buffer() + else: + # Try to connect + self._schedule_connect() diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 4336bf4892..215e29cc97 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -27,39 +27,24 @@ from abc import ABC, abstractmethod from types import TracebackType from typing import ( TYPE_CHECKING, + Any, Awaitable, BinaryIO, Generator, Optional, ) -import attr -try: - from zope.interface import implementer -except ImportError: - pass +import asyncio -try: - from twisted.internet import interfaces - from twisted.internet.defer import Deferred - from twisted.internet.interfaces import IConsumer - from twisted.python.failure import Failure -except ImportError: - interfaces = None # type: ignore[assignment] - Deferred = None # type: ignore[assignment,misc] - IConsumer = None # type: ignore[assignment,misc] - Failure = BaseException # type: ignore[assignment,misc] +import attr from synapse.api.errors import Codes, cs_error from synapse.http.server import finish_request, respond_with_json from synapse.http.site import SynapseRequest from synapse.logging.context import ( - PreserveLoggingContext, - defer_to_threadpool, - make_deferred_yieldable, + defer_to_thread, run_in_background, ) -from synapse.util.async_helpers import DeferredEvent from synapse.util.clock import Clock from synapse.util.stringutils import is_ascii @@ -163,7 +148,7 @@ async def respond_with_file( def add_file_headers( - request: Request, + request: SynapseRequest, media_type: str, file_size: int | None, upload_name: str | None, @@ -243,7 +228,7 @@ def add_file_headers( request.setHeader(b"X-Robots-Tag", "noindex, nofollow, noarchive, noimageindex") -def _add_cache_headers(request: Request) -> None: +def _add_cache_headers(request: SynapseRequest) -> None: """Adds the appropriate cache headers to the response""" # Cache on the client for at least a day. @@ -494,7 +479,7 @@ class Responder(ABC): """ @abstractmethod - def write_to_consumer(self, consumer: IConsumer) -> Awaitable: + def write_to_consumer(self, consumer: Any) -> Awaitable: """Stream response into consumer Args: @@ -685,155 +670,49 @@ class ConsumerRequestedStopError(Exception): """A consumer asked us to stop producing""" -@implementer(interfaces.IPushProducer) class ThreadedFileSender: """ - A producer that sends the contents of a file to a consumer, reading from the - file on a thread. - - This works by having a loop in a threadpool repeatedly reading from the - file, until the consumer pauses the producer. There is then a loop in the - main thread that waits until the consumer resumes the producer and then - starts reading in the threadpool again. - - This is done to ensure that we're never waiting in the threadpool, as - otherwise its easy to starve it of threads. + Sends the contents of a file to a consumer, reading from the file + in a thread executor to avoid blocking the event loop. """ # How much data to read in one go. CHUNK_SIZE = 2**14 - # How long we wait for the consumer to be ready again before aborting the - # read. - TIMEOUT_SECONDS = 90.0 - def __init__(self, hs: "HomeServer") -> None: self.reactor = hs.get_reactor() self.clock = hs.get_clock() - self.thread_pool = hs.get_media_sender_thread_pool() - self.file: BinaryIO | None = None - self.deferred: "Deferred[None]" = Deferred() - self.consumer: Optional[IConsumer] = None + async def beginFileTransfer(self, file: BinaryIO, consumer: Any) -> None: + """Transfer a file's contents to a consumer (e.g. a request object). - # Signals if the thread should keep reading/sending data. Set means - # continue, clear means pause. - self.wakeup_event = DeferredEvent(self.clock) - - # Signals if the thread should terminate, e.g. because the consumer has - # gone away. - self.stop_writing = False - - def beginFileTransfer( - self, file: BinaryIO, consumer: interfaces.IConsumer - ) -> "Deferred[None]": + Reads the file in chunks in a thread executor, then writes each chunk + to the consumer on the main thread. """ - Begin transferring a file - """ - self.file = file - self.consumer = consumer + loop = asyncio.get_event_loop() - self.consumer.registerProducer(self, True) + try: + consumer.registerProducer(self, True) - # We set the wakeup signal as we should start producing immediately. - self.wakeup_event.set() - run_in_background(self.start_read_loop) - - return make_deferred_yieldable(self.deferred) + while True: + chunk = await loop.run_in_executor(None, file.read, self.CHUNK_SIZE) + if not chunk: + break + consumer.write(chunk) + except Exception: + if hasattr(consumer, 'unregisterProducer'): + consumer.unregisterProducer() + raise + finally: + file.close() + if hasattr(consumer, 'unregisterProducer'): + consumer.unregisterProducer() def resumeProducing(self) -> None: - """interfaces.IPushProducer""" - self.wakeup_event.set() + pass def pauseProducing(self) -> None: - """interfaces.IPushProducer""" - self.wakeup_event.clear() + pass def stopProducing(self) -> None: - """interfaces.IPushProducer""" - - # Unregister the consumer so we don't try and interact with it again. - if self.consumer: - self.consumer.unregisterProducer() - - self.consumer = None - - # Terminate the loop. - self.stop_writing = True - self.wakeup_event.set() - - if not self.deferred.called: - with PreserveLoggingContext(): - self.deferred.errback( - ConsumerRequestedStopError("Consumer asked us to stop producing") - ) - - async def start_read_loop(self) -> None: - """This is the loop that drives reading/writing""" - try: - while not self.stop_writing: - # Start the loop in the threadpool to read data. - more_data = await defer_to_threadpool( - self.reactor, self.thread_pool, self._on_thread_read_loop - ) - if not more_data: - # Reached EOF, we can just return. - return - - if not self.wakeup_event.is_set(): - ret = await self.wakeup_event.wait(self.TIMEOUT_SECONDS) - if not ret: - raise Exception("Timed out waiting to resume") - except Exception: - self._error(Failure()) - finally: - self._finish() - - def _on_thread_read_loop(self) -> bool: - """This is the loop that happens on a thread. - - Returns: - Whether there is more data to send. - """ - - while not self.stop_writing and self.wakeup_event.is_set(): - # The file should always have been set before we get here. - assert self.file is not None - - chunk = self.file.read(self.CHUNK_SIZE) - if not chunk: - return False - - self.reactor.callFromThread(self._write, chunk) - - return True - - def _write(self, chunk: bytes) -> None: - """Called from the thread to write a chunk of data""" - if self.consumer: - self.consumer.write(chunk) - - def _error(self, failure: Failure) -> None: - """Called when there was a fatal error""" - if self.consumer: - self.consumer.unregisterProducer() - self.consumer = None - - if not self.deferred.called: - with PreserveLoggingContext(): - self.deferred.errback(failure) - - def _finish(self) -> None: - """Called when we have finished writing (either on success or - failure).""" - if self.file: - self.file.close() - self.file = None - - if self.consumer: - self.consumer.unregisterProducer() - self.consumer = None - - if not self.deferred.called: - with PreserveLoggingContext(): - self.deferred.callback(None) + pass diff --git a/synapse/media/media_storage.py b/synapse/media/media_storage.py index 9bdaaaba1a..6963890a69 100644 --- a/synapse/media/media_storage.py +++ b/synapse/media/media_storage.py @@ -41,19 +41,6 @@ from typing import ( from uuid import uuid4 import attr -try: - from zope.interface import implementer -except ImportError: - pass - -try: - from twisted.internet import interfaces - from twisted.internet.defer import Deferred - from twisted.internet.interfaces import IConsumer -except ImportError: - interfaces = None # type: ignore[assignment] - Deferred = None # type: ignore[assignment,misc] - IConsumer = None # type: ignore[assignment,misc] from synapse.api.errors import NotFoundError from synapse.logging.context import defer_to_thread, run_in_background @@ -497,8 +484,8 @@ class FileResponder(Responder): self.hs = hs self.open_file = open_file - def write_to_consumer(self, consumer: IConsumer) -> Deferred: - return ThreadedFileSender(self.hs).beginFileTransfer(self.open_file, consumer) + async def write_to_consumer(self, consumer: Any) -> None: + await ThreadedFileSender(self.hs).beginFileTransfer(self.open_file, consumer) def __exit__( self, @@ -544,8 +531,6 @@ class ReadableFileWrapper: await self.clock.sleep(Duration(seconds=0)) -@implementer(interfaces.IConsumer) -@implementer(interfaces.IPushProducer) class MultipartFileConsumer: """Wraps a given consumer so that any data that gets written to it gets converted to a multipart format. @@ -554,7 +539,7 @@ class MultipartFileConsumer: def __init__( self, clock: Clock, - wrapped_consumer: interfaces.IConsumer, + wrapped_consumer: Any, file_content_type: str, json_object: JsonDict, disposition: str, @@ -570,7 +555,7 @@ class MultipartFileConsumer: # The producer that registered with us, and if it's a push or pull # producer. - self.producer: "interfaces.IProducer" | None = None + self.producer: Any = None self.streaming: bool | None = None # Whether the wrapped consumer has asked us to pause. @@ -582,7 +567,7 @@ class MultipartFileConsumer: ### IConsumer APIs ### def registerProducer( - self, producer: "interfaces.IProducer", streaming: bool + self, producer: Any, streaming: bool ) -> None: """ Register to receive data from a producer. @@ -680,7 +665,7 @@ class MultipartFileConsumer: self.paused = True if self.streaming: - cast("interfaces.IPushProducer", self.producer).pauseProducing() + self.producer.pauseProducing() else: self.paused = True @@ -694,7 +679,7 @@ class MultipartFileConsumer: assert self.producer is not None if self.streaming: - cast("interfaces.IPushProducer", self.producer).resumeProducing() + self.producer.resumeProducing() else: # If the producer is not a streaming producer we need to start # repeatedly calling `resumeProducing` in a loop. @@ -731,7 +716,7 @@ class MultipartFileConsumer: async def _resumeProducingRepeatedly(self) -> None: assert self.producer is not None assert not self.streaming - producer = cast("interfaces.IPullProducer", self.producer) + producer = self.producer self.paused = False while not self.paused: diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 1023bc75b0..424f64e712 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -36,13 +36,6 @@ import asyncio import attr -try: - from twisted.internet.defer import Deferred - from twisted.internet.error import DNSLookupError -except ImportError: - Deferred = asyncio.Future # type: ignore[assignment,misc] - DNSLookupError = OSError # type: ignore[assignment,misc] - from synapse.api.errors import Codes, SynapseError from synapse.http.native_client import SimpleHttpClient from synapse.logging.context import make_deferred_yieldable, run_in_background @@ -490,7 +483,7 @@ class UrlPreviewer: # handler will return a SynapseError to the client instead of # blank data or a 500. raise - except DNSLookupError: + except OSError: # DNS lookup returned no results # Note: This will also be the case if one of the resolved IP # addresses is blocked. @@ -752,7 +745,7 @@ class UrlPreviewer: return open_graph_result, oembed_response.author_name, expiration_ms - def _start_expire_url_cache_data(self) -> Deferred: + def _start_expire_url_cache_data(self) -> asyncio.Future: return self.hs.run_as_background_process( "expire_url_cache_data", self._expire_url_cache_data ) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 6e46dd1eec..0d232ea3ca 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -61,10 +61,7 @@ from synapse.metrics._types import Collector from synapse.types import StrSequence from synapse.util import SYNAPSE_VERSION -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource logger = logging.getLogger(__name__) diff --git a/synapse/metrics/_gc.py b/synapse/metrics/_gc.py index 581ff7f356..8692fe0b6b 100644 --- a/synapse/metrics/_gc.py +++ b/synapse/metrics/_gc.py @@ -20,17 +20,13 @@ # +import asyncio import gc import logging import platform import time from typing import Iterable -try: - from twisted.internet import task -except ImportError: - task = None # type: ignore[assignment] - from prometheus_client.core import ( REGISTRY, CounterMetricFamily, @@ -141,10 +137,20 @@ def install_gc_manager() -> None: gc_time.labels(i).observe(end - start) gc_unreachable.labels(i).set(unreachable) - # We can ignore the lint here since this looping call does not hold a `HomeServer` - # reference so can be cleaned up by other means on shutdown. - gc_task = task.LoopingCall(_maybe_gc) # type: ignore[prefer-synapse-clock-looping-call] - gc_task.start(0.1) + async def _gc_loop() -> None: + """Async loop that periodically runs _maybe_gc.""" + while True: + await asyncio.sleep(0.1) + _maybe_gc() + + # Schedule the GC loop. We get the running event loop; if there isn't one + # yet (e.g. during module-level init), the caller is expected to start the + # loop later, and this will be a no-op. + try: + loop = asyncio.get_event_loop() + loop.create_task(_gc_loop()) + except RuntimeError: + pass # No event loop yet; GC manager will be less effective. # diff --git a/synapse/metrics/_reactor_metrics.py b/synapse/metrics/_reactor_metrics.py index d1efaba703..c3dd19d262 100644 --- a/synapse/metrics/_reactor_metrics.py +++ b/synapse/metrics/_reactor_metrics.py @@ -19,6 +19,7 @@ # # +import asyncio import logging import time from selectors import SelectSelector, _PollLikeSelector # type: ignore[attr-defined] @@ -27,7 +28,6 @@ from typing import Any, Callable, Iterable from prometheus_client import Histogram, Metric from prometheus_client.core import REGISTRY, GaugeMetricFamily -from synapse.app.complement_fork_proxied_reactor import ProxiedReactor from synapse.metrics._types import Collector try: @@ -38,46 +38,16 @@ except ImportError: pass -try: - from twisted.internet.epollreactor import EPollReactor -except ImportError: - - class EPollReactor: # type: ignore[no-redef] - pass - - -try: - from twisted.internet.pollreactor import PollReactor -except ImportError: - - class PollReactor: # type: ignore[no-redef] - pass - - -try: - from twisted.internet import selectreactor -except ImportError: - selectreactor = None # type: ignore[assignment] - - -try: - from twisted.internet.asyncioreactor import AsyncioSelectorReactor -except ImportError: - - class AsyncioSelectorReactor: # type: ignore[no-redef] - pass - - logger = logging.getLogger(__name__) # -# Twisted reactor metrics +# Event loop metrics # # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. tick_time = Histogram( # type: ignore[missing-server-name-label] "python_twisted_reactor_tick_time", - "Tick time of the Twisted reactor (sec)", + "Tick time of the asyncio event loop (sec)", buckets=[0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2, 5], ) @@ -91,7 +61,7 @@ class CallWrapper: def __call__(self, *args, **kwargs) -> Any: # type: ignore[no-untyped-def] # record the time since this was last called. This gives a good proxy for - # how long it takes to run everything in the reactor - ie, how long anything + # how long it takes to run everything in the event loop - ie, how long anything # waiting for the next tick will have to wait. tick_time.observe(time.time() - self.last_polled) @@ -130,57 +100,73 @@ class ReactorLastSeenMetric(Collector): # This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`. cm = GaugeMetricFamily( # type: ignore[missing-server-name-label] "python_twisted_reactor_last_seen", - "Seconds since the Twisted reactor was last seen", + "Seconds since the asyncio event loop was last seen", ) cm.add_metric([], time.time() - self._call_wrapper.last_polled) yield cm -def install_reactor_metrics(target_reactor: Any) -> None: - # Twisted has already select a reasonable reactor for us, so assumptions can be - # made about the shape. +def install_reactor_metrics(target: Any = None) -> None: + """Install metrics for the asyncio event loop. + + Wraps the underlying selector's poll/select method to measure tick times. + + Args: + target: Either an asyncio event loop, or a Twisted reactor (for backward + compatibility). If None, attempts to get the running asyncio event loop. + """ wrapper = None try: - if isinstance(target_reactor, (PollReactor, EPollReactor)): - target_reactor._poller = ObjWrapper(reactor._poller, "poll") # type: ignore[attr-defined] - wrapper = target_reactor._poller._wrapped_method # type: ignore[attr-defined] - - elif selectreactor is not None and isinstance(target_reactor, selectreactor.SelectReactor): - # Twisted uses a module-level _select function. - wrapper = selectreactor._select = CallWrapper(selectreactor._select) - - elif isinstance(target_reactor, AsyncioSelectorReactor): - # For asyncio look at the underlying asyncio event loop. - asyncio_loop = ( - target_reactor._asyncioEventloop - ) # A sub-class of BaseEventLoop, - - # A sub-class of BaseSelector. - selector = asyncio_loop._selector # type: ignore[attr-defined] - - if isinstance(selector, SelectSelector): - wrapper = selector._select = CallWrapper(selector._select) # type: ignore[attr-defined] - - # poll, epoll, and /dev/poll. - elif isinstance(selector, _PollLikeSelector): - selector._selector = ObjWrapper(selector._selector, "poll") # type: ignore[attr-defined] - wrapper = selector._selector._wrapped_method # type: ignore[attr-defined] - - elif isinstance(selector, KqueueSelector): - selector._selector = ObjWrapper(selector._selector, "control") # type: ignore[attr-defined] - wrapper = selector._selector._wrapped_method # type: ignore[attr-defined] - - else: - # E.g. this does not support the (Windows-only) ProactorEventLoop. + # Determine the asyncio event loop to instrument + asyncio_loop = None + if target is None: + try: + asyncio_loop = asyncio.get_event_loop() + except RuntimeError: logger.warning( - "Skipping configuring reactor metrics: unexpected asyncio loop selector: %r via %r", - selector, - asyncio_loop, + "Skipping configuring reactor metrics: no running asyncio event loop" ) + return + elif hasattr(target, '_selector'): + # It's an asyncio event loop directly + asyncio_loop = target + elif hasattr(target, '_asyncioEventloop'): + # It's a Twisted AsyncioSelectorReactor + asyncio_loop = target._asyncioEventloop else: logger.warning( - "Skipping configuring reactor metrics: unexpected reactor type: %r", - target_reactor, + "Skipping configuring reactor metrics: unexpected target type: %r", + target, + ) + return + + if asyncio_loop is None: + logger.warning( + "Skipping configuring reactor metrics: could not find asyncio event loop" + ) + return + + # A sub-class of BaseSelector. + selector = asyncio_loop._selector # type: ignore[attr-defined] + + if isinstance(selector, SelectSelector): + wrapper = selector._select = CallWrapper(selector._select) # type: ignore[attr-defined] + + # poll, epoll, and /dev/poll. + elif isinstance(selector, _PollLikeSelector): + selector._selector = ObjWrapper(selector._selector, "poll") # type: ignore[attr-defined] + wrapper = selector._selector._wrapped_method # type: ignore[attr-defined] + + elif isinstance(selector, KqueueSelector): + selector._selector = ObjWrapper(selector._selector, "control") # type: ignore[attr-defined] + wrapper = selector._selector._wrapped_method # type: ignore[attr-defined] + + else: + # E.g. this does not support the (Windows-only) ProactorEventLoop. + logger.warning( + "Skipping configuring reactor metrics: unexpected asyncio loop selector: %r via %r", + selector, + asyncio_loop, ) except Exception as e: logger.warning("Configuring reactor metrics failed: %r", e) @@ -190,15 +176,12 @@ def install_reactor_metrics(target_reactor: Any) -> None: REGISTRY.register(ReactorLastSeenMetric(wrapper)) # type: ignore[missing-server-name-label] -# Install reactor metrics for the global reactor. -# -# Skip installation if using `ProxiedReactor` (used by `complement_fork_starter.py`). -# The `ProxiedReactor` will handle calling `install_reactor_metrics(...)` itself when -# ready. Skipping allows us to avoid seeing confusing `Skipping configuring reactor -# metrics: unexpected reactor type: ProxiedReactor` warnings. +# Install reactor metrics for the global asyncio event loop (if available). +# This may be called before the event loop is running, in which case +# it will be a no-op and should be called again later. try: - from twisted.internet import reactor - if not isinstance(reactor, ProxiedReactor): - install_reactor_metrics(reactor) -except ImportError: - pass # Twisted not installed; reactor metrics unavailable + loop = asyncio.get_event_loop() + if loop.is_running(): + install_reactor_metrics(loop) +except RuntimeError: + pass # No event loop yet; metrics will be installed later diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 449ddebd90..221dbe885e 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -157,10 +157,7 @@ from synapse.util.duration import Duration from synapse.util.frozenutils import freeze from synapse.util.sentinel import Sentinel -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource if TYPE_CHECKING: diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 6a2dbf5d25..9e084454d5 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -27,11 +27,8 @@ from typing import TYPE_CHECKING, Any, Awaitable, Callable, ClassVar from prometheus_client import Counter, Gauge -try: - from twisted.internet.error import ConnectError, DNSLookupError -except ImportError: - ConnectError = OSError # type: ignore[assignment,misc] - DNSLookupError = OSError # type: ignore[assignment,misc] +ConnectError = OSError +DNSLookupError = OSError from synapse.api.errors import HttpResponseException, SynapseError from synapse.config.workers import MAIN_PROCESS_INSTANCE_NAME diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index db9c6126cd..8768effc85 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -22,15 +22,11 @@ import asyncio """A replication client for use by synapse workers.""" import logging +from asyncio import Future from typing import TYPE_CHECKING, Iterable from sortedcontainers import SortedList -try: - from twisted.internet.defer import Deferred -except ImportError: - from asyncio import Future as Deferred # type: ignore[assignment] - from synapse.api.constants import EventTypes, Membership, ReceiptTypes from synapse.federation import send_queue from synapse.federation.sender import FederationSender @@ -103,7 +99,7 @@ class ReplicationDataHandler: # Map from stream and instance to list of deferreds waiting for the stream to # arrive at a particular position. The lists are sorted by stream position. self._streams_to_waiters: dict[ - tuple[str, str], SortedList[tuple[int, Deferred]] + tuple[str, str], SortedList[tuple[int, Future]] ] = {} async def on_rdata( @@ -354,7 +350,7 @@ class ReplicationDataHandler: # Create a new deferred that times out after N seconds, as we don't want # to wedge here forever. - deferred: "Deferred[None]" = Deferred() + deferred: "Future[None]" = Future() deferred = timeout_deferred( deferred=deferred, timeout=_WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, diff --git a/synapse/replication/tcp/context.py b/synapse/replication/tcp/context.py index 3c13ae10b1..5953dbbd30 100644 --- a/synapse/replication/tcp/context.py +++ b/synapse/replication/tcp/context.py @@ -18,27 +18,24 @@ # [This file includes modifications made by New Vector Limited] # # -from OpenSSL.SSL import Context - -try: - from twisted.internet import ssl as twisted_ssl - _ContextFactoryBase = twisted_ssl.ClientContextFactory -except ImportError: - _ContextFactoryBase = object # type: ignore[assignment,misc] +import ssl from synapse.config.redis import RedisConfig -class ClientContextFactory(_ContextFactoryBase): +class ClientContextFactory: + """Creates SSL contexts for Redis connections.""" + def __init__(self, redis_config: RedisConfig): self.redis_config = redis_config - def getContext(self) -> Context: - ctx = super().getContext() + def getContext(self) -> ssl.SSLContext: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) if self.redis_config.redis_certificate: - ctx.use_certificate_file(self.redis_config.redis_certificate) - if self.redis_config.redis_private_key: - ctx.use_privatekey_file(self.redis_config.redis_private_key) + ctx.load_cert_chain( + certfile=self.redis_config.redis_certificate, + keyfile=self.redis_config.redis_private_key, + ) if self.redis_config.redis_ca_file: ctx.load_verify_locations(cafile=self.redis_config.redis_ca_file) elif self.redis_config.redis_ca_path: diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 4068b60f50..5faa1ba2e6 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -28,13 +28,19 @@ import fcntl import logging import struct from inspect import isawaitable -from typing import TYPE_CHECKING, Any, Collection +from typing import TYPE_CHECKING, Any, Callable, Collection from prometheus_client import Counter try: from zope.interface import Interface, implementer except ImportError: - pass + # Provide minimal stubs + class Interface: # type: ignore[no-redef] + pass + def implementer(*args: Any, **kwargs: Any) -> Any: # type: ignore[no-redef] + def decorator(cls: Any) -> Any: + return cls + return decorator try: from twisted.internet import task diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index ad9e1c30bb..86374adcdc 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -24,21 +24,32 @@ from inspect import isawaitable from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast import attr -from txredisapi import ( - ConnectionHandler, - RedisFactory, - SubscriberProtocol, - UnixConnectionHandler, -) +try: + from txredisapi import ( + ConnectionHandler, + RedisFactory, + SubscriberProtocol, + UnixConnectionHandler, + ) +except ImportError: + ConnectionHandler = object # type: ignore[misc,assignment] + RedisFactory = object # type: ignore[misc,assignment] + SubscriberProtocol = object # type: ignore[misc,assignment] + UnixConnectionHandler = object # type: ignore[misc,assignment] + try: from zope.interface import implementer except ImportError: - pass + def implementer(*args: Any, **kwargs: Any) -> Any: # type: ignore[no-redef] + def decorator(cls: Any) -> Any: + return cls + return decorator try: - from twisted.internet.interfaces import IConnector + from twisted.internet.interfaces import IAddress, IConnector from twisted.python.failure import Failure except ImportError: + IAddress = Any # type: ignore[assignment,misc] IConnector = Any # type: ignore[assignment,misc] Failure = BaseException # type: ignore[assignment,misc] @@ -360,8 +371,8 @@ class SynapseRedisFactory(RedisFactory): super().clientConnectionLost(connector, reason) -def format_address(address: IAddress) -> str: - if isinstance(address, (IPv4Address, IPv6Address)): +def format_address(address: Any) -> str: + if hasattr(address, 'host') and hasattr(address, 'port'): return "%s:%i" % (address.host, address.port) return str(address) diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 00f06a9494..fd4f07f68d 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -22,7 +22,7 @@ import logging import random -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from prometheus_client import Counter @@ -36,8 +36,10 @@ from synapse.util.metrics import Measure try: from twisted.internet.protocol import ServerFactory + from twisted.internet.interfaces import IAddress except ImportError: ServerFactory = object # type: ignore[misc,assignment] + IAddress = Any # type: ignore[misc,assignment] if TYPE_CHECKING: from synapse.server import HomeServer diff --git a/synapse/rest/health.py b/synapse/rest/health.py index 3e2e8ae367..2ff9e688a0 100644 --- a/synapse/rest/health.py +++ b/synapse/rest/health.py @@ -22,10 +22,7 @@ from synapse.api.errors import Codes from synapse.http.aiohttp_shim import SynapseRequest as Request -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py index 7eeab9bbe4..8fc5832b69 100644 --- a/synapse/rest/synapse/client/__init__.py +++ b/synapse/rest/synapse/client/__init__.py @@ -30,10 +30,7 @@ from synapse.rest.synapse.client.sso_register import SsoRegisterResource from synapse.rest.synapse.client.unsubscribe import UnsubscribeResource from synapse.rest.synapse.mas import MasResource -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource if TYPE_CHECKING: diff --git a/synapse/rest/synapse/client/oidc/__init__.py b/synapse/rest/synapse/client/oidc/__init__.py index 8af165e8c0..c8af15b557 100644 --- a/synapse/rest/synapse/client/oidc/__init__.py +++ b/synapse/rest/synapse/client/oidc/__init__.py @@ -27,10 +27,7 @@ from synapse.rest.synapse.client.oidc.backchannel_logout_resource import ( ) from synapse.rest.synapse.client.oidc.callback_resource import OIDCCallbackResource -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource if TYPE_CHECKING: diff --git a/synapse/rest/synapse/client/pick_username.py b/synapse/rest/synapse/client/pick_username.py index 3c5a459a94..283a9c29f5 100644 --- a/synapse/rest/synapse/client/pick_username.py +++ b/synapse/rest/synapse/client/pick_username.py @@ -34,10 +34,7 @@ from synapse.http.site import SynapseRequest from synapse.types import JsonDict from synapse.util.templates import build_jinja_env -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource if TYPE_CHECKING: diff --git a/synapse/rest/synapse/client/saml2/__init__.py b/synapse/rest/synapse/client/saml2/__init__.py index 2792708949..358ae2ed7c 100644 --- a/synapse/rest/synapse/client/saml2/__init__.py +++ b/synapse/rest/synapse/client/saml2/__init__.py @@ -24,10 +24,7 @@ from typing import TYPE_CHECKING from synapse.rest.synapse.client.saml2.metadata_resource import SAML2MetadataResource from synapse.rest.synapse.client.saml2.response_resource import SAML2ResponseResource -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource if TYPE_CHECKING: diff --git a/synapse/rest/synapse/client/saml2/metadata_resource.py b/synapse/rest/synapse/client/saml2/metadata_resource.py index 47e9ab9721..af2abbab60 100644 --- a/synapse/rest/synapse/client/saml2/metadata_resource.py +++ b/synapse/rest/synapse/client/saml2/metadata_resource.py @@ -22,10 +22,7 @@ from typing import TYPE_CHECKING import saml2.metadata -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource if TYPE_CHECKING: diff --git a/synapse/rest/synapse/mas/__init__.py b/synapse/rest/synapse/mas/__init__.py index ab3d84026d..96074a4990 100644 --- a/synapse/rest/synapse/mas/__init__.py +++ b/synapse/rest/synapse/mas/__init__.py @@ -34,10 +34,7 @@ from synapse.rest.synapse.mas.users import ( MasUnsetDisplayNameResource, ) -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource if TYPE_CHECKING: from synapse.server import HomeServer diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py index de9c09daf0..6d132551a0 100644 --- a/synapse/rest/well_known.py +++ b/synapse/rest/well_known.py @@ -28,10 +28,7 @@ from synapse.types import JsonDict from synapse.util.json import json_encoder from synapse.util.stringutils import parse_server_name -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource if TYPE_CHECKING: diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 880311a97e..d7b2b7ce07 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -115,7 +115,7 @@ UNIQUE_INDEX_BACKGROUND_UPDATES = { class _PoolConnection(Connection): """ - A Connection from twisted.enterprise.adbapi.Connection. + A pooled database connection with a reconnect method. """ def reconnect(self) -> None: ... @@ -319,7 +319,7 @@ class LoggingTransaction: def call_after( self, callback: Callable[P, object], *args: P.args, **kwargs: P.kwargs ) -> None: - """Call the given callback on the main twisted thread after the transaction has + """Call the given callback on the main thread after the transaction has finished successfully. Mostly used to invalidate the caches on the correct thread. @@ -340,7 +340,7 @@ class LoggingTransaction: def async_call_after( self, callback: Callable[P, Awaitable], *args: P.args, **kwargs: P.kwargs ) -> None: - """Call the given asynchronous callback on the main twisted thread after + """Call the given asynchronous callback on the main thread after the transaction has finished successfully (but before those added in `call_after`). Mostly used to invalidate remote caches after transactions. @@ -361,7 +361,7 @@ class LoggingTransaction: def call_on_exception( self, callback: Callable[P, object], *args: P.args, **kwargs: P.kwargs ) -> None: - """Call the given callback on the main twisted thread after the transaction has + """Call the given callback on the main thread after the transaction has failed. Note that transactions may be retried a few times if they encounter database @@ -617,7 +617,7 @@ class DatabasePool: self._current_txn_total_time = 0.0 self._previous_loop_ts = 0.0 - # Transaction counter: key is the twisted thread id, value is the current count + # Transaction counter: key is the thread id, value is the current count self._txn_counters: dict[int, int] = defaultdict(int) # TODO(paul): These can eventually be removed once the metrics code @@ -951,7 +951,7 @@ class DatabasePool: Arguments: desc: description of the transaction, for logging and metrics func: callback function, which will be called with a - database transaction (twisted.enterprise.adbapi.Transaction) as + database transaction (LoggingTransaction) as its first argument, followed by `args` and `kwargs`. db_autocommit: Whether to run the function in "autocommit" mode, @@ -1032,7 +1032,7 @@ class DatabasePool: Arguments: func: callback function, which will be called with a - database connection (twisted.enterprise.adbapi.Connection) as + database connection (a Connection object) as its first argument, followed by `args` and `kwargs`. args: positional args to pass to `func` db_autocommit: Whether to run the function in "autocommit" mode, diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 400b5144aa..3b6ea1061a 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -53,24 +53,6 @@ from signedjson.key import decode_verify_key_bytes from signedjson.types import VerifyKey from typing_extensions import Self from unpaddedbase64 import decode_base64 -try: - from zope.interface import Interface -except ImportError: - pass - -try: - from asyncio import CancelledError - from twisted.internet.interfaces import ( - IReactorCore, - IReactorPluggableNameResolver, - IReactorSSL, - IReactorTCP, - IReactorThreads, - IReactorTime, - IReactorUNIX, - ) -except ImportError: - pass from synapse.api.errors import Codes, SynapseError from synapse.util.cancellation import cancellable @@ -121,33 +103,11 @@ StrCollection = tuple[str, ...] | list[str] | AbstractSet[str] StrSequence = tuple[str, ...] | list[str] -# Note that this seems to require inheriting *directly* from Interface in order -# for mypy-zope to realize it is an interface. -class ISynapseThreadlessReactor( - IReactorTCP, - IReactorSSL, - IReactorUNIX, - IReactorPluggableNameResolver, - IReactorTime, - IReactorCore, - Interface, -): - """ - The interfaces necessary for Synapse to function (without threads). - - Helpful because we use `twisted.internet.testing.MemoryReactorClock` in tests which - doesn't implement `IReactorThreads`. - """ - - -# Note that this seems to require inheriting *directly* from Interface in order -# for mypy-zope to realize it is an interface. -class ISynapseReactor( - ISynapseThreadlessReactor, - IReactorThreads, - Interface, -): - """The interfaces necessary for Synapse to function.""" +# Type aliases replacing the former Zope interface definitions. +# These were previously Zope Interface classes combining Twisted reactor +# interfaces. They are only used for type annotations, so Any suffices. +ISynapseThreadlessReactor = Any +ISynapseReactor = Any _ELT = TypeVar("_ELT") diff --git a/synapse/util/clock.py b/synapse/util/clock.py index 817711d860..8572569ac1 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -18,7 +18,6 @@ import asyncio import inspect import logging import time as time_mod -from functools import wraps from typing import ( Any, Callable, @@ -26,25 +25,9 @@ from typing import ( from weakref import WeakSet from typing_extensions import ParamSpec -try: - from zope.interface import implementer -except ImportError: - pass - -try: - from twisted.internet import defer, task - from twisted.internet.defer import Deferred - from twisted.internet.task import LoopingCall -except ImportError: - defer = None # type: ignore[assignment] - Deferred = None # type: ignore[assignment,misc] - LoopingCall = None # type: ignore[assignment,misc] - task = None # type: ignore[assignment] from synapse.logging import context from synapse.logging.loggers import ExplicitlyConfiguredLogger -from synapse.types import ISynapseThreadlessReactor -from synapse.util import log_failure from synapse.util.duration import Duration from synapse.util.stringutils import random_string_insecure_fast @@ -73,552 +56,6 @@ this setting won't inherit the log level from the parent logger. logging.setLoggerClass(original_logger_class) -def _try_wakeup_deferred(d: Deferred) -> None: - """Try to wake up a deferred, but ignore any exceptions raised by the - callback. This is useful when we want to wake up a deferred that may have - already been cancelled, and we don't care about the result.""" - try: - d.callback(None) - except Exception: - pass - - -class Clock: - """ - A Clock wraps a Twisted reactor and provides utilities on top of it. - - This clock should be used in place of calls to the base reactor wherever `LoopingCall` - or `DelayedCall` are made (such as when calling `reactor.callLater`. This is to - ensure the calls made by this `HomeServer` instance are tracked and can be cleaned - up during `HomeServer.shutdown()`. - - We enforce usage of this clock instead of using the reactor directly via lints in - `scripts-dev/mypy_synapse_plugin.py`. - - - Args: - reactor: The Twisted reactor to use. - """ - - _reactor: ISynapseThreadlessReactor - - def __init__(self, reactor: ISynapseThreadlessReactor, server_name: str) -> None: - self._reactor = reactor - self._server_name = server_name - - self._delayed_call_id: int = 0 - """Unique ID used to track delayed calls""" - - self._looping_calls: WeakSet[LoopingCall] = WeakSet() - """List of active looping calls""" - - self._call_id_to_delayed_call: dict[int, Any] = {} - """ - Mapping from unique call ID to delayed call. - - For "performance", this only tracks a subset of delayed calls: those created - with `call_later` with `call_later_cancel_on_shutdown=True`. - """ - - self._is_shutdown = False - """Whether shutdown has been requested by the HomeServer""" - - def shutdown(self) -> None: - self._is_shutdown = True - self.cancel_all_looping_calls() - self.cancel_all_delayed_calls() - - async def sleep(self, duration: Duration) -> None: - from synapse.util.async_helpers import make_awaitable_promise - d: Any = make_awaitable_promise() - # Start task in the `sentinel` logcontext, to avoid leaking the current context - # into the reactor once it finishes. - with context.PreserveLoggingContext(): - # We can ignore the lint here since this class is the one location callLater should - # be called. - self._reactor.callLater( - duration.as_secs(), - lambda _: _try_wakeup_deferred(d), - duration.as_secs(), - ) # type: ignore[call-later-not-tracked] - await d - - def time(self) -> float: - """Returns the current system time in seconds since epoch.""" - return self._reactor.seconds() - - def time_msec(self) -> int: - """Returns the current system time in milliseconds since epoch.""" - return int(self.time() * 1000) - - def looping_call( - self, - f: Callable[P, object], - duration: Duration, - *args: P.args, - **kwargs: P.kwargs, - ) -> LoopingCall: - """Call a function repeatedly. - - Waits `duration` initially before calling `f` for the first time. - - If the function given to `looping_call` returns an awaitable/deferred, the next - call isn't scheduled until after the returned awaitable has finished. We get - this functionality thanks to this function being a thin wrapper around - `twisted.internet.task.LoopingCall`. - - Note that the function will be called with generic `looping_call` logcontext, so - if it is anything other than a trivial task, you probably want to wrap it in - `run_as_background_process` to give it more specific label and track metrics. - - Args: - f: The function to call repeatedly. - duration: How long to wait between calls. - *args: Positional arguments to pass to function. - **kwargs: Key arguments to pass to function. - """ - return self._looping_call_common(f, duration, False, *args, **kwargs) - - def looping_call_now( - self, - f: Callable[P, object], - duration: Duration, - *args: P.args, - **kwargs: P.kwargs, - ) -> LoopingCall: - """Call a function immediately, and then repeatedly thereafter. - - As with `looping_call`: subsequent calls are not scheduled until after the - the Awaitable returned by a previous call has finished. - - Note that the function will be called with generic `looping_call` logcontext, so - if it is anything other than a trivial task, you probably want to wrap it in - `run_as_background_process` to give it more specific label and track metrics. - - Args: - f: The function to call repeatedly. - duration: How long to wait between calls. - *args: Positional arguments to pass to function. - **kwargs: Key arguments to pass to function. - """ - return self._looping_call_common(f, duration, True, *args, **kwargs) - - def _looping_call_common( - self, - f: Callable[P, object], - duration: Duration, - now: bool, - *args: P.args, - **kwargs: P.kwargs, - ) -> LoopingCall: - """Common functionality for `looping_call` and `looping_call_now`""" - instance_id = random_string_insecure_fast(5) - - if self._is_shutdown: - raise Exception("Cannot start looping call. Clock has been shutdown") - - looping_call_context_string = "looping_call" - if now: - looping_call_context_string = "looping_call_now" - - @wraps(f) - def wrapped_f(*args: P.args, **kwargs: P.kwargs) -> Deferred: - clock_debug_logger.debug( - "%s(%s): Executing callback", looping_call_context_string, instance_id - ) - - # Because this is a callback from the reactor, we will be using the - # `sentinel` log context at this point. We want the function to log with - # some logcontext as we want to know which server the logs came from. - # - # We use `PreserveLoggingContext` to prevent our new `looping_call` - # logcontext from finishing as soon as we exit this function, in case `f` - # returns an awaitable/deferred which would continue running and may try to - # restore the `loop_call` context when it's done (because it's trying to - # adhere to the Synapse logcontext rules.) - # - # This also ensures that we return to the `sentinel` context when we exit - # this function and yield control back to the reactor to avoid leaking the - # current logcontext to the reactor (which would then get picked up and - # associated with the next thing the reactor does) - with context.PreserveLoggingContext( - context.LoggingContext( - name="looping_call", server_name=self._server_name - ) - ): - # We use `run_in_background` to reset the logcontext after `f` (or the - # awaitable returned by `f`) completes to avoid leaking the current - # logcontext to the reactor - return context.run_in_background(f, *args, **kwargs) - - # We can ignore the lint here since this is the one location LoopingCall's - # should be created. - call = task.LoopingCall(wrapped_f, *args, **kwargs) # type: ignore[prefer-synapse-clock-looping-call] - call.clock = self._reactor - # If `now=true`, the function will be called here immediately so we need to be - # in the sentinel context now. - # - # We want to start the task in the `sentinel` logcontext, to avoid leaking the - # current context into the reactor after the function finishes. - with context.PreserveLoggingContext(): - d = call.start(duration.as_secs(), now=now) - if hasattr(d, 'addErrback'): - d.addErrback(log_failure, "Looping call died", consumeErrors=False) - elif hasattr(d, 'add_done_callback'): - def _log_err(f: Any) -> None: - exc = f.exception() if hasattr(f, 'exception') else None - if exc: - logger.exception("Looping call died", exc_info=exc) - d.add_done_callback(_log_err) - self._looping_calls.add(call) - - clock_debug_logger.debug( - "%s(%s): Scheduled looping call every %sms later", - looping_call_context_string, - instance_id, - duration.as_millis(), - # Find out who is scheduling the call which makes it easy to follow in the - # logs. - stack_info=True, - ) - - return call - - def cancel_all_looping_calls(self, consumeErrors: bool = True) -> None: - """ - Stop all running looping calls. - - Args: - consumeErrors: Whether to re-raise errors encountered when cancelling the - scheduled call. - """ - for call in self._looping_calls: - try: - call.stop() - except Exception: - if not consumeErrors: - raise - self._looping_calls.clear() - - def call_later( - self, - delay: Duration, - callback: Callable, - *args: Any, - call_later_cancel_on_shutdown: bool = True, - **kwargs: Any, - ) -> "DelayedCallWrapper": - """Call something later - - Note that the function will be called with generic `call_later` logcontext, so - if it is anything other than a trivial task, you probably want to wrap it in - `run_as_background_process` to give it more specific label and track metrics. - - Args: - delay: How long to wait. - callback: Function to call - *args: Postional arguments to pass to function. - call_later_cancel_on_shutdown: Whether this call should be tracked for cleanup during - shutdown. In general, all calls should be tracked. There may be a use case - not to track calls with a `timeout` of 0 (or similarly short) since tracking - them may result in rapid insertions and removals of tracked calls - unnecessarily. But unless a specific instance of tracking proves to be an - issue, we can just track all delayed calls. - **kwargs: Key arguments to pass to function. - """ - call_id = self._delayed_call_id - self._delayed_call_id = self._delayed_call_id + 1 - - if self._is_shutdown: - raise Exception("Cannot start delayed call. Clock has been shutdown") - - @wraps(callback) - def wrapped_callback(*args: Any, **kwargs: Any) -> None: - clock_debug_logger.debug("call_later(%s): Executing callback", call_id) - - # Because this is a callback from the reactor, we will be using the - # `sentinel` log context at this point. We want the function to log with - # some logcontext as we want to know which server the logs came from. - # - # We use `PreserveLoggingContext` to prevent our new `call_later` - # logcontext from finishing as soon as we exit this function, in case `f` - # returns an awaitable/deferred which would continue running and may try to - # restore the `call_later` context when it's done (because it's trying to - # adhere to the Synapse logcontext rules.) - # - # This also ensures that we return to the `sentinel` context when we exit - # this function and yield control back to the reactor to avoid leaking the - # current logcontext to the reactor (which would then get picked up and - # associated with the next thing the reactor does) - try: - with context.PreserveLoggingContext( - context.LoggingContext( - name="call_later", server_name=self._server_name - ) - ): - # We use `run_in_background` to reset the logcontext after `f` (or the - # awaitable returned by `f`) completes to avoid leaking the current - # logcontext to the reactor - context.run_in_background(callback, *args, **kwargs) - finally: - if call_later_cancel_on_shutdown: - # We still want to remove the call from the tracking map. Even if - # the callback raises an exception. - self._call_id_to_delayed_call.pop(call_id) - - # We can ignore the lint here since this class is the one location callLater should - # be called. - call = self._reactor.callLater( - delay.as_secs(), wrapped_callback, *args, **kwargs - ) # type: ignore[call-later-not-tracked] - - clock_debug_logger.debug( - "call_later(%s): Scheduled call for %ss later (tracked for shutdown: %s)", - call_id, - delay, - call_later_cancel_on_shutdown, - # Find out who is scheduling the call which makes it easy to follow in the - # logs. - stack_info=True, - ) - - wrapped_call = DelayedCallWrapper(call, call_id, self) - if call_later_cancel_on_shutdown: - self._call_id_to_delayed_call[call_id] = wrapped_call - - return wrapped_call - - def cancel_call_later( - self, wrapped_call: "DelayedCallWrapper", ignore_errs: bool = False - ) -> None: - try: - clock_debug_logger.debug( - "cancel_call_later: cancelling scheduled call %s", wrapped_call.call_id - ) - wrapped_call.delayed_call.cancel() - except Exception: - if not ignore_errs: - raise - - def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: - """ - Stop all scheduled calls that were marked with `cancel_on_shutdown` when they were created. - - Args: - ignore_errs: Whether to re-raise errors encountered when cancelling the - scheduled call. - """ - # We make a copy here since calling `cancel()` on a delayed_call - # will result in the call removing itself from the map mid-iteration. - for call_id, call in list(self._call_id_to_delayed_call.items()): - try: - clock_debug_logger.debug( - "cancel_all_delayed_calls: cancelling scheduled call %s", call_id - ) - call.cancel() - except Exception: - if not ignore_errs: - raise - self._call_id_to_delayed_call.clear() - - def call_when_running( - self, - callback: Callable[P, object], - *args: P.args, - **kwargs: P.kwargs, - ) -> None: - """ - Call a function when the reactor is running. - - If the reactor has not started, the callable will be scheduled to run when it - does start. Otherwise, the callable will be invoked immediately. - - Args: - callback: Function to call - *args: Postional arguments to pass to function. - **kwargs: Key arguments to pass to function. - """ - instance_id = random_string_insecure_fast(5) - - def wrapped_callback(*args: Any, **kwargs: Any) -> None: - clock_debug_logger.debug( - "call_when_running(%s): Executing callback", instance_id - ) - - # Since this callback can be invoked immediately if the reactor is already - # running, we can't always assume that we're running in the sentinel - # logcontext (i.e. we can't assert that we're in the sentinel context like - # we can in other methods). - # - # We will only be running in the sentinel logcontext if the reactor was not - # running when `call_when_running` was invoked and later starts up. - # - # assert context.current_context() is context.SENTINEL_CONTEXT - - # Because this is a callback from the reactor, we will be using the - # `sentinel` log context at this point. We want the function to log with - # some logcontext as we want to know which server the logs came from. - # - # We use `PreserveLoggingContext` to prevent our new `call_when_running` - # logcontext from finishing as soon as we exit this function, in case `f` - # returns an awaitable/deferred which would continue running and may try to - # restore the `loop_call` context when it's done (because it's trying to - # adhere to the Synapse logcontext rules.) - # - # This also ensures that we return to the `sentinel` context when we exit - # this function and yield control back to the reactor to avoid leaking the - # current logcontext to the reactor (which would then get picked up and - # associated with the next thing the reactor does) - with context.PreserveLoggingContext( - context.LoggingContext( - name="call_when_running", server_name=self._server_name - ) - ): - # We use `run_in_background` to reset the logcontext after `f` (or the - # awaitable returned by `f`) completes to avoid leaking the current - # logcontext to the reactor - context.run_in_background(callback, *args, **kwargs) - - # We can ignore the lint here since this class is the one location - # callWhenRunning should be called. - self._reactor.callWhenRunning(wrapped_callback, *args, **kwargs) # type: ignore[prefer-synapse-clock-call-when-running] - - clock_debug_logger.debug( - "call_when_running(%s): Scheduled call", - instance_id, - # Find out who is scheduling the call which makes it easy to follow in the - # logs. - stack_info=True, - ) - - def add_system_event_trigger( - self, - phase: str, - event_type: str, - callback: Callable[P, object], - *args: P.args, - **kwargs: P.kwargs, - ) -> Any: - """ - Add a function to be called when a system event occurs. - - Equivalent to `reactor.addSystemEventTrigger` (see the that docstring for more - details), but ensures that the callback is run in a logging context. - - Args: - phase: a time to call the event -- either the string 'before', 'after', or - 'during', describing when to call it relative to the event's execution. - eventType: this is a string describing the type of event. - callback: Function to call - *args: Postional arguments to pass to function. - **kwargs: Key arguments to pass to function. - - Returns: - an ID that can be used to remove this call with `reactor.removeSystemEventTrigger`. - """ - instance_id = random_string_insecure_fast(5) - - def wrapped_callback(*args: Any, **kwargs: Any) -> None: - clock_debug_logger.debug( - "add_system_event_trigger(%s): Executing %s %s callback", - instance_id, - phase, - event_type, - ) - - # Because this is a callback from the reactor, we will be using the - # `sentinel` log context at this point. We want the function to log with - # some logcontext as we want to know which server the logs came from. - # - # We use `PreserveLoggingContext` to prevent our new `system_event` - # logcontext from finishing as soon as we exit this function, in case `f` - # returns an awaitable/deferred which would continue running and may try to - # restore the `loop_call` context when it's done (because it's trying to - # adhere to the Synapse logcontext rules.) - # - # This also ensures that we return to the `sentinel` context when we exit - # this function and yield control back to the reactor to avoid leaking the - # current logcontext to the reactor (which would then get picked up and - # associated with the next thing the reactor does) - with context.PreserveLoggingContext( - context.LoggingContext( - name="system_event", server_name=self._server_name - ) - ): - # We use `run_in_background` to reset the logcontext after `f` (or the - # awaitable returned by `f`) completes to avoid leaking the current - # logcontext to the reactor - context.run_in_background(callback, *args, **kwargs) - - clock_debug_logger.debug( - "add_system_event_trigger(%s) for %s %s", - instance_id, - phase, - event_type, - # Find out who is scheduling the call which makes it easy to follow in the - # logs. - stack_info=True, - ) - - # We can ignore the lint here since this class is the one location - # `addSystemEventTrigger` should be called. - return self._reactor.addSystemEventTrigger( - phase, event_type, wrapped_callback, *args, **kwargs - ) # type: ignore[prefer-synapse-clock-add-system-event-trigger] - - -class DelayedCallWrapper: - """Wraps an `Any` so that we can intercept the call to `cancel()` and - properly cleanup the delayed call from the tracking map of the `Clock`. - - args: - delayed_call: The actual `Any` - call_id: Unique identifier for this delayed call - clock: The clock instance tracking this call - """ - - def __init__(self, delayed_call: Any, call_id: int, clock: Clock): - self.delayed_call = delayed_call - self.call_id = call_id - self.clock = clock - - def cancel(self) -> None: - """Remove the call from the tracking map and propagate the call to the - underlying delayed_call. - """ - self.delayed_call.cancel() - try: - self.clock._call_id_to_delayed_call.pop(self.call_id) - except KeyError: - # If the delayed call isn't being tracked anymore we can just move on. - pass - - def getTime(self) -> float: - """Propagate the call to the underlying delayed_call.""" - return self.delayed_call.getTime() - - def delay(self, secondsLater: float) -> None: - """Propagate the call to the underlying delayed_call.""" - self.delayed_call.delay(secondsLater) - - def reset(self, secondsFromNow: float) -> None: - """Propagate the call to the underlying delayed_call.""" - self.delayed_call.reset(secondsFromNow) - - def active(self) -> bool: - """Propagate the call to the underlying delayed_call.""" - return self.delayed_call.active() - - -# =========================================================================== -# Phase 2: asyncio-native Clock implementation -# -# NativeClock provides the same public interface as Clock but uses asyncio -# primitives instead of Twisted. It is unused until later phases switch -# hs.get_clock() to return a NativeClock. -# =========================================================================== - - class NativeLoopingCall: """asyncio-native equivalent of Twisted's LoopingCall. @@ -707,10 +144,10 @@ class NativeDelayedCallWrapper: class NativeClock: - """asyncio-native equivalent of Clock. + """asyncio-native Clock implementation. - Provides the same public interface as Clock but uses asyncio primitives - (asyncio.sleep, loop.call_later, asyncio.create_task) instead of Twisted. + Provides scheduling utilities (looping calls, delayed calls, sleep) using + asyncio primitives (asyncio.sleep, loop.call_later, asyncio.create_task). Args: server_name: The server name for logging context. @@ -983,3 +420,8 @@ class NativeClock: stack_info=True, ) return trigger_id + + +# Backward-compatible aliases so existing imports continue to work. +Clock = NativeClock +DelayedCallWrapper = NativeDelayedCallWrapper diff --git a/synapse/util/file_consumer.py b/synapse/util/file_consumer.py index c7591e8ca6..974e2f9827 100644 --- a/synapse/util/file_consumer.py +++ b/synapse/util/file_consumer.py @@ -18,15 +18,11 @@ # # +import asyncio import queue from typing import Any, BinaryIO, Optional, Union, cast -try: - from twisted.internet.defer import Deferred -except ImportError: - from asyncio import Future as Deferred # type: ignore[assignment] - -from synapse.logging.context import make_deferred_yieldable, run_in_background +from synapse.logging.context import make_deferred_yieldable from synapse.types import ISynapseReactor @@ -64,10 +60,10 @@ class BackgroundFileConsumer: # unregister a final None is sent. self._bytes_queue: queue.Queue[bytes | None] = queue.Queue() - # Deferred that is resolved when finished writing + # asyncio.Future that is resolved when finished writing # - # This is really Deferred[None], but mypy doesn't seem to like that. - self._finished_deferred: Deferred[Any] | None = None + # This is really asyncio.Future[None], but mypy doesn't seem to like that. + self._finished_deferred: asyncio.Future[Any] | None = None # If the _writer thread throws an exception it gets stored here. self._write_exception: Exception | None = None @@ -87,16 +83,8 @@ class BackgroundFileConsumer: self._producer = producer self.streaming = streaming - self._finished_deferred = run_in_background( - threads.deferToThreadPool, - # mypy seems to get confused with the chaining of ParamSpec from - # run_in_background to deferToThreadPool. - # - # For Twisted trunk, ignore arg-type; for Twisted release ignore unused-ignore. - self._reactor, # type: ignore[arg-type,unused-ignore] - self._reactor.getThreadPool(), # type: ignore[arg-type,unused-ignore] - self._writer, # type: ignore[arg-type,unused-ignore] - ) + self._loop = asyncio.get_event_loop() + self._finished_deferred = self._loop.run_in_executor(None, self._writer) if not streaming: self._producer.resumeProducing() @@ -104,7 +92,7 @@ class BackgroundFileConsumer: """Part of IProducer interface""" self._producer = None assert self._finished_deferred is not None - if not self._finished_deferred.called: + if not self._finished_deferred.done(): self._bytes_queue.put_nowait(None) def write(self, write_bytes: bytes) -> None: @@ -113,7 +101,7 @@ class BackgroundFileConsumer: raise self._write_exception assert self._finished_deferred is not None - if self._finished_deferred.called: + if self._finished_deferred.done(): raise Exception("consumer has closed") self._bytes_queue.put_nowait(write_bytes) @@ -128,13 +116,14 @@ class BackgroundFileConsumer: def _writer(self) -> None: """This is run in a background thread to write to the file.""" + loop = self._loop try: while self._producer or not self._bytes_queue.empty(): # If we've paused the producer check if we should resume the # producer. if self._producer and self._paused_producer: if self._bytes_queue.qsize() <= self._RESUME_ON_QUEUE_SIZE: - self._reactor.callFromThread(self._resume_paused_producer) + loop.call_soon_threadsafe(self._resume_paused_producer) bytes = self._bytes_queue.get() @@ -146,14 +135,14 @@ class BackgroundFileConsumer: # If its a pull producer then we need to explicitly ask for # more stuff. if not self.streaming and self._producer: - self._reactor.callFromThread(self._producer.resumeProducing) + loop.call_soon_threadsafe(self._producer.resumeProducing) except Exception as e: self._write_exception = e raise finally: self._file_obj.close() - def wait(self) -> "Deferred[None]": + def wait(self) -> "asyncio.Future[None]": """Returns a deferred that resolves when finished writing to file""" assert self._finished_deferred is not None return make_deferred_yieldable(self._finished_deferred) diff --git a/synapse/util/gai_resolver.py b/synapse/util/gai_resolver.py index 41e90a8ce6..9be54ef26c 100644 --- a/synapse/util/gai_resolver.py +++ b/synapse/util/gai_resolver.py @@ -1,7 +1,8 @@ -# This is a direct lift from -# https://github.com/twisted/twisted/blob/release-21.2.0-10091/src/twisted/internet/_resolver.py. -# We copy it here as we need to instantiate `GAIResolver` manually, but it is a -# private class. +# This module previously contained a copy of Twisted's GAIResolver. +# It has been replaced with a thin wrapper around asyncio's getaddrinfo. +# +# For backward compatibility, the GAIResolver class is still provided but +# now uses asyncio internally when Twisted is not available. from socket import ( AF_INET, @@ -16,17 +17,16 @@ from socket import ( ) from typing import ( TYPE_CHECKING, + Any, Callable, NoReturn, Sequence, ) +# Try Twisted first for backward compatibility try: from zope.interface import implementer -except ImportError: - pass -try: from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.interfaces import ( IAddress, @@ -36,155 +36,128 @@ try: IResolutionReceiver, ) from twisted.internet.threads import deferToThreadPool -except ImportError: - pass -if TYPE_CHECKING: - # The types below are copied from - # https://github.com/twisted/twisted/blob/release-21.2.0-10091/src/twisted/internet/interfaces.py - # so that the type hints can match the interfaces. - try: - from twisted.python.runtime import platform - except ImportError: - platform = None # type: ignore[assignment] - - if platform and platform.supportsThreads(): + if TYPE_CHECKING: try: - from twisted.python.threadpool import ThreadPool + from twisted.python.runtime import platform except ImportError: - ThreadPool = object # type: ignore[misc, assignment] - else: - ThreadPool = object # type: ignore[misc, assignment] + platform = None # type: ignore[assignment] - -@implementer(IHostResolution) -class HostResolution: - """ - The in-progress resolution of a given hostname. - """ - - def __init__(self, name: str): - """ - Create a L{HostResolution} with the given name. - """ - self.name = name - - def cancel(self) -> NoReturn: - # IHostResolution.cancel - raise NotImplementedError() - - -_any = frozenset([IPv4Address, IPv6Address]) - -_typesToAF = { - frozenset([IPv4Address]): AF_INET, - frozenset([IPv6Address]): AF_INET6, - _any: AF_UNSPEC, -} - -_afToType = { - AF_INET: IPv4Address, - AF_INET6: IPv6Address, -} - -_transportToSocket = { - "TCP": SOCK_STREAM, - "UDP": SOCK_DGRAM, -} - -_socktypeToType = { - SOCK_STREAM: "TCP", - SOCK_DGRAM: "UDP", -} - - -_GETADDRINFO_RESULT = list[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], - ] -] - - -@implementer(IHostnameResolver) -class GAIResolver: - """ - L{IHostnameResolver} implementation that resolves hostnames by calling - L{getaddrinfo} in a thread. - """ - - def __init__( - self, - reactor: IReactorThreads, - getThreadPool: Callable[[], "ThreadPool"] | None = None, - getaddrinfo: Callable[[str, int, int, int], _GETADDRINFO_RESULT] = getaddrinfo, - ): - """ - Create a L{GAIResolver}. - @param reactor: the reactor to schedule result-delivery on - @type reactor: L{IReactorThreads} - @param getThreadPool: a function to retrieve the thread pool to use for - scheduling name resolutions. If not supplied, the use the given - C{reactor}'s thread pool. - @type getThreadPool: 0-argument callable returning a - L{twisted.python.threadpool.ThreadPool} - @param getaddrinfo: a reference to the L{getaddrinfo} to use - mainly - parameterized for testing. - @type getaddrinfo: callable with the same signature as L{getaddrinfo} - """ - self._reactor = reactor - self._getThreadPool = ( - reactor.getThreadPool if getThreadPool is None else getThreadPool - ) - self._getaddrinfo = getaddrinfo - - # The types on IHostnameResolver is incorrect in Twisted, see - # https://twistedmatrix.com/trac/ticket/10276 - def resolveHostName( - self, - resolutionReceiver: IResolutionReceiver, - hostName: str, - portNumber: int = 0, - addressTypes: Sequence[type[IAddress]] | None = None, - transportSemantics: str = "TCP", - ) -> IHostResolution: - """ - See L{IHostnameResolver.resolveHostName} - @param resolutionReceiver: see interface - @param hostName: see interface - @param portNumber: see interface - @param addressTypes: see interface - @param transportSemantics: see interface - @return: see interface - """ - pool = self._getThreadPool() - addressFamily = _typesToAF[ - _any if addressTypes is None else frozenset(addressTypes) - ] - socketType = _transportToSocket[transportSemantics] - - def get() -> _GETADDRINFO_RESULT: + if platform and platform.supportsThreads(): try: - return self._getaddrinfo( - hostName, portNumber, addressFamily, socketType - ) - except gaierror: - return [] + from twisted.python.threadpool import ThreadPool + except ImportError: + ThreadPool = object # type: ignore[misc, assignment] + else: + ThreadPool = object # type: ignore[misc, assignment] - d = deferToThreadPool(self._reactor, pool, get) - resolution = HostResolution(hostName) - resolutionReceiver.resolutionBegan(resolution) - @d.addCallback - def deliverResults(result: _GETADDRINFO_RESULT) -> None: - for family, socktype, _proto, _cannoname, sockaddr in result: - addrType = _afToType[family] - resolutionReceiver.addressResolved( - addrType(_socktypeToType.get(socktype, "TCP"), *sockaddr) - ) - resolutionReceiver.resolutionComplete() + @implementer(IHostResolution) + class HostResolution: + """ + The in-progress resolution of a given hostname. + """ - return resolution + def __init__(self, name: str): + self.name = name + + def cancel(self) -> NoReturn: + raise NotImplementedError() + + + _any = frozenset([IPv4Address, IPv6Address]) + + _typesToAF = { + frozenset([IPv4Address]): AF_INET, + frozenset([IPv6Address]): AF_INET6, + _any: AF_UNSPEC, + } + + _afToType = { + AF_INET: IPv4Address, + AF_INET6: IPv6Address, + } + + _transportToSocket = { + "TCP": SOCK_STREAM, + "UDP": SOCK_DGRAM, + } + + _socktypeToType = { + SOCK_STREAM: "TCP", + SOCK_DGRAM: "UDP", + } + + _GETADDRINFO_RESULT = list[ + tuple[ + AddressFamily, + SocketKind, + int, + str, + tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], + ] + ] + + + @implementer(IHostnameResolver) + class GAIResolver: + """ + L{IHostnameResolver} implementation that resolves hostnames by calling + L{getaddrinfo} in a thread. + """ + + def __init__( + self, + reactor: IReactorThreads, + getThreadPool: Callable[[], "ThreadPool"] | None = None, + getaddrinfo: Callable[[str, int, int, int], _GETADDRINFO_RESULT] = getaddrinfo, + ): + self._reactor = reactor + self._getThreadPool = ( + reactor.getThreadPool if getThreadPool is None else getThreadPool + ) + self._getaddrinfo = getaddrinfo + + def resolveHostName( + self, + resolutionReceiver: IResolutionReceiver, + hostName: str, + portNumber: int = 0, + addressTypes: Sequence[type[IAddress]] | None = None, + transportSemantics: str = "TCP", + ) -> IHostResolution: + pool = self._getThreadPool() + addressFamily = _typesToAF[ + _any if addressTypes is None else frozenset(addressTypes) + ] + socketType = _transportToSocket[transportSemantics] + + def get() -> _GETADDRINFO_RESULT: + try: + return self._getaddrinfo( + hostName, portNumber, addressFamily, socketType + ) + except gaierror: + return [] + + d = deferToThreadPool(self._reactor, pool, get) + resolution = HostResolution(hostName) + resolutionReceiver.resolutionBegan(resolution) + + @d.addCallback + def deliverResults(result: _GETADDRINFO_RESULT) -> None: + for family, socktype, _proto, _cannoname, sockaddr in result: + addrType = _afToType[family] + resolutionReceiver.addressResolved( + addrType(_socktypeToType.get(socktype, "TCP"), *sockaddr) + ) + resolutionReceiver.resolutionComplete() + + return resolution + +except ImportError: + # Twisted is not available. Provide a no-op stub. + class GAIResolver: # type: ignore[no-redef] + """Stub: Twisted is not installed. Use asyncio.get_event_loop().getaddrinfo() instead.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass diff --git a/synapse/util/httpresourcetree.py b/synapse/util/httpresourcetree.py index 56420c8e1d..c15c28392c 100644 --- a/synapse/util/httpresourcetree.py +++ b/synapse/util/httpresourcetree.py @@ -23,10 +23,7 @@ import logging from synapse.http.server import UnrecognizedRequestResource -try: - from twisted.web.resource import Resource -except ImportError: - Resource = object # type: ignore[assignment,misc] +from synapse.http.resource import Resource logger = logging.getLogger(__name__) diff --git a/synapse/util/manhole.py b/synapse/util/manhole.py index ecd2600188..080ba012d1 100644 --- a/synapse/util/manhole.py +++ b/synapse/util/manhole.py @@ -24,6 +24,7 @@ import sys import traceback from typing import Any +_TWISTED_MANHOLE_AVAILABLE = False try: from twisted.conch import manhole_ssh from twisted.conch.insults import insults @@ -32,6 +33,7 @@ try: from twisted.cred import checkers, portal from twisted.internet import defer from twisted.internet.protocol import ServerFactory + _TWISTED_MANHOLE_AVAILABLE = True except ImportError: pass @@ -75,7 +77,7 @@ EddTrx3TNpr1D5m/f+6mnXWrc8u9y1+GNx9yz889xMjIBTBI9KqaaOs= -----END RSA PRIVATE KEY-----""" -def manhole(settings: ManholeConfig, globals: dict[str, Any]) -> ServerFactory: +def manhole(settings: ManholeConfig, globals: dict[str, Any]) -> Any: """Starts a ssh listener with password authentication using the given username and password. Clients connecting to the ssh listener will find themselves in a colored python shell with @@ -89,6 +91,12 @@ def manhole(settings: ManholeConfig, globals: dict[str, Any]) -> ServerFactory: Returns: A factory to pass to ``listenTCP`` """ + if not _TWISTED_MANHOLE_AVAILABLE: + raise ImportError( + "Twisted with conch support is required for the manhole feature. " + "Install it with: pip install twisted[conch]" + ) + username = settings.username password = settings.password.encode("ascii") priv_key = settings.priv_key diff --git a/synmark/suites/logging.py b/synmark/suites/logging.py index 243f9dbca0..f523bb017b 100644 --- a/synmark/suites/logging.py +++ b/synmark/suites/logging.py @@ -115,7 +115,7 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: } logger = logging.getLogger("synapse") - one_time_logging_setup(logBeginner=beginner) + one_time_logging_setup() _setup_stdlib_logging( hs_config, # type: ignore[arg-type] None,