From 3cb1bba904dfac6a57ebc102e2d4c88e6effdf5d Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 28 Jul 2026 08:52:10 +0100 Subject: [PATCH] Add /ready endpoint --- synapse/app/generic_worker.py | 5 ++ synapse/app/homeserver.py | 9 ++- synapse/http/site.py | 2 +- synapse/rest/ready.py | 81 ++++++++++++++++++++ tests/rest/test_ready.py | 137 ++++++++++++++++++++++++++++++++++ 5 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 synapse/rest/ready.py create mode 100644 tests/rest/test_ready.py diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 159cd44237..41b8e9df16 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -55,6 +55,7 @@ from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource from synapse.rest import ClientRestResource, admin from synapse.rest.health import HealthResource from synapse.rest.key.v2 import KeyResource +from synapse.rest.ready import ReadyResource from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.rest.well_known import well_known_resource from synapse.server import HomeServer @@ -187,6 +188,7 @@ class GenericWorkerServer(HomeServer): resources: dict[str, Resource] = { # We always include a health resource. "/health": HealthResource(), + "/ready": ReadyResource(self), "/_synapse/admin": admin_resource, } @@ -243,6 +245,9 @@ class GenericWorkerServer(HomeServer): elif name == "health": # Skip loading, health resource is always included continue + elif name == "ready": + # Skip loading, ready resource is always included + continue if name == "openid" and "federation" not in res.names: # Only load the openid resource separately if federation resource diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2b1760416b..98bf3ee76f 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -66,6 +66,7 @@ from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource from synapse.rest import ClientRestResource, admin from synapse.rest.health import HealthResource from synapse.rest.key.v2 import KeyResource +from synapse.rest.ready import ReadyResource from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.rest.well_known import well_known_resource from synapse.server import HomeServer @@ -98,7 +99,10 @@ class SynapseHomeServer(HomeServer): site_tag = listener_config.get_site_tag() # We always include a health resource. - resources: dict[str, Resource] = {"/health": HealthResource()} + resources: dict[str, Resource] = { + "/health": HealthResource(), + "/ready": ReadyResource(self), + } for res in listener_config.http_options.resources: for name in res.names: @@ -115,6 +119,9 @@ class SynapseHomeServer(HomeServer): if name == "health": # Skip loading, health resource is always included continue + if name == "ready": + # Skip loading, ready resource is always included + continue resources.update(self._configure_named_resource(name, res.compress)) additional_resources = listener_config.http_options.additional_resources diff --git a/synapse/http/site.py b/synapse/http/site.py index 9b7fd5c936..a6df3c670e 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -675,7 +675,7 @@ class SynapseRequest(Request): def _should_log_request(self) -> bool: """Whether we should log at INFO that we processed the request.""" - if self.path == b"/health": + if self.path in (b"/health", b"/ready"): return False if self.method == b"OPTIONS": diff --git a/synapse/rest/ready.py b/synapse/rest/ready.py new file mode 100644 index 0000000000..5b8b383e79 --- /dev/null +++ b/synapse/rest/ready.py @@ -0,0 +1,81 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from typing import TYPE_CHECKING + +from twisted.web.server import Request + +from synapse.api.errors import UnrecognizedRequestError +from synapse.http.server import DirectServeJsonResource +from synapse.types import JsonDict + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ReadyResource(DirectServeJsonResource): + """A resource which reports whether this process is ready to serve traffic. + + This endpoint reflects whether Synapse can handle traffic, rather than if it + is "up" (which /health covers). It is intended for use as a readiness probe + that removes an instance from load-balancer rotation, not as a liveness + probe that restarts it. + + Note: `SynapseRequest._should_log_request` ensures that requests to + `/ready` do not get logged at INFO. + """ + + isLeaf = True + + def __init__(self, hs: "HomeServer"): + super().__init__(clock=hs.get_clock()) + self._hs = hs + self._store = hs.get_datastores().main + self._is_worker = hs.config.worker.worker_app is not None + self._replication_handler = hs.get_replication_command_handler() + + async def _async_render_GET(self, request: Request) -> tuple[int, JsonDict]: + # Prevent path traversal by ensuring the request path is exactly /ready. + if request.path != b"/ready": + raise UnrecognizedRequestError(code=404) + + db_ok = await self._check_db() + + # A worker isn't ready until it can reach the main process (whether via + # direct TCP replication or Redis pub/sub). The main process itself + # isn't gated on this: it having zero attached workers/Redis clients + # right now doesn't make it unhealthy. + replication_ok = ( + self._replication_handler.connected() if self._is_worker else True + ) + + startup_ok = self._hs.is_synapse_started() + + checks = { + "db": db_ok, + "replication": replication_ok, + "startup_complete": startup_ok, + } + return (200 if all(checks.values()) else 503, checks) + + async def _check_db(self) -> bool: + try: + await self._store.db_pool.execute("ready_check", "SELECT 1") + return True + except Exception: + logger.warning("Readiness check: database unreachable", exc_info=True) + return False diff --git a/tests/rest/test_ready.py b/tests/rest/test_ready.py new file mode 100644 index 0000000000..aecd0b2a30 --- /dev/null +++ b/tests/rest/test_ready.py @@ -0,0 +1,137 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from unittest import mock + +from twisted.internet.testing import MemoryReactor + +from synapse.app.generic_worker import GenericWorkerServer +from synapse.rest.ready import ReadyResource +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock + +from tests import unittest + + +class ReadyCheckTests(unittest.HomeserverTestCase): + def create_test_resource(self) -> ReadyResource: + # replace the JsonResource with a ReadyResource. + return ReadyResource(self.hs) + + def test_ready_not_started(self) -> None: + """Before startup has completed, /ready should report unready.""" + channel = self.make_request("GET", "/ready", shorthand=False) + + self.assertEqual(channel.code, 503) + self.assertEqual( + channel.json_body, + {"db": True, "replication": True, "startup_complete": False}, + ) + + def test_ready_all_ok(self) -> None: + self.hs.set_synapse_started() + + channel = self.make_request("GET", "/ready", shorthand=False) + + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.json_body, + {"db": True, "replication": True, "startup_complete": True}, + ) + + def test_ready_path_traversal(self) -> None: + """ + Test that the ready endpoint does not allow extra path segments, + which could be used to access other resources. + """ + channel = self.make_request("GET", "/ready/extra/path", shorthand=False) + + self.assertEqual(channel.code, 404) + self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED") + self.assertIn("error", channel.json_body) + + def test_ready_db_down(self) -> None: + self.hs.set_synapse_started() + + store = self.hs.get_datastores().main + with mock.patch.object(store.db_pool, "execute", side_effect=Exception("boom")): + channel = self.make_request("GET", "/ready", shorthand=False) + + self.assertEqual(channel.code, 503) + self.assertEqual( + channel.json_body, + {"db": False, "replication": True, "startup_complete": True}, + ) + + def test_ready_main_process_ignores_replication_connected(self) -> None: + """The main process shouldn't be considered unready just because no + workers/redis clients happen to be attached to it right now.""" + self.hs.set_synapse_started() + + with mock.patch.object( + self.hs.get_replication_command_handler(), + "connected", + return_value=False, + ): + channel = self.make_request("GET", "/ready", shorthand=False) + + self.assertEqual(channel.code, 200) + self.assertTrue(channel.json_body["replication"]) + + +class WorkerReadyCheckTests(unittest.HomeserverTestCase): + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + return self.setup_test_homeserver(homeserver_to_use=GenericWorkerServer) + + def default_config(self) -> JsonDict: + conf = super().default_config() + conf["worker_app"] = "synapse.app.generic_worker" + conf["instance_map"] = {"main": {"host": "127.0.0.1", "port": 0}} + return conf + + def create_test_resource(self) -> ReadyResource: + return ReadyResource(self.hs) + + def test_ready_replication_down(self) -> None: + self.hs.set_synapse_started() + + with mock.patch.object( + self.hs.get_replication_command_handler(), + "connected", + return_value=False, + ): + channel = self.make_request("GET", "/ready", shorthand=False) + + self.assertEqual(channel.code, 503) + self.assertEqual( + channel.json_body, + {"db": True, "replication": False, "startup_complete": True}, + ) + + def test_ready_replication_up(self) -> None: + self.hs.set_synapse_started() + + with mock.patch.object( + self.hs.get_replication_command_handler(), + "connected", + return_value=True, + ): + channel = self.make_request("GET", "/ready", shorthand=False) + + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.json_body, + {"db": True, "replication": True, "startup_complete": True}, + )