synapse/tests/http/test_rust_federation_client.py

#
# This file is licensed under the Affero General Public License (AGPL) version 3.
#
# Copyright (C) 2026 Element Creations Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# See the GNU Affero General Public License for more details:
# <https://www.gnu.org/licenses/agpl-3.0.html>.
#
"""Tests for `MatrixFederationHttpClient` at `rust_federation_sending: client`,
where its JSON requests go through the Rust request layer.

The Rust client sends through a Python callable here, in place of the network.
It runs on a tokio thread, so the tests wait for it with `get_success`, and
advance the reactor to get past the sleeps between retries.
"""

import json
import re
import time
from collections import deque
from typing import Any

import attr
from signedjson.key import get_verify_key
from signedjson.sign import verify_signed_json

from twisted.internet.defer import Deferred, ensureDeferred
from twisted.internet.testing import MemoryReactor

from synapse.api.errors import (
    FederationDeniedError,
    HttpResponseException,
    RequestSendFailed,
)
from synapse.http.matrixfederationclient import ByteParser, LegacyJsonSendParser
from synapse.server import HomeServer
from synapse.synapse_rust.federation import (
    RustFederationClient,
    federation_gil_stats,
)
from synapse.types import JsonDict
from synapse.util.clock import Clock
from synapse.util.retryutils import NotRetryingDestination

from tests import unittest

DEST = "remote.example"


@attr.s(auto_attribs=True, frozen=True)
class SentRequest:
    destination: str
    method: str
    path: str
    headers: list[tuple[bytes, bytes]]
    body: bytes

    def header(self, name: bytes) -> bytes | None:
        for key, value in self.headers:
            if key == name.lower():
                return value
        return None


Response = tuple[int, list[tuple[bytes, bytes]], bytes]


def json_response(status: int, body: Any) -> Response:
    return status, [(b"content-type", b"application/json")], json.dumps(body).encode()


class FakeTransport:
    """Answers each request with the next queued response."""

    def __init__(self) -> None:
        self.responses: deque[Response] = deque()
        self.requests: list[SentRequest] = []

    def __call__(
        self,
        destination: str,
        method: str,
        path: str,
        headers: list[tuple[bytes, bytes]],
        body: bytes,
    ) -> Response:
        self.requests.append(SentRequest(destination, method, path, headers, body))
        if not self.responses:
            raise AssertionError(f"unexpected request {method} {path}")
        return self.responses.popleft()


class RustFederationClientTestCase(unittest.HomeserverTestCase):
    def default_config(self) -> dict[str, Any]:
        config = super().default_config()
        config.setdefault("experimental_features", {})["rust_federation_sending"] = (
            "client"
        )
        return config

    def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
        self.cl = hs.get_federation_http_client()
        assert self.cl._rust_client is not None

        self.transport = FakeTransport()
        self.cl._rust_client = RustFederationClient(
            hs,
            hs.version_string,
            hs.version_string,
            self.cl._write_retry_timings,
            self.cl._record_request_metrics,
            self.transport,
        )

    def _wait(self, d: "Deferred[Any]", step: float = 0.5, limit: int = 4000) -> None:
        """Advance the reactor in `step` second steps, giving tokio a moment
        after each, until `d` fires."""
        for _ in range(limit):
            if d.called and not d.paused:
                return
            self.reactor.advance(step)
            time.sleep(0.001)
        raise AssertionError("request did not finish")

    def test_get_json(self) -> None:
        self.transport.responses.append(json_response(200, {"foo": "bar"}))

        result = self.get_success(
            self.cl.get_json(DEST, "/_matrix/federation/v1/x", args={"a": ["1", "2"]})
        )

        self.assertEqual(result, {"foo": "bar"})
        (request,) = self.transport.requests
        self.assertEqual(request.destination, DEST)
        self.assertEqual(request.method, "GET")
        self.assertEqual(request.path, "/_matrix/federation/v1/x?a=1&a=2")
        self.assertEqual(request.body, b"")
        self.assertIsNone(request.header(b"Content-Type"))
        self.assertEqual(request.header(b"User-Agent"), self.hs.version_string.encode())
        self._assert_signed(request, None)

    def test_put_json_body(self) -> None:
        self.transport.responses.append(json_response(200, {}))

        self.get_success(
            self.cl.put_json(DEST, "/_matrix/federation/v1/y", data={"b": 2, "a": "é"})
        )

        (request,) = self.transport.requests
        self.assertEqual(request.method, "PUT")
        self.assertEqual(request.header(b"Content-Type"), b"application/json")
        self.assertEqual(request.body, '{"a":"é","b":2}'.encode())
        self._assert_signed(request, {"b": 2, "a": "é"})

    def _assert_signed(self, request: SentRequest, content: JsonDict | None) -> None:
        """Check the `X-Matrix` header, as the receiving server would."""
        header = request.header(b"Authorization")
        assert header is not None
        params = dict(re.findall(r'(\w+)="([^"]*)"', header.decode()))
        self.assertEqual(params["origin"], self.hs.hostname)
        self.assertEqual(params["destination"], DEST)

        signed: JsonDict = {
            "method": request.method,
            "uri": request.path,
            "origin": params["origin"],
            "destination": params["destination"],
            "signatures": {params["origin"]: {params["key"]: params["sig"]}},
        }
        if content is not None:
            signed["content"] = content
        verify_key = get_verify_key(self.hs.signing_key)
        verify_signed_json(signed, params["origin"], verify_key)

    def test_retries_server_errors(self) -> None:
        self.transport.responses.extend(
            [json_response(502, {}), json_response(500, {}), json_response(200, {})]
        )

        d = ensureDeferred(self.cl.get_json(DEST, "/x"))
        self._wait(d)

        self.assertEqual(self.successResultOf(d), {})
        self.assertEqual(len(self.transport.requests), 3)

    def test_client_error_raises(self) -> None:
        self.transport.responses.append(
            json_response(404, {"errcode": "M_NOT_FOUND", "error": "nope"})
        )

        failure = self.get_failure(self.cl.get_json(DEST, "/x"), HttpResponseException)

        self.assertEqual(failure.value.code, 404)
        self.assertEqual(failure.value.to_synapse_error().errcode, "M_NOT_FOUND")
        self.assertEqual(len(self.transport.requests), 1)

    def test_failures_start_a_backoff(self) -> None:
        """Running out of retries records a backoff, which the next request
        honours. This goes through the Python write and its cache
        invalidation, which reaches the Rust client's copy of the timings."""
        # The first attempt plus the three short retries.
        self.transport.responses.extend([json_response(500, {})] * 4)

        d = ensureDeferred(self.cl.get_json(DEST, "/x"))
        self._wait(d)
        failure = self.failureResultOf(d, RequestSendFailed)
        assert isinstance(failure.value, RequestSendFailed)
        self.assertTrue(failure.value.can_retry)
        self.assertIsInstance(failure.value.inner_exception, HttpResponseException)

        timings = self.get_success(
            self.hs.get_datastores().main.get_destination_retry_timings(DEST)
        )
        assert timings is not None
        self.assertEqual(
            timings.retry_interval,
            self.hs.config.federation.destination_min_retry_interval_ms,
        )

        self.get_failure(self.cl.get_json(DEST, "/x"), NotRetryingDestination)
        self.assertEqual(len(self.transport.requests), 4)

        # Ignoring the backoff sends, and success clears it.
        self.transport.responses.append(json_response(200, {}))
        self.get_success(self.cl.get_json(DEST, "/x", ignore_backoff=True))
        self.reactor.advance(0)
        timings = self.get_success(
            self.hs.get_datastores().main.get_destination_retry_timings(DEST)
        )
        self.assertIsNone(timings)

    def test_send_direct_request_raises_the_http_error(self) -> None:
        self.transport.responses.extend([json_response(503, {})] * 4)

        d = ensureDeferred(self.cl.send_direct_request("GET", DEST, "/x"))
        self._wait(d)

        failure = self.failureResultOf(d, HttpResponseException)
        self.assertEqual(failure.value.code, 503)

    def test_transport_error_with_a_timeout_is_not_retried(self) -> None:
        def fail(*args: Any) -> Response:
            raise ConnectionError("boom")

        self.cl._rust_client = RustFederationClient(
            self.hs,
            self.hs.version_string,
            self.hs.version_string,
            self.cl._write_retry_timings,
            self.cl._record_request_metrics,
            fail,
        )
        d = ensureDeferred(self.cl.get_json(DEST, "/x", timeout=10_000))
        self._wait(d)
        failure = self.failureResultOf(d, RequestSendFailed)
        assert isinstance(failure.value, RequestSendFailed)
        self.assertIsInstance(failure.value.inner_exception, ConnectionError)
        self.assertTrue(failure.value.can_retry)

    @unittest.override_config({"federation_domain_whitelist": ["other.example"]})
    def test_whitelist(self) -> None:
        self.get_failure(self.cl.get_json(DEST, "/x"), FederationDeniedError)
        self.assertEqual(self.transport.requests, [])

    def test_invalid_json(self) -> None:
        self.transport.responses.append(
            (200, [(b"content-type", b"application/json")], b"{not json")
        )
        failure = self.get_failure(self.cl.get_json(DEST, "/x"), RequestSendFailed)
        self.assertFalse(failure.value.can_retry)
        self.assertIsInstance(failure.value.inner_exception, ValueError)

    def test_json_that_is_not_an_object(self) -> None:
        self.transport.responses.append(json_response(200, [1, 2]))
        failure = self.get_failure(self.cl.get_json(DEST, "/x"), RequestSendFailed)
        self.assertFalse(failure.value.can_retry)
        self.assertIn("list", str(failure.value.inner_exception))

    def test_json_python_accepts_and_serde_does_not(self) -> None:
        """A lone surrogate is valid JSON to Python."""
        self.transport.responses.append(
            (200, [(b"content-type", b"application/json")], b'{"a": "\\ud800"}')
        )
        result = self.get_success(self.cl.get_json(DEST, "/x"))
        self.assertEqual(result, {"a": "\ud800"})

    def test_wrong_content_type(self) -> None:
        self.transport.responses.append((200, [(b"content-type", b"text/html")], b"{}"))
        failure = self.get_failure(self.cl.get_json(DEST, "/x"), RequestSendFailed)
        self.assertFalse(failure.value.can_retry)
        self.assertIn("text/html", str(failure.value.inner_exception))

    def test_legacy_json_send_parser(self) -> None:
        self.transport.responses.append(json_response(200, [200, {"ok": True}]))
        result = self.get_success(
            self.cl.put_json(
                DEST, "/send_join", data={"a": 1}, parser=LegacyJsonSendParser()
            )
        )
        self.assertEqual(result, [200, {"ok": True}])

        self.transport.responses.append(json_response(200, {"ok": True}))
        self.get_failure(
            self.cl.put_json(
                DEST, "/send_join", data={"a": 1}, parser=LegacyJsonSendParser()
            ),
            RequestSendFailed,
        )

    def test_custom_parser(self) -> None:
        class Collect(ByteParser[bytes]):
            CONTENT_TYPE = "application/json"

            def __init__(self) -> None:
                self.data = b""

            def write(self, data: bytes) -> int:
                self.data += data
                return len(data)

            def finish(self) -> bytes:
                return self.data

        body = json.dumps({"big": "x" * 200_000}).encode()
        self.transport.responses.append(
            (200, [(b"content-type", b"application/json")], body)
        )
        result = self.get_success(self.cl.get_json(DEST, "/x", parser=Collect()))
        self.assertEqual(result, body)

    def test_get_json_with_headers(self) -> None:
        self.transport.responses.append(
            (
                200,
                [(b"content-type", b"application/json"), (b"x-thing", b"1")],
                b"{}",
            )
        )
        _, headers = self.get_success(self.cl.get_json_with_headers(DEST, "/x"))
        self.assertEqual(headers[b"X-Thing"], [b"1"])
        self.assertEqual(headers[b"Content-Type"], [b"application/json"])

    def test_transaction_age_rewrite(self) -> None:
        """`send_transaction`'s `json_data_callback` is replaced by the Rust
        client rewriting `age_ts` on each attempt."""
        self.transport.responses.append(json_response(200, {}))
        now = self.clock.time_msec()
        data = {"pdus": [{"event_id": "$a", "age_ts": now - 1000}]}

        def callback() -> JsonDict:
            raise AssertionError("the callback is not called")

        self.get_success(
            self.cl.put_json(DEST, "/send/1", data=data, json_data_callback=callback)
        )
        (request,) = self.transport.requests
        self.assertEqual(
            json.loads(request.body)["pdus"],
            [{"event_id": "$a", "unsigned": {"age": 1000}}],
        )

    def test_gil_crossings(self) -> None:
        """A successful request takes the GIL once below the facade, to hand
        back the result. The transport here is Python and is not counted."""
        self.transport.responses.append(json_response(200, {}))
        before = federation_gil_stats()

        self.get_success(self.cl.get_json(DEST, "/x"))

        after = federation_gil_stats()
        calls = {
            site: after[site][0] - before.get(site, (0, 0.0))[0]
            for site in after
            if after[site][0] != before.get(site, (0, 0.0))[0]
        }
        self.assertEqual(calls, {"response": 1})