diff --git a/docs/opentracing.md b/docs/opentracing.md index c82c6affba..ac0b7b779f 100644 --- a/docs/opentracing.md +++ b/docs/opentracing.md @@ -108,6 +108,16 @@ configure. A Rust span with a Python parent follows the parent's sampling decision. A trace that starts in Rust is sampled according to the `const` or `probabilistic` sampler. Under any other sampler, such traces are not sampled. +The same reporter also sends the Python spans, in place of jaeger-client's own +reporter and of the optional `rust_python_jaeger_reporter` package, which is no +longer used. The rest of `jaeger_config` (the sampler, remote or not, the +throttler, `logging`) applies as before. Its `reporter_queue_size`, +`reporter_batch_size` and `reporter_flush_interval` options and jaeger-client's +`jaeger:reporter_*` metrics no longer apply: the Rust reporter queues up to +1000 spans, sends a batch at 100 spans, at 65000 bytes or after a second, and +exports `synapse_rust_tracing_*` metrics instead. If tracing in Rust cannot +start, jaeger-client's reporter sends the Python spans. + The Rust reporter's counters are exported as Prometheus metrics named `synapse_rust_tracing_*`. A growing `synapse_rust_tracing_spans_dropped_total` means spans are finishing faster than they can be sent. diff --git a/mypy.ini b/mypy.ini index ec73ce9f6e..3c578a308a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -95,9 +95,6 @@ ignore_missing_imports = True [mypy-pyperf.*] ignore_missing_imports = True -[mypy-rust_python_jaeger_reporter.*] -ignore_missing_imports = True - [mypy-saml2.*] ignore_missing_imports = True diff --git a/rust/src/logging/opentracing/mod.rs b/rust/src/logging/opentracing/mod.rs index 8379efcbb4..7a5b9a4c78 100644 --- a/rust/src/logging/opentracing/mod.rs +++ b/rust/src/logging/opentracing/mod.rs @@ -24,6 +24,7 @@ //! While it is off, a capture costs one relaxed atomic load. pub mod layer; +pub mod python_reporter; pub mod reporter; pub mod thrift; @@ -442,6 +443,11 @@ fn set_tracing_enabled_for_test(enabled: bool) { /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module = PyModule::new(py, "opentracing")?; + child_module.add_class::()?; + child_module.add_function(wrap_pyfunction!( + python_reporter::encode_python_span_for_test, + &child_module + )?)?; child_module.add_function(wrap_pyfunction!(captured_span_for_test, &child_module)?)?; child_module.add_function(wrap_pyfunction!( span_in_blocking_task_for_test, diff --git a/rust/src/logging/opentracing/python_reporter.rs b/rust/src/logging/opentracing/python_reporter.rs new file mode 100644 index 0000000000..5e9d50f870 --- /dev/null +++ b/rust/src/logging/opentracing/python_reporter.rs @@ -0,0 +1,204 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Report Python's jaeger-client spans through the Rust reporter, so that one +//! UDP emitter sends both Python's and Rust's spans. +//! +//! `PythonSpanReporter` is the reporter `init_tracer` gives the Python +//! tracer. `report_span` converts a finished jaeger-client span into a +//! [`FinishedSpan`] with the GIL held, which is a handful of attribute +//! reads, and queues it on the homeserver's reporter. The encoding and the +//! send happen on the reporter thread, as for Rust's own spans. +//! +//! The conversion follows jaeger-client's `thrift.make_jaeger_batch`, so the +//! datagrams are what jaeger-client's own reporter sends. + +use std::borrow::Cow; +use std::time::{Duration, UNIX_EPOCH}; + +use pyo3::prelude::*; +use pyo3::types::PyList; +use pyo3::{intern, IntoPyObjectExt}; + +use super::thrift::{FinishedSpan, Log, RefType, SpanRef, Tag, TagValue}; +use crate::runtime::RustRuntime; + +/// The reporter for Python's spans. See the module docs. +#[pyclass(frozen)] +pub struct PythonSpanReporter { + runtime: RustRuntime, +} + +#[pymethods] +impl PythonSpanReporter { + #[new] + fn py_new(runtime: &Bound<'_, RustRuntime>) -> Self { + PythonSpanReporter { + runtime: runtime.get().clone(), + } + } + + /// Queue a finished jaeger-client span. Never blocks: drops the span if + /// the queue is full, or if tracing is not enabled for the homeserver. + fn report_span(&self, span: &Bound<'_, PyAny>) -> PyResult<()> { + let Some(tracer) = self.runtime.tracer() else { + return Ok(()); + }; + tracer.report(finished_span_from_py(span)?); + Ok(()) + } +} + +/// jaeger-client's `timestamp_micros`, `int(ts * 1000000)`, for times since +/// the epoch: `as` truncates towards zero, as `int()` does. Negative values +/// (a clock that stepped back) become 0. +fn micros(seconds: f64) -> u64 { + (seconds * 1_000_000.0) as u64 +} + +/// The low `bits` of the Python int `value`. jaeger-client masks the trace ID +/// to 128 bits; IDs from a remote context can be any length of hex. +fn masked FromPyObject<'a, 'a>>(value: &Bound<'_, PyAny>, bits: u32) -> PyResult { + let mask = (1u128 << (bits - 1)) | ((1u128 << (bits - 1)) - 1); + value.bitand(mask)?.extract().map_err(Into::into) +} + +/// Convert a finished jaeger-client `Span`. +pub(crate) fn finished_span_from_py(span: &Bound<'_, PyAny>) -> PyResult { + let py = span.py(); + let context = span.getattr(intern!(py, "context"))?; + let trace_id: u128 = masked(&context.getattr(intern!(py, "trace_id"))?, 128)?; + let span_id: u64 = masked(&context.getattr(intern!(py, "span_id"))?, 64)?; + let parent_id = context.getattr(intern!(py, "parent_id"))?; + let parent_id: u64 = if parent_id.is_none() { + 0 + } else { + masked(&parent_id, 64)? + }; + // Only the sampled and debug bits mean anything. + let flags: u8 = masked(&context.getattr(intern!(py, "flags"))?, 8)?; + let operation: String = span.getattr(intern!(py, "operation_name"))?.extract()?; + + let start_time: f64 = span.getattr(intern!(py, "start_time"))?.extract()?; + let end_time: Option = span.getattr(intern!(py, "end_time"))?.extract()?; + let end_time = end_time.unwrap_or(start_time); + let start_micros = micros(start_time); + let duration_micros = micros(end_time - start_time); + + let tags = tags_from_py(&span.getattr(intern!(py, "tags"))?)?; + let logs = span + .getattr(intern!(py, "logs"))? + .cast_into::()? + .iter() + .map(|log| { + let timestamp: u64 = log.getattr(intern!(py, "timestamp"))?.extract()?; + Ok(Log { + timestamp: UNIX_EPOCH + Duration::from_micros(timestamp), + fields: tags_from_py(&log.getattr(intern!(py, "fields"))?)?, + }) + }) + .collect::>>()?; + + let references = span.getattr(intern!(py, "references"))?; + let references = if references.is_none() { + Vec::new() + } else { + references + .try_iter()? + .map(|reference| { + let reference = reference?; + let ref_type: String = reference.getattr(intern!(py, "type"))?.extract()?; + let referenced = reference.getattr(intern!(py, "referenced_context"))?; + Ok(SpanRef { + ref_type: if ref_type == "follows_from" { + RefType::FollowsFrom + } else { + RefType::ChildOf + }, + trace_id: masked(&referenced.getattr(intern!(py, "trace_id"))?, 128)?, + span_id: masked(&referenced.getattr(intern!(py, "span_id"))?, 64)?, + }) + }) + .collect::>>()? + }; + + Ok(FinishedSpan { + trace_id, + span_id, + parent_id, + flags, + operation: Cow::Owned(operation), + start: UNIX_EPOCH + Duration::from_micros(start_micros), + duration: Duration::from_micros(duration_micros), + tags, + logs, + references, + }) +} + +/// Convert a list of jaeger thrift `Tag`s, which jaeger-client builds as +/// tags are set. +fn tags_from_py(tags: &Bound<'_, PyAny>) -> PyResult> { + let py = tags.py(); + tags.try_iter()? + .map(|tag| { + let tag = tag?; + let key: String = tag.getattr(intern!(py, "key"))?.extract()?; + let v_type: i32 = tag.getattr(intern!(py, "vType"))?.extract()?; + let value = match v_type { + 1 => TagValue::Double(tag.getattr(intern!(py, "vDouble"))?.extract()?), + 2 => TagValue::Bool(tag.getattr(intern!(py, "vBool"))?.extract()?), + 3 => TagValue::Long(tag.getattr(intern!(py, "vLong"))?.extract()?), + // Generated code writes `vBinary` as a string's UTF-8. + 4 => { + let value = tag.getattr(intern!(py, "vBinary"))?; + let bytes = match value.extract::>() { + Ok(bytes) if !value.is_instance_of::() => bytes, + _ => value.str()?.to_string().into_bytes(), + }; + TagValue::Binary(bytes) + } + _ => TagValue::Str( + tag.getattr(intern!(py, "vStr"))? + .extract::()? + .into(), + ), + }; + Ok(Tag::new(key, value)) + }) + .collect() +} + +/// Encode `span`, a finished jaeger-client span, as one `emitBatch` +/// message with `process`, as the Rust reporter would send it. For tests +/// comparing with jaeger-client's own encoding. +#[pyfunction] +pub fn encode_python_span_for_test<'py>( + py: Python<'py>, + span: &Bound<'py, PyAny>, + service_name: String, + process_tags: Vec<(String, String)>, +) -> PyResult> { + let process = super::thrift::Process { + service_name, + tags: process_tags + .into_iter() + .map(|(key, value)| Tag::string(key, value)) + .collect(), + }; + let encoded = super::thrift::encode_span(&finished_span_from_py(span)?); + let message = super::thrift::encode_emit_batch(&process, &[encoded], 1); + pyo3::types::PyBytes::new(py, &message).into_bound_py_any(py) +} diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 1d9b5cdc7a..2e04ad5215 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -193,7 +193,6 @@ from typing import ( overload, ) -import attr from typing_extensions import Concatenate, ParamSpec from twisted.internet import defer @@ -208,6 +207,7 @@ if TYPE_CHECKING: from synapse.http.site import SynapseRequest from synapse.server import HomeServer + from synapse.synapse_rust.opentracing import PythonSpanReporter # Helper class @@ -264,36 +264,40 @@ except ImportError: LogContextScopeManager = None # type: ignore +# jaeger-client 4.7.0 requires that reporters inherit from BaseReporter, which +# didn't exist before that version. try: - from rust_python_jaeger_reporter import Reporter - - # jaeger-client 4.7.0 requires that reporters inherit from BaseReporter, which - # didn't exist before that version. - try: - from jaeger_client.reporter import BaseReporter - except ImportError: - - class BaseReporter: # type: ignore[no-redef] - pass - - @attr.s(slots=True, frozen=True, auto_attribs=True) - class _WrappedRustReporter(BaseReporter): - """Wrap the reporter to ensure `report_span` never throws.""" - - _reporter: Reporter = attr.Factory(Reporter) - - def set_process(self, *args: Any, **kwargs: Any) -> None: - return self._reporter.set_process(*args, **kwargs) - - def report_span(self, span: "opentracing.Span") -> None: - try: - return self._reporter.report_span(span) - except Exception: - logger.exception("Failed to report span") - - RustReporter: type[_WrappedRustReporter] | None = _WrappedRustReporter + from jaeger_client.reporter import BaseReporter except ImportError: - RustReporter = None + + class BaseReporter: # type: ignore[no-redef] + pass + + +class _RustSpanReporter(BaseReporter): + """Reports the Python tracer's spans through the homeserver's Rust + reporter, which also sends the spans of Rust code. Never raises.""" + + def __init__(self, reporter: "PythonSpanReporter") -> None: + super().__init__() + self._reporter = reporter + self._logged_failure = False + + def set_process(self, *args: Any, **kwargs: Any) -> None: + # The Rust reporter takes the process from `enable_tracing`. + pass + + def report_span(self, span: "opentracing.Span") -> None: + try: + self._reporter.report_span(span) + except Exception: + # Once with the traceback: a bad span context from a remote + # server would otherwise log one per span of its trace. + if not self._logged_failure: + self._logged_failure = True + logger.exception("Failed to report span") + else: + logger.debug("Failed to report span", exc_info=True) logger = logging.getLogger(__name__) @@ -462,8 +466,6 @@ def init_tracer(hs: "HomeServer") -> None: set_homeserver_whitelist(hs.config.tracing.opentracer_whitelist) - from jaeger_client.metrics.prometheus import PrometheusMetricsFactory - # Instance names are opaque strings but by stripping off the number suffix, # we can get something that looks like a "worker type", e.g. # "client_reader-1" -> "client_reader" so we don't spread the traces across @@ -484,29 +486,60 @@ def init_tracer(hs: "HomeServer") -> None: config=jaeger_config, service_name=f"{hs.config.server.server_name} {instance_name_by_type}", scope_manager=LogContextScopeManager(), - metrics_factory=PrometheusMetricsFactory(), + metrics_factory=_jaeger_metrics_factory(), ) - tracer: "JaegerTracer | None" - # If we have the rust jaeger reporter available let's use that. - if RustReporter: - logger.info("Using rust_python_jaeger_reporter library") - assert config.sampler is not None - tracer = config.create_tracer(RustReporter(), config.sampler) - opentracing.set_global_tracer(tracer) - else: - # `None` if another homeserver in this process already initialised - # the global tracer. That homeserver then reports this one's Python - # spans too, and this one reports no Rust spans of its own. - tracer = config.initialize_tracer() + # `None` if another homeserver in this process already initialised the + # global tracer. That homeserver then reports this one's Python spans + # too, and this one reports no Rust spans of its own. + tracer = config.initialize_tracer() + if tracer is None: + return - if tracer is not None: - try: - _enable_rust_tracing(hs, config, tracer) - except Exception: - # Python tracing works without the Rust side, so do not stop - # startup over it. - logger.exception("Failed to enable opentracing for Rust code") + try: + _enable_rust_tracing(hs, config, tracer) + except Exception: + # Python tracing works without the Rust side, so do not stop startup + # over it: jaeger-client's reporter goes on sending Python's spans. + logger.exception("Failed to enable opentracing for Rust code") + return + + _report_python_spans_in_rust(hs, tracer) + + +def _report_python_spans_in_rust(hs: "HomeServer", tracer: "JaegerTracer") -> None: + """Send the Python tracer's spans through the Rust reporter, with Rust's, + in place of jaeger-client's reporter. The tracer is otherwise as + jaeger-client set it up from the config: its sampler (remote included), + throttler and logging reporter.""" + from jaeger_client.reporter import CompositeReporter + + from synapse.synapse_rust.opentracing import PythonSpanReporter + + rust_reporter = _RustSpanReporter(PythonSpanReporter(hs.get_rust_runtime())) + reporter = tracer.reporter + if isinstance(reporter, CompositeReporter): + # `logging: true`: jaeger-client's reporter and a `LoggingReporter`. + reporter.reporters = tuple( + rust_reporter if index == 0 else inner + for index, inner in enumerate(reporter.reporters) + ) + else: + tracer.reporter = rust_reporter + + +_JAEGER_METRICS_FACTORY: Any = None + + +def _jaeger_metrics_factory() -> Any: + """One `PrometheusMetricsFactory` per process: each registers its metrics + in the global registry, so a second one would clash with the first.""" + global _JAEGER_METRICS_FACTORY + if _JAEGER_METRICS_FACTORY is None: + from jaeger_client.metrics.prometheus import PrometheusMetricsFactory + + _JAEGER_METRICS_FACTORY = PrometheusMetricsFactory() + return _JAEGER_METRICS_FACTORY def _enable_rust_tracing( diff --git a/synapse/synapse_rust/opentracing.pyi b/synapse/synapse_rust/opentracing.pyi index 04c5a709ea..5cce8c4ade 100644 --- a/synapse/synapse_rust/opentracing.pyi +++ b/synapse/synapse_rust/opentracing.pyi @@ -10,8 +10,25 @@ # See the GNU Affero General Public License for more details: # . +from typing import Any + from synapse.synapse_rust.runtime import RustRuntime +class PythonSpanReporter: + """Reports the Python tracer's finished spans through the homeserver's + Rust reporter, which sends them to the Jaeger agent with Rust's own.""" + + def __init__(self, runtime: RustRuntime) -> None: ... + def report_span(self, span: Any) -> None: + """Queue a finished jaeger-client span. Never blocks: the span is + dropped if the queue is full or tracing is not enabled.""" + +def encode_python_span_for_test( + span: Any, service_name: str, process_tags: list[tuple[str, str]] +) -> bytes: + """The `emitBatch` message the Rust reporter would send for the finished + jaeger-client `span`, with sequence ID 1. For tests.""" + def captured_span_for_test() -> tuple[int, int, int, dict[str, str]] | None: """The `(trace_id, span_id, flags, baggage)` of the span a Rust task spawned here would take as its parent, or None. For tests.""" diff --git a/tests/logging/test_rust_tracing.py b/tests/logging/test_rust_tracing.py index ff8f3429db..36f226f172 100644 --- a/tests/logging/test_rust_tracing.py +++ b/tests/logging/test_rust_tracing.py @@ -21,7 +21,11 @@ from unittest import SkipTest from twisted.internet.testing import MemoryReactor from synapse.logging.context import LoggingContext -from synapse.logging.opentracing import _enable_rust_tracing, start_active_span +from synapse.logging.opentracing import ( + _enable_rust_tracing, + start_active_span, + start_active_span_follows_from, +) from synapse.logging.rust_tracing_metrics import rust_tracing_collector from synapse.rest import admin from synapse.rest.client import login, room, sync @@ -29,6 +33,7 @@ from synapse.server import HomeServer from synapse.storage.rust_pool import RustConnectionPool from synapse.synapse_rust.opentracing import ( captured_span_for_test, + encode_python_span_for_test, set_tracing_enabled_for_test, span_in_blocking_task_for_test, span_in_task_for_test, @@ -46,6 +51,7 @@ try: from jaeger_client.thrift_gen.jaeger import ttypes from thrift.protocol.TCompactProtocol import ( # type: ignore[import-untyped] TCompactProtocol, + TCompactProtocolFactory, ) from thrift.transport.TTransport import ( # type: ignore[import-untyped] TMemoryBuffer, @@ -832,3 +838,152 @@ class SlidingSyncTracingTestCase(_AgentTestCase): ), [s.operationName for s in spans], ) + + +def _python_encoding( + span: "jaeger_client.Span", service_name: str, process_tags: dict[str, str] +) -> bytes: + """The `emitBatch` message jaeger-client's own reporter sends for `span`, + with sequence ID 1.""" + from jaeger_client import thrift as jaeger_thrift + from jaeger_client.thrift_gen.agent import Agent + + process = jaeger_thrift.make_process(service_name, process_tags, 1024) + batch = jaeger_thrift.make_jaeger_batch([span], process) + buffer = TMemoryBuffer() + client = Agent.Client.__new__(Agent.Client) + client._transport = buffer + client._oprot_factory = TCompactProtocolFactory() + client._seqid = 0 + client.emitBatch(batch) + return buffer.getvalue() + + +class PythonSpanEncodingTestCase(TestCase): + """The Rust reporter sends Python's spans exactly as jaeger-client's own + reporter does.""" + + if jaeger_client is None: + skip = "Requires jaeger_client" # type: ignore[unreachable] + + def test_same_bytes(self) -> None: + reporter = jaeger_client.reporter.InMemoryReporter() + config = jaeger_client.config.Config( + config={}, service_name="test", scope_manager=LogContextScopeManager() + ) + tracer = config.create_tracer(sampler=ConstSampler(True), reporter=reporter) + + with LoggingContext(name="ctx", server_name="test_server"): + with start_active_span("root", tracer=tracer) as root: + root.span.set_tag("string", "value") + root.span.set_tag("number", 12) + root.span.set_tag("ratio", 0.25) + root.span.set_tag("flag", True) + root.span.log_kv({"message": "hello", "count": 3}) + with start_active_span("child", tracer=tracer): + pass + with start_active_span_follows_from( + "follower", [root.span.context], tracer=tracer + ): + pass + + # Also IDs at and above 2^63, which thrift carries negative. + big = tracer.start_span("big ids") + big_context = cast(jaeger_client.SpanContext, big.context) + big_context.span_id = (1 << 63) + 5 + big_context.trace_id = (1 << 100) + 7 + big.finish() + + # A trace ID from a remote context can be wider than 128 bits: + # jaeger-client sends the low 128. + wide = tracer.start_span("wide trace id") + cast(jaeger_client.SpanContext, wide.context).trace_id = (1 << 130) + 9 + wide.finish() + + spans = reporter.get_spans() + self.assertEqual(len(spans), 5) + process_tags = {"hostname": "host", "jaeger.version": "Python-4.8.0"} + for span in spans: + rust = encode_python_span_for_test( + span, "test service", list(process_tags.items()) + ) + python = _python_encoding(span, "test service", process_tags) + self.assertEqual(rust, python, span.operation_name) + + +class InitTracerTestCase(HomeserverTestCase): + """`init_tracer` reports the Python tracer's spans through the Rust + reporter.""" + + if jaeger_client is None: + skip = "Requires jaeger_client" # type: ignore[unreachable] + + def test_python_spans_go_through_rust(self) -> None: + import synapse.logging.opentracing as synapse_opentracing + + self.addCleanup(set_tracing_enabled_for_test, False) + previous_tracer = opentracing.global_tracer() + self.addCleanup(opentracing.set_global_tracer, previous_tracer) + + agent = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(agent.close) + agent.bind(("127.0.0.1", 0)) + agent.settimeout(5) + port = agent.getsockname()[1] + + tracing_config = self.hs.config.tracing + tracing_config.opentracer_enabled = True + tracing_config.opentracer_whitelist = [] + tracing_config.jaeger_config = { + "sampler": {"type": "const", "param": 1}, + "local_agent": {"reporting_host": "127.0.0.1", "reporting_port": port}, + } + # `initialize_tracer` initialises jaeger-client once per process. + self.addCleanup(setattr, jaeger_client.Config, "_initialized", False) + synapse_opentracing.init_tracer(self.hs) + runtime = self.hs.get_rust_runtime() + self.addCleanup(runtime.shutdown_tracing_for_test) + + tracer = cast(jaeger_client.Tracer, opentracing.global_tracer()) + self.assertIsInstance(tracer.reporter, synapse_opentracing._RustSpanReporter) + with LoggingContext(name="ctx", server_name="test_server"): + with start_active_span("from python", tracer=tracer) as scope: + span_in_task_for_test(runtime, "from rust") + self.assertTrue(runtime.flush_tracing_for_test(5)) + + data, _ = agent.recvfrom(70_000) + batch = _decode_emit_batch(data) + names = {span.operationName for span in batch.spans} + self.assertEqual(names, {"from python", "from rust"}) + self.assertEqual(batch.process.serviceName, tracer.service_name) + (rust_span,) = [s for s in batch.spans if s.operationName == "from rust"] + self.assertEqual(rust_span.parentSpanId, _signed(_context(scope.span).span_id)) + + def test_logging_reporter_is_kept(self) -> None: + """With `logging: true`, spans still go to jaeger-client's logging + reporter too. A second `init_tracer` in the process is fine.""" + from jaeger_client.reporter import CompositeReporter, LoggingReporter + + import synapse.logging.opentracing as synapse_opentracing + + self.addCleanup(set_tracing_enabled_for_test, False) + previous_tracer = opentracing.global_tracer() + self.addCleanup(opentracing.set_global_tracer, previous_tracer) + self.addCleanup(setattr, jaeger_client.Config, "_initialized", False) + + tracing_config = self.hs.config.tracing + tracing_config.opentracer_enabled = True + tracing_config.opentracer_whitelist = [] + tracing_config.jaeger_config = { + "sampler": {"type": "const", "param": 0}, + "logging": True, + } + synapse_opentracing.init_tracer(self.hs) + self.addCleanup(self.hs.get_rust_runtime().shutdown_tracing_for_test) + + tracer = cast(jaeger_client.Tracer, opentracing.global_tracer()) + reporter = tracer.reporter + assert isinstance(reporter, CompositeReporter) + first, second = reporter.reporters + self.assertIsInstance(first, synapse_opentracing._RustSpanReporter) + self.assertIsInstance(second, LoggingReporter)