From bf1c069044d5aa5306a7d2587c2afac75116ebda Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 9 Oct 2026 16:29:40 +0200 Subject: [PATCH] Implement the Jaeger trace propagator in-tree The `opentelemetry-jaeger-propagator` crate was removed upstream after 0.32 (open-telemetry/opentelemetry-rust#3710), but Synapse propagates traces with the Jaeger `uber-trace-id` header, so MAS needs to keep supporting it. The in-tree propagator only handles that header, including the URL-encoded form jaeger-client-python sends and the debug flag set by Synapse's forced tracing. Baggage (`uberctx-*` headers) is not propagated. A missing or malformed header is ignored without logging, so the filter for the upstream crate's "invalid header" warning goes away too. --- Cargo.lock | 10 - Cargo.toml | 3 +- crates/cli/Cargo.toml | 2 +- crates/cli/src/main.rs | 11 +- crates/cli/src/telemetry.rs | 11 +- crates/cli/src/telemetry/jaeger.rs | 294 +++++++++++++++++++++++++++++ 6 files changed, 302 insertions(+), 29 deletions(-) create mode 100644 crates/cli/src/telemetry/jaeger.rs diff --git a/Cargo.lock b/Cargo.lock index 5ee9309af..1ef2bf461 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3395,7 +3395,6 @@ dependencies = [ "opentelemetry-http", "opentelemetry-instrumentation-process", "opentelemetry-instrumentation-tokio", - "opentelemetry-jaeger-propagator", "opentelemetry-otlp", "opentelemetry-prometheus-text-exporter", "opentelemetry-resource-detectors", @@ -4383,15 +4382,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "opentelemetry-jaeger-propagator" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c2cebef9a57a493394ba0901b692a6915cc4a1314e93f905db1344392b4aa4" -dependencies = [ - "opentelemetry", -] - [[package]] name = "opentelemetry-otlp" version = "0.32.0" diff --git a/Cargo.toml b/Cargo.toml index 392ed21ff..819729192 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,4 @@ +# Copyright 2025, 2026 Element Creations Ltd. # Copyright 2025 New Vector Ltd. # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial @@ -410,8 +411,6 @@ features = ["reqwest"] version = "0.2.0" [workspace.dependencies.opentelemetry-instrumentation-tokio] version = "0.2.0" -[workspace.dependencies.opentelemetry-jaeger-propagator] -version = "0.32.0" [workspace.dependencies.opentelemetry-otlp] version = "0.32.0" default-features = false diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index fae239425..f46d024ea 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -1,3 +1,4 @@ +# Copyright 2025, 2026 Element Creations Ltd. # Copyright 2025 New Vector Ltd. # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial @@ -59,7 +60,6 @@ opentelemetry.workspace = true opentelemetry-http.workspace = true opentelemetry-instrumentation-process.workspace = true opentelemetry-instrumentation-tokio.workspace = true -opentelemetry-jaeger-propagator.workspace = true opentelemetry-otlp.workspace = true opentelemetry-prometheus-text-exporter.workspace = true opentelemetry-resource-detectors.workspace = true diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 28e7a9a9d..2de45e7b0 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,3 +1,4 @@ +// Copyright 2025, 2026 Element Creations Ltd. // Copyright 2024, 2025 New Vector Ltd. // Copyright 2021-2024 The Matrix.org Foundation C.I.C. // @@ -13,10 +14,7 @@ use clap::Parser; use mas_config::{ConfigurationSectionExt, TelemetryConfig}; use sentry_tracing::EventFilter; use tracing_subscriber::{ - EnvFilter, Layer, Registry, - filter::{LevelFilter, filter_fn}, - layer::SubscriberExt, - util::SubscriberInitExt, + EnvFilter, Layer, Registry, filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt, }; mod app_state; @@ -104,10 +102,6 @@ async fn try_main() -> anyhow::Result { .or_else(|_| EnvFilter::try_new("info")) .context("could not setup logging filter")?; - // Suppress the following warning from the Jaeger propagator: - // Invalid jaeger header format header_value="" - let suppress_layer = filter_fn(|metadata| metadata.name() != "JaegerPropagator.InvalidHeader"); - // Setup the rustls crypto provider rustls::crypto::aws_lc_rs::default_provider() .install_default() @@ -163,7 +157,6 @@ async fn try_main() -> anyhow::Result { .with_filter(LevelFilter::INFO); let subscriber = Registry::default() - .with(suppress_layer) .with(sentry_layer) .with(telemetry_layer) .with(filter_layer) diff --git a/crates/cli/src/telemetry.rs b/crates/cli/src/telemetry.rs index 12add6ec4..68e30e2f2 100644 --- a/crates/cli/src/telemetry.rs +++ b/crates/cli/src/telemetry.rs @@ -1,3 +1,4 @@ +// Copyright 2025, 2026 Element Creations Ltd. // Copyright 2024, 2025 New Vector Ltd. // Copyright 2021-2024 The Matrix.org Foundation C.I.C. // @@ -33,6 +34,8 @@ use opentelemetry_sdk::{ }; use opentelemetry_semantic_conventions as semcov; +mod jaeger; + static SCOPE: LazyLock = LazyLock::new(|| { InstrumentationScope::builder(env!("CARGO_PKG_NAME")) .with_version(env!("CARGO_PKG_VERSION")) @@ -83,13 +86,7 @@ fn match_propagator(propagator: Propagator) -> Box Box::new(TraceContextPropagator::new()), P::Baggage => Box::new(BaggagePropagator::new()), - // opentelemetry-jaeger-propagator is deprecated upstream but still - // published; we keep supporting the Jaeger format until it's removed. - #[expect( - deprecated, - reason = "the Jaeger propagator is deprecated upstream but we still expose it as an option" - )] - P::Jaeger => Box::new(opentelemetry_jaeger_propagator::Propagator::new()), + P::Jaeger => Box::new(jaeger::JaegerPropagator::new()), } } diff --git a/crates/cli/src/telemetry/jaeger.rs b/crates/cli/src/telemetry/jaeger.rs new file mode 100644 index 000000000..eaa01eea8 --- /dev/null +++ b/crates/cli/src/telemetry/jaeger.rs @@ -0,0 +1,294 @@ +// Copyright 2026 Element Creations Ltd. +// +// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +// Please see LICENSE files in the repository root for full details. + +//! A minimal propagator for the Jaeger `uber-trace-id` header, the trace +//! context format Synapse uses. +//! +//! The header value is `{trace-id}:{span-id}:{parent-span-id}:{flags}`, all +//! hex-encoded, as described in +//! . +//! The parent span ID is deprecated in that format, so it is ignored on +//! extraction and sent as `0`. Baggage (`uberctx-*` headers) is not +//! propagated. + +use std::borrow::Cow; + +use opentelemetry::{ + Context, + propagation::{Extractor, Injector, TextMapPropagator, text_map_propagator::FieldIter}, + trace::{SpanContext, SpanId, TraceContextExt as _, TraceFlags, TraceId, TraceState}, +}; + +const HEADER_NAME: &str = "uber-trace-id"; + +/// Jaeger's debug flag has no W3C equivalent, so it is carried in a bit that +/// the W3C propagator masks out. +const TRACE_FLAG_DEBUG: TraceFlags = TraceFlags::new(0x04); + +/// Propagates the trace context through the Jaeger `uber-trace-id` header +#[derive(Debug)] +pub struct JaegerPropagator { + fields: [String; 1], +} + +impl JaegerPropagator { + pub fn new() -> Self { + Self { + fields: [HEADER_NAME.to_owned()], + } + } +} + +/// Parse the value of an `uber-trace-id` header +fn parse_header(value: &str) -> Option { + // jaeger-client-python, which Synapse uses, URL-encodes the whole value + let value = if value.contains(':') { + Cow::Borrowed(value) + } else { + Cow::Owned(value.replace("%3A", ":").replace("%3a", ":")) + }; + + let mut parts = value.split(':'); + let trace_id = parts.next()?; + let span_id = parts.next()?; + let _parent_span_id = parts.next()?; + let flags = parts.next()?; + if parts.next().is_some() { + return None; + } + + // Jaeger clients drop leading zeros from the IDs, so shorter values are valid + if trace_id.is_empty() || trace_id.len() > 32 { + return None; + } + if span_id.is_empty() || span_id.len() > 16 { + return None; + } + if flags.is_empty() || flags.len() > 2 { + return None; + } + + let trace_id = TraceId::from_hex(trace_id).ok()?; + let span_id = SpanId::from_hex(span_id).ok()?; + let flags = u8::from_str_radix(flags, 16).ok()?; + // The debug flag (0x02) only has an effect on sampled traces + let trace_flags = match flags & 0x03 { + 0x03 => TraceFlags::SAMPLED | TRACE_FLAG_DEBUG, + 0x01 => TraceFlags::SAMPLED, + _ => TraceFlags::default(), + }; + + let span_context = SpanContext::new(trace_id, span_id, trace_flags, true, TraceState::NONE); + span_context.is_valid().then_some(span_context) +} + +impl TextMapPropagator for JaegerPropagator { + fn inject_context(&self, cx: &Context, injector: &mut dyn Injector) { + let span = cx.span(); + let span_context = span.span_context(); + if !span_context.is_valid() { + return; + } + + let flags: u8 = if !span_context.is_sampled() { + 0x00 + } else if span_context.trace_flags() & TRACE_FLAG_DEBUG == TRACE_FLAG_DEBUG { + 0x03 + } else { + 0x01 + }; + injector.set( + HEADER_NAME, + format!( + "{}:{}:0:{flags:x}", + span_context.trace_id(), + span_context.span_id() + ), + ); + } + + fn extract_with_context(&self, cx: &Context, extractor: &dyn Extractor) -> Context { + extractor + .get(HEADER_NAME) + .and_then(parse_header) + .map_or_else( + || cx.clone(), + |span_context| cx.with_remote_span_context(span_context), + ) + } + + fn fields(&self) -> FieldIter<'_> { + FieldIter::new(&self.fields) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn extract(value: &str) -> Option { + let headers = HashMap::from([(HEADER_NAME.to_owned(), value.to_owned())]); + let cx = JaegerPropagator::new().extract(&headers); + let span = cx.span(); + let span_context = span.span_context(); + span_context.is_valid().then(|| span_context.clone()) + } + + #[test] + fn extract_sampled() { + let span_context = + extract("4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:1").unwrap(); + assert_eq!( + span_context.trace_id(), + TraceId::from_hex("4bf92f3577b34da6a3ce929d0e0e4736").unwrap() + ); + assert_eq!( + span_context.span_id(), + SpanId::from_hex("00f067aa0ba902b7").unwrap() + ); + assert!(span_context.is_sampled()); + assert!(span_context.is_remote()); + } + + #[test] + fn extract_not_sampled() { + let span_context = + extract("4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:0").unwrap(); + assert!(!span_context.is_sampled()); + } + + #[test] + fn extract_debug_flag() { + let span_context = + extract("4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:3").unwrap(); + assert!(span_context.is_sampled()); + assert_eq!( + span_context.trace_flags() & TRACE_FLAG_DEBUG, + TRACE_FLAG_DEBUG + ); + } + + #[test] + fn extract_debug_flag_without_sampled() { + let span_context = + extract("4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:2").unwrap(); + assert_eq!(span_context.trace_flags(), TraceFlags::default()); + } + + #[test] + fn extract_synapse_header() { + // jaeger-client-python URL-encodes the value, drops leading zeros and + // sends a non-zero parent span ID + let span_context = + extract("6f3a0c1d2e4b5a69%3A1b2c3d4e5f6a7b8%3A2a3b4c5d6e7f8091%3A1").unwrap(); + assert_eq!( + span_context.trace_id(), + TraceId::from_hex("00000000000000006f3a0c1d2e4b5a69").unwrap() + ); + assert_eq!( + span_context.span_id(), + SpanId::from_hex("01b2c3d4e5f6a7b8").unwrap() + ); + assert!(span_context.is_sampled()); + } + + #[test] + fn extract_short_ids() { + let span_context = extract("a3ce929d0e0e4736:f067aa0ba902b7:0:1").unwrap(); + assert_eq!( + span_context.trace_id(), + TraceId::from_hex("0000000000000000a3ce929d0e0e4736").unwrap() + ); + assert_eq!( + span_context.span_id(), + SpanId::from_hex("00f067aa0ba902b7").unwrap() + ); + } + + #[test] + fn extract_url_encoded() { + let span_context = + extract("4bf92f3577b34da6a3ce929d0e0e4736%3A00f067aa0ba902b7%3A0%3A1").unwrap(); + assert!(span_context.is_sampled()); + } + + #[test] + fn extract_invalid() { + for value in [ + "", + "4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:1", + "4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:1:0", + "4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:", + "4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:100", + ":00f067aa0ba902b7:0:1", + "04bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:1", + "4bf92f3577b34da6a3ce929d0e0e4736:000f067aa0ba902b7:0:1", + "not-hex:00f067aa0ba902b7:0:1", + "0:00f067aa0ba902b7:0:1", + "4bf92f3577b34da6a3ce929d0e0e4736:0:0:1", + ] { + assert!(extract(value).is_none(), "{value:?} should be rejected"); + } + } + + #[test] + fn extract_missing_header() { + let headers: HashMap = HashMap::new(); + let cx = JaegerPropagator::new().extract(&headers); + assert!(!cx.span().span_context().is_valid()); + } + + #[test] + fn inject_round_trip() { + let span_context = SpanContext::new( + TraceId::from_hex("4bf92f3577b34da6a3ce929d0e0e4736").unwrap(), + SpanId::from_hex("00f067aa0ba902b7").unwrap(), + TraceFlags::SAMPLED, + false, + TraceState::NONE, + ); + let cx = Context::new().with_remote_span_context(span_context.clone()); + + let mut headers = HashMap::new(); + JaegerPropagator::new().inject_context(&cx, &mut headers); + assert_eq!( + headers.get(HEADER_NAME).map(String::as_str), + Some("4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:1") + ); + + let extracted = extract(&headers[HEADER_NAME]).unwrap(); + assert_eq!(extracted.trace_id(), span_context.trace_id()); + assert_eq!(extracted.span_id(), span_context.span_id()); + assert!(extracted.is_sampled()); + } + + #[test] + fn inject_debug_flag() { + let span_context = SpanContext::new( + TraceId::from_hex("4bf92f3577b34da6a3ce929d0e0e4736").unwrap(), + SpanId::from_hex("00f067aa0ba902b7").unwrap(), + TraceFlags::SAMPLED | TRACE_FLAG_DEBUG, + false, + TraceState::NONE, + ); + let cx = Context::new().with_remote_span_context(span_context); + + let mut headers = HashMap::new(); + JaegerPropagator::new().inject_context(&cx, &mut headers); + assert_eq!( + headers.get(HEADER_NAME).map(String::as_str), + Some("4bf92f3577b34da6a3ce929d0e0e4736:00f067aa0ba902b7:0:3") + ); + } + + #[test] + fn inject_without_span() { + let mut headers = HashMap::new(); + JaegerPropagator::new().inject_context(&Context::new(), &mut headers); + assert!(headers.is_empty()); + } +}