From 49c163679db040bf352e146604b5f99a4e4c3ff1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Jul 2026 09:25:02 +0000 Subject: [PATCH] Port ContextResourceUsage to a Rust pyclass The first slice of moving the logcontext machinery into the Rust extension: ContextResourceUsage becomes a native class in the new synapse.synapse_rust.logcontext module (rust/src/logging/context.rs), re-exported from synapse.logging.context so callers are unchanged. The public attribute surface, operators and repr are a compatibility contract with the Python callers (Measure, request/background-process metrics, the task scheduler, ...). Keeping the tracker native lets the upcoming switch machinery do its rusage accounting without allocating a Python object per operation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JFbRtswu7rsHrttJFauUUb --- rust/src/lib.rs | 2 + rust/src/logging/context.rs | 154 ++++++++++++++++++++++++++++ rust/src/logging/mod.rs | 23 +++++ synapse/logging/context.py | 105 +------------------ synapse/synapse_rust/logcontext.pyi | 31 ++++++ 5 files changed, 214 insertions(+), 101 deletions(-) create mode 100644 rust/src/logging/context.rs create mode 100644 rust/src/logging/mod.rs create mode 100644 synapse/synapse_rust/logcontext.pyi diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 28783afbba..f68b7c17d7 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -16,6 +16,7 @@ pub mod http; pub mod http_client; pub mod identifier; pub mod json; +pub mod logging; pub mod matrix_const; pub mod msc4388_rendezvous; pub mod push; @@ -70,6 +71,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?; acl::register_module(py, m)?; + logging::context::register_module(py, m)?; deferred::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; diff --git a/rust/src/logging/context.rs b/rust/src/logging/context.rs new file mode 100644 index 0000000000..26f19ed5d1 --- /dev/null +++ b/rust/src/logging/context.rs @@ -0,0 +1,154 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Native counterparts to `synapse.logging.context`. +//! +//! Currently just [`ContextResourceUsage`]; the goal is per-operation resource +//! accounting with no Python allocation on the switch path. +//! +//! TODO: the storage for the "current" logcontext, `LoggingContext` itself and +//! the sentinel follow — see `synapse.logging.context` for the Python +//! implementations being replaced. + +use pyo3::prelude::*; + +/// Tracks the resources used by a log context. +/// +/// The public attribute surface, operators and `repr` are a compatibility +/// contract with the Python callers (Measure, request/background-process +/// metrics, the task scheduler, ...) — change both sides together. Native so the +/// switch machinery can do its rusage accounting without allocating a Python +/// object per operation. +// `skip_from_py_object`: pyo3 0.28 requires `Clone` pyclasses to explicitly +// opt in or out of a generated extract-by-clone `FromPyObject` (a bare +// `#[pyclass]` is a deprecation warning, and we build with `-D warnings`). +// Nothing extracts this type by value, so opt out. +#[pyclass(skip_from_py_object, get_all, set_all)] +#[derive(Clone, Default)] +pub struct ContextResourceUsage { + /// System CPU time, in seconds. + pub ru_stime: f64, + /// User CPU time, in seconds. + pub ru_utime: f64, + /// Number of database transactions done. + pub db_txn_count: i64, + /// Time spent doing database transactions (excluding scheduling), in seconds. + pub db_txn_duration_sec: f64, + /// Time spent waiting for a database connection, in seconds. + pub db_sched_duration_sec: f64, + /// Number of events requested from the database. + pub evt_db_fetch_count: i64, +} + +impl ContextResourceUsage { + fn add_assign(&mut self, other: &ContextResourceUsage) { + self.ru_utime += other.ru_utime; + self.ru_stime += other.ru_stime; + self.db_txn_count += other.db_txn_count; + self.db_txn_duration_sec += other.db_txn_duration_sec; + self.db_sched_duration_sec += other.db_sched_duration_sec; + self.evt_db_fetch_count += other.evt_db_fetch_count; + } + + fn sub_assign(&mut self, other: &ContextResourceUsage) { + self.ru_utime -= other.ru_utime; + self.ru_stime -= other.ru_stime; + self.db_txn_count -= other.db_txn_count; + self.db_txn_duration_sec -= other.db_txn_duration_sec; + self.db_sched_duration_sec -= other.db_sched_duration_sec; + self.evt_db_fetch_count -= other.evt_db_fetch_count; + } +} + +#[pymethods] +impl ContextResourceUsage { + /// `ContextResourceUsage(copy_from=None)` — if `copy_from` is given, copy its + /// stats; otherwise start at zero. + #[new] + #[pyo3(signature = (copy_from=None))] + fn new(copy_from: Option<&ContextResourceUsage>) -> Self { + copy_from.cloned().unwrap_or_default() + } + + /// Return a copy of this object. + fn copy(&self) -> ContextResourceUsage { + self.clone() + } + + /// Reset all stats to zero. + fn reset(&mut self) { + *self = ContextResourceUsage::default(); + } + + fn __repr__(&self) -> String { + // The single-quoted, `repr()`-style value formatting is the shape + // this class logs in, and scrapers may match on it — keep it stable. + // Rust's `{:?}` renders exponent-form floats as e.g. `1e-7` (Python + // `repr` writes `1e-07`); the only consumer of the string form is + // Measure's "Failed to save metrics!" warning, so we don't chase exact + // parity with Python there. + format!( + "", + self.ru_stime, + self.ru_utime, + self.db_txn_count, + self.db_txn_duration_sec, + self.db_sched_duration_sec, + self.evt_db_fetch_count, + ) + } + + /// `self += other`; mutate in place. pyo3 returns `self` for the in-place slot. + fn __iadd__(&mut self, other: &ContextResourceUsage) { + self.add_assign(other); + } + + /// `self -= other`; mutate in place. pyo3 returns `self` for the in-place slot. + fn __isub__(&mut self, other: &ContextResourceUsage) { + self.sub_assign(other); + } + + /// `self + other`, returning a new object. + fn __add__(&self, other: &ContextResourceUsage) -> ContextResourceUsage { + let mut res = self.clone(); + res.add_assign(other); + res + } + + /// `self - other`, returning a new object. + fn __sub__(&self, other: &ContextResourceUsage) -> ContextResourceUsage { + let mut res = self.clone(); + res.sub_assign(other); + res + } +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module: Bound<'_, PyModule> = PyModule::new(py, "logcontext")?; + child_module.add_class::()?; + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import logcontext` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.logcontext", child_module)?; + + Ok(()) +} diff --git a/rust/src/logging/mod.rs b/rust/src/logging/mod.rs new file mode 100644 index 0000000000..1d6d898cb4 --- /dev/null +++ b/rust/src/logging/mod.rs @@ -0,0 +1,23 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Rust counterparts to the `synapse.logging` Python package. +//! +//! Submodules live at the Rust module path that mirrors their Python logging +//! namespace (e.g. `synapse::logging::context` -> `synapse.logging.context`), so +//! that log records emitted from here via the `log` crate land under the matching +//! Python logger without an explicit `target:`. + +pub mod context; diff --git a/synapse/logging/context.py b/synapse/logging/context.py index b6535be388..0142182836 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -52,6 +52,10 @@ from twisted.internet import defer, threads from twisted.python.threadpool import ThreadPool from synapse.logging.loggers import ExplicitlyConfiguredLogger +from synapse.synapse_rust.logcontext import ( + # Not used in this module, but re-exported: callers import it from here. + ContextResourceUsage, # noqa: F401 +) from synapse.util.stringutils import random_string_insecure_fast if TYPE_CHECKING: @@ -113,107 +117,6 @@ def logcontext_error(msg: str) -> None: get_thread_id = threading.get_ident -class ContextResourceUsage: - """Object for tracking the resources used by a log context - - Attributes: - ru_utime (float): user CPU time (in seconds) - ru_stime (float): system CPU time (in seconds) - db_txn_count (int): number of database transactions done - db_sched_duration_sec (float): amount of time spent waiting for a - database connection - db_txn_duration_sec (float): amount of time spent doing database - transactions (excluding scheduling time) - evt_db_fetch_count (int): number of events requested from the database - """ - - __slots__ = [ - "ru_stime", - "ru_utime", - "db_txn_count", - "db_txn_duration_sec", - "db_sched_duration_sec", - "evt_db_fetch_count", - ] - - def __init__(self, copy_from: "ContextResourceUsage | None" = None) -> None: - """Create a new ContextResourceUsage - - Args: - copy_from: if not None, an object to copy stats from - """ - if copy_from is None: - self.reset() - else: - # FIXME: mypy can't infer the types set via reset() above, so specify explicitly for now - self.ru_utime: float = copy_from.ru_utime - self.ru_stime: float = copy_from.ru_stime - self.db_txn_count: int = copy_from.db_txn_count - - self.db_txn_duration_sec: float = copy_from.db_txn_duration_sec - self.db_sched_duration_sec: float = copy_from.db_sched_duration_sec - self.evt_db_fetch_count: int = copy_from.evt_db_fetch_count - - def copy(self) -> "ContextResourceUsage": - return ContextResourceUsage(copy_from=self) - - def reset(self) -> None: - self.ru_stime = 0.0 - self.ru_utime = 0.0 - self.db_txn_count = 0 - - self.db_txn_duration_sec = 0.0 - self.db_sched_duration_sec = 0.0 - self.evt_db_fetch_count = 0 - - def __repr__(self) -> str: - return ( - "" - ) % ( - self.ru_stime, - self.ru_utime, - self.db_txn_count, - self.db_txn_duration_sec, - self.db_sched_duration_sec, - self.evt_db_fetch_count, - ) - - def __iadd__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - """Add another ContextResourceUsage's stats to this one's. - - Args: - other: the other resource usage object - """ - self.ru_utime += other.ru_utime - self.ru_stime += other.ru_stime - self.db_txn_count += other.db_txn_count - self.db_txn_duration_sec += other.db_txn_duration_sec - self.db_sched_duration_sec += other.db_sched_duration_sec - self.evt_db_fetch_count += other.evt_db_fetch_count - return self - - def __isub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - self.ru_utime -= other.ru_utime - self.ru_stime -= other.ru_stime - self.db_txn_count -= other.db_txn_count - self.db_txn_duration_sec -= other.db_txn_duration_sec - self.db_sched_duration_sec -= other.db_sched_duration_sec - self.evt_db_fetch_count -= other.evt_db_fetch_count - return self - - def __add__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - res = ContextResourceUsage(copy_from=self) - res += other - return res - - def __sub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - res = ContextResourceUsage(copy_from=self) - res -= other - return res - - @attr.s(slots=True, auto_attribs=True) class ContextRequest: """ diff --git a/synapse/synapse_rust/logcontext.pyi b/synapse/synapse_rust/logcontext.pyi new file mode 100644 index 0000000000..dc0e955518 --- /dev/null +++ b/synapse/synapse_rust/logcontext.pyi @@ -0,0 +1,31 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import Optional + +class ContextResourceUsage: + """Tracks the resources used by a log context.""" + + ru_stime: float + ru_utime: float + db_txn_count: int + db_txn_duration_sec: float + db_sched_duration_sec: float + evt_db_fetch_count: int + + def __init__(self, copy_from: "Optional[ContextResourceUsage]" = None) -> None: ... + def copy(self) -> "ContextResourceUsage": ... + def reset(self) -> None: ... + def __iadd__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": ... + def __isub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": ... + def __add__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": ... + def __sub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": ...