Files
synapse/synapse/metrics/thread_cpu.py
T
Erik Johnston 20d925bb33 Add a metric for CPU time per thread type (#20405)
Adds a `synapse_thread_cpu_seconds_total{thread_type, mode}` metric,
grouping threads into:
1. the `main` thread
2. each registered threadpool (`threadpool_*`)
3. `tokio`
4. `other`

The kernel only reports CPU time for live threads, so the collector sums
per-thread deltas between scrapes. A thread that exits loses the CPU it
used since the last scrape.

This is primarily useful for seeing how busy the main thread is vs other
ones.
2026-11-09 10:30:04 +00:00

227 lines
8.5 KiB
Python

#
# This file is licensed under the Affero General Public License (AGPL) version 3.
#
# Copyright (C) 2026 Element Creations Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# See the GNU Affero General Public License for more details:
# <https://www.gnu.org/licenses/agpl-3.0.html>.
#
"""Prometheus metrics for CPU time per thread type."""
import logging
import os
import threading
from collections import Counter
from typing import Iterable
from weakref import WeakKeyDictionary
import attr
from prometheus_client.core import REGISTRY, CounterMetricFamily, Metric
from twisted.python.threadpool import ThreadPool
from synapse.metrics._types import Collector
logger = logging.getLogger(__name__)
@attr.s(slots=True, frozen=True, auto_attribs=True)
class _ThreadSample:
"""A thread's CPU usage as of one scrape."""
start_time: int
"""When the thread started, in ticks since boot. Used to distinguish a new
thread that reuses the ID of one that exited.
"""
utime: int
"""User CPU time used by the thread, in ticks."""
stime: int
"""System CPU time used by the thread, in ticks."""
class ThreadCPUMetrics(Collector):
"""Reports the CPU time used by each type of thread in the process.
Threads are grouped by what they run: the main (reactor) thread, a
registered type, the `tokio` runtime, and `other` for everything else.
Note: "other" can include threads like prometheus HTTP server threads.
Linux only, as it reads `/proc/self/task`. The kernel only reports CPU time
for live threads, so the collector sums the CPU time each thread used
between scrapes. The CPU time a thread uses between the last scrape and
its exit is not counted.
"""
def __init__(self) -> None:
# The kernel reports thread CPU time in clock ticks. This converts
# ticks to seconds. It is 100 on almost every Linux system.
self._ticks_per_sec = 100
try:
self._ticks_per_sec = os.sysconf("SC_CLK_TCK")
except (ValueError, TypeError, AttributeError):
pass
# Guards the fields below, as the metrics listener may scrape from any
# thread.
self._lock = threading.Lock()
self._threadpools: "WeakKeyDictionary[ThreadPool, str]" = WeakKeyDictionary()
self._threads: "WeakKeyDictionary[threading.Thread, str]" = WeakKeyDictionary()
# Thread ID to the thread's sample from the previous scrape.
self._last_seen: dict[int, _ThreadSample] = {}
# (thread type, mode) to the total ticks used by threads of that type.
self._totals: Counter[tuple[str, str]] = Counter()
def register_threadpool(self, name: str, threadpool: ThreadPool) -> None:
"""Report the threads of `threadpool` as the type `threadpool_<name>`.
A threadpool keeps the name it was first registered under.
"""
with self._lock:
self._threadpools.setdefault(threadpool, name)
def register_thread(self, thread_type: str, thread: threading.Thread) -> None:
"""Report `thread` as the type `thread_type`."""
with self._lock:
self._threads[thread] = thread_type
def _registered_thread_types(self) -> dict[int, str]:
"""Map the native IDs of registered threads and threadpool threads to
their thread type.
Requires the caller to hold `self._lock`.
"""
assert self._lock.locked()
types: dict[int, str] = {}
for thread, thread_type in list(self._threads.items()):
if thread.native_id is not None and thread.is_alive():
types[thread.native_id] = thread_type
for threadpool, name in list(self._threadpools.items()):
# Twisted never removes exited threads from `threads`. The kernel
# may have given their IDs to new threads.
for thread in list(threadpool.threads):
if thread.native_id is not None and thread.is_alive():
types[thread.native_id] = f"threadpool_{name}"
return types
def _thread_type(self, tid: int, registered_types: dict[int, str]) -> str:
"""Work out the thread type to report a thread's CPU time under.
Args:
tid: The thread's native ID.
registered_types: The output of `_registered_thread_types`.
Returns:
`main`, a registered type, `tokio` or `other`.
"""
if tid == os.getpid():
return "main"
registered_type = registered_types.get(tid)
if registered_type is not None:
return registered_type
try:
with open(f"/proc/self/task/{tid}/comm") as f:
comm = f.read().strip()
except OSError:
return "other"
# Tokio names its worker and blocking threads `tokio-rt-worker`, or
# `tokio-runtime-worker` in older versions.
if comm.startswith("tokio-"):
return "tokio"
return "other"
def collect(self) -> Iterable[Metric]:
"""Add the CPU time each live thread used since the previous scrape to
its type's total, and report the totals.
A thread that exits loses the CPU it used since the last scrape.
"""
try:
tids = [int(tid) for tid in os.listdir("/proc/self/task")]
except OSError:
return
with self._lock:
registered_types = self._registered_thread_types()
last_seen: dict[int, _ThreadSample] = {}
for tid in tids:
try:
with open(f"/proc/self/task/{tid}/stat") as f:
line = f.read()
except OSError:
# The thread exited since we listed it. We lose this bit of
# CPU tracing, but it's hard to recover it after the thread
# has exited.
#
# We can see in the metrics how much thread CPU has been
# lost, by comparing total CPU vs. the sum of CPU used by
# all threads.
continue
# The thread name may contain spaces or brackets, so split after
# its closing bracket. See `man 5 proc_pid_stat` for the field
# numbers (note they're off by three due to zero-based indexing
# and we have split off the first two fields).
#
# Example of the start of such a line:
#
# 171453 (cat) R 30798 171453 ...
fields = line.rsplit(")", 1)[1].split()
utime = int(fields[11])
stime = int(fields[12])
start_time = int(fields[19])
prev = self._last_seen.get(tid)
if prev is not None and prev.start_time == start_time:
prev_utime, prev_stime = prev.utime, prev.stime
# It's possible (though unlikely) that a thread has exited
# and got reused in the same tick, so ensure that times have
# strictly increased.
if utime < prev_utime or stime < prev_stime:
prev_utime = prev_stime = 0
else:
prev_utime = prev_stime = 0
thread_type = self._thread_type(tid, registered_types)
self._totals[(thread_type, "user")] += utime - prev_utime
self._totals[(thread_type, "system")] += stime - prev_stime
last_seen[tid] = _ThreadSample(start_time, utime, stime)
self._last_seen = last_seen
# This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`.
cpu = CounterMetricFamily( # type: ignore[missing-server-name-label]
"synapse_thread_cpu_seconds",
"CPU time used by each type of thread in the process",
labels=["thread_type", "mode"],
)
for (thread_type, mode), ticks in sorted(self._totals.items()):
cpu.add_metric([thread_type, mode], ticks / self._ticks_per_sec)
yield cpu
# This is a process-level metric, so it does not have the `SERVER_NAME_LABEL`.
thread_cpu_metrics = ThreadCPUMetrics() # type: ignore[missing-server-name-label]
REGISTRY.register(thread_cpu_metrics)