mirror of
https://github.com/element-hq/synapse.git
synced 2026-11-04 11:17:51 +00:00
`PersistReader` wraps a `SyncReader` and adds the reads the persist controller makes before its write transaction: forward and state DAG extremities, the prev event walks of `_get_events_which_are_prevs` and `_get_prevs_before_rejected`, `have_seen_events`, memberships by event ID, `is_local_host_in_room_ignoring_users`, the room version, the sliding sync joined room check, `get_last_event_pos_in_room` and the soft failed sticky events. None of them is cached. Each runs once per chunk. The chain cover reads are free functions over a `Querier`, not reader methods. The chain cover step reads, allocates chain IDs and writes in one transaction, so they must run on that transaction's connection. To allow that, `WriteTxn` now implements `Querier`. A `WriteError` that passes through `anyhow` on the way out comes back as itself, so a conflict is still retried. The persister builds its own `SyncStore` over the shared caches. `tests/storage/test_persist_reader.py` runs each read against the Python method or statement it ports, on the native driver. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UHKYPSc37kWqGACn9aKxKL
157 lines
6.5 KiB
Python
157 lines
6.5 KiB
Python
#
|
|
# This file is licensed under the Affero General Public License (AGPL) version 3.
|
|
#
|
|
# Copyright (C) 2026 Element Creations Ltd
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# See the GNU Affero General Public License for more details:
|
|
# <https://www.gnu.org/licenses/agpl-3.0.html>.
|
|
|
|
from typing import Any
|
|
|
|
from synapse.events import EventBase
|
|
from synapse.events.snapshot import EventContext
|
|
from synapse.synapse_rust.database.postgres import ConnectionPool
|
|
|
|
class Persister:
|
|
"""Persists events in Rust, on the events writer, over the native driver.
|
|
See `experimental_features.rust_event_persistence`. Get it from
|
|
`hs.get_rust_handlers().persister`."""
|
|
|
|
def take_metrics(self) -> dict[str, Any]:
|
|
"""What persisting has counted since the last call. See
|
|
`flush_rust_persist_metrics`."""
|
|
|
|
class PersistEffects:
|
|
"""The Python cache updates a Rust persist transaction asks for once it
|
|
has committed, which `DataStore._apply_rust_persist_effects` applies.
|
|
Each attribute matches one kind of `txn.call_after` callback of the
|
|
Python transaction."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
events: list[
|
|
tuple[int, str, str, str, str | None, str | None, str | None, bool]
|
|
] = ...,
|
|
event_cache_invalidations: list[str] = ...,
|
|
state_caches: list[tuple[str, list[str]]] = ...,
|
|
cache_invalidations: list[tuple[str, list[str]]] = ...,
|
|
state_group_prefills: list[tuple[str, int]] = ...,
|
|
latest_event_ids: tuple[str, list[str]] | None = None,
|
|
stream_changes: list[tuple[str, str, int]] = ...,
|
|
stream_ids: list[int] = ...,
|
|
backfilled: bool = False,
|
|
new_replication_data: bool = False,
|
|
) -> None:
|
|
"""Build effects in Python, for tests."""
|
|
|
|
@property
|
|
def events(
|
|
self,
|
|
) -> list[tuple[int, str, str, str, str | None, str | None, str | None, bool]]:
|
|
"""The arguments of `_invalidate_caches_for_event`, per event:
|
|
stream ordering, event ID, room ID, type, state key, redacts,
|
|
relates to, backfilled."""
|
|
|
|
@property
|
|
def event_cache_invalidations(self) -> list[str]:
|
|
"""Events to drop from the event cache."""
|
|
|
|
@property
|
|
def state_caches(self) -> list[tuple[str, list[str]]]:
|
|
"""`_invalidate_state_caches(room_id, members_changed)` calls."""
|
|
|
|
@property
|
|
def cache_invalidations(self) -> list[tuple[str, list[str]]]:
|
|
"""`(cache name, key)`: the local half of
|
|
`_invalidate_cache_and_stream`."""
|
|
|
|
@property
|
|
def state_group_prefills(self) -> list[tuple[str, int]]:
|
|
"""`(event ID, state group)` for `_get_state_group_for_event`."""
|
|
|
|
@property
|
|
def latest_event_ids(self) -> tuple[str, list[str]] | None:
|
|
"""`(room ID, forward extremities)` for
|
|
`get_latest_event_ids_in_room`."""
|
|
|
|
@property
|
|
def stream_changes(self) -> list[tuple[str, str, int]]:
|
|
"""`(cache, entity, stream ID)`, where cache is "events",
|
|
"current_state_deltas" or "membership"."""
|
|
|
|
@property
|
|
def stream_ids(self) -> list[int]:
|
|
"""The chunk's stream IDs, still unfinished. Python marks them
|
|
finished after applying everything else, so the current token never
|
|
passes an event the stream change caches don't know about. Positive
|
|
on the backfill stream too, as the ID generator tracks them."""
|
|
|
|
@property
|
|
def backfilled(self) -> bool:
|
|
"""Whether `stream_ids` are on the backfill stream."""
|
|
|
|
@property
|
|
def new_replication_data(self) -> bool:
|
|
"""Whether rows were written to the cache invalidation stream."""
|
|
|
|
class PersistReaderForTest:
|
|
"""The persist reads (`rust/src/persist/reader.rs`), each run to
|
|
completion on the tokio runtime over the native driver's pool. The chain
|
|
cover reads each get a connection of their own. For
|
|
`tests/storage/test_persist_reader.py`."""
|
|
|
|
def __init__(self, pool: ConnectionPool, persister: Persister) -> None: ...
|
|
def latest_event_ids_in_room(self, room_id: str) -> set[str]: ...
|
|
def state_dag_extremities(self, room_id: str) -> set[str]: ...
|
|
def events_which_are_prevs(self, event_ids: list[str]) -> list[str]: ...
|
|
def prevs_before_rejected(
|
|
self, event_ids: list[str], include_soft_failed: bool
|
|
) -> set[str]: ...
|
|
def have_seen_events(self, event_ids: list[str]) -> set[str]: ...
|
|
def membership_from_event_ids(
|
|
self, event_ids: list[str]
|
|
) -> dict[str, tuple[str, str]]:
|
|
"""Event ID to `(user_id, membership)`, for membership events only."""
|
|
def is_local_host_in_room_ignoring_users(
|
|
self, room_id: str, ignore_users: list[str]
|
|
) -> bool: ...
|
|
def room_version_id(self, room_id: str) -> str | None: ...
|
|
def has_sliding_sync_joined_room(self, room_id: str) -> bool: ...
|
|
def last_event_pos_in_room(
|
|
self, room_id: str, event_types: list[str] | None = None
|
|
) -> tuple[str, str, int] | None:
|
|
"""`(event_id, instance_name, stream)`."""
|
|
def soft_failed_sticky_events(
|
|
self, room_id: str, senders: list[str] | None, now_ms: int
|
|
) -> list[str]: ...
|
|
def has_auth_chain_index(self, room_id: str) -> bool: ...
|
|
def events_with_chain_ids(self, event_ids: list[str]) -> set[str]: ...
|
|
def chains_to_calculate(self, room_ids: list[str]) -> list[tuple[str, str, str]]:
|
|
"""`(event_id, type, state_key)` rows."""
|
|
def auth_event_ids(self, event_id: str) -> list[str]: ...
|
|
def auth_event_chains(
|
|
self, event_ids: list[str]
|
|
) -> list[tuple[str, str, str, int | None, int | None]]:
|
|
"""`(event_id, type, state_key, chain_id, sequence_number)` rows."""
|
|
def chain_links(self, chains: list[int]) -> list[tuple[int, int, int, int]]:
|
|
"""`(origin_chain_id, origin_sequence_number, target_chain_id,
|
|
target_sequence_number)` rows."""
|
|
def max_sequence_numbers(self, chain_ids: list[int]) -> dict[int, int]: ...
|
|
|
|
def persist_gil_stats() -> dict[str, tuple[int, float]]:
|
|
"""`{site: (calls, seconds held)}` for every time event persistence took
|
|
the GIL from tokio."""
|
|
|
|
def persistence_json_for_test(event: EventBase) -> tuple[str, str]:
|
|
"""The `internal_metadata` and `json` columns of `event_json` as Rust
|
|
would write them for `event`."""
|
|
|
|
def persist_context_for_test(context: EventContext) -> dict[str, Any]:
|
|
"""The fields of `context` as the Rust persister reads them."""
|