mirror of
https://github.com/element-hq/synapse.git
synced 2026-09-25 22:04:00 +00:00
83193b378d
If we failed to persist an event and then retried, this would fail due to a conflict error trying to reinsert the state groups that we are persisting. This blocks all new events being persisted in the room until the instance is restarted (which clears entries from the table matching the instance name). This is because we always mark the [latest persisted state group in the room as being persisted](https://github.com/element-hq/synapse/blob/781d08df263e640d85ca5c8b5ff514433e019c6f/synapse/storage/databases/state/deletion.py#L242-L243). Also add an `inserted_ts` field which we use to detect stale entries, e.g. from instances that have been taken offline permanently. These aren't a problem beyond blocking deletion of the associated state groups Introduced in https://github.com/element-hq/synapse/pull/18107 --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
524 lines
19 KiB
Python
524 lines
19 KiB
Python
#
|
|
# This file is licensed under the Affero General Public License (AGPL) version 3.
|
|
#
|
|
# Copyright (C) 2025 New Vector, Ltd
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# See the GNU Affero General Public License for more details:
|
|
# <https://www.gnu.org/licenses/agpl-3.0.html>.
|
|
#
|
|
|
|
|
|
import logging
|
|
from collections.abc import Collection
|
|
from unittest.mock import patch
|
|
|
|
from twisted.internet.testing import MemoryReactor
|
|
|
|
from synapse.rest import admin
|
|
from synapse.rest.client import login, room
|
|
from synapse.server import HomeServer
|
|
from synapse.storage.database import LoggingTransaction
|
|
from synapse.util.clock import Clock
|
|
|
|
from tests.test_utils.event_injection import create_event
|
|
from tests.unittest import HomeserverTestCase
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class StateDeletionStoreTestCase(HomeserverTestCase):
|
|
"""Tests for the StateDeletionStore."""
|
|
|
|
servlets = [
|
|
admin.register_servlets,
|
|
room.register_servlets,
|
|
login.register_servlets,
|
|
]
|
|
|
|
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
|
|
self.store = hs.get_datastores().main
|
|
self.state_store = hs.get_datastores().state
|
|
self.state_deletion_store = hs.get_datastores().state_deletion
|
|
self.purge_events = hs.get_storage_controllers().purge_events
|
|
|
|
# We want to disable the automatic deletion of state groups in the
|
|
# background, so we can do controlled tests.
|
|
self.purge_events._delete_state_loop_call.stop()
|
|
|
|
self.user_id = self.register_user("test", "password")
|
|
self.tok = self.login("test", "password")
|
|
self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok)
|
|
|
|
def get_persisting_marker_rows(self) -> list[tuple[int, str, int]]:
|
|
"""Return the contents of the `state_groups_persisting` table."""
|
|
|
|
return self.get_success(
|
|
self.state_deletion_store.db_pool.simple_select_list(
|
|
table="state_groups_persisting",
|
|
keyvalues=None,
|
|
retcols=("state_group", "instance_name", "inserted_ts"),
|
|
desc="get_persisting_marker_rows",
|
|
)
|
|
)
|
|
|
|
def check_if_can_be_deleted(self, state_group: int) -> bool:
|
|
"""Check if the state group is pending deletion."""
|
|
|
|
state_group_to_sequence_number = self.get_success(
|
|
self.state_deletion_store.get_pending_deletions([state_group])
|
|
)
|
|
|
|
can_be_deleted = self.get_success(
|
|
self.state_deletion_store.db_pool.runInteraction(
|
|
"test_existing_pending_deletion_is_cleared",
|
|
self.state_deletion_store.get_state_groups_ready_for_potential_deletion_txn,
|
|
state_group_to_sequence_number,
|
|
)
|
|
)
|
|
|
|
return state_group in can_be_deleted
|
|
|
|
def test_no_deletion(self) -> None:
|
|
"""Test that calling persisting_state_group_references is fine if
|
|
nothing is pending deletion"""
|
|
event, context = self.get_success(
|
|
create_event(
|
|
self.hs,
|
|
room_id=self.room_id,
|
|
type="m.test",
|
|
sender=self.user_id,
|
|
)
|
|
)
|
|
|
|
ctx_mgr = self.state_deletion_store.persisting_state_group_references(
|
|
[(event, context)]
|
|
)
|
|
|
|
self.get_success(ctx_mgr.__aenter__())
|
|
self.get_success(ctx_mgr.__aexit__(None, None, None))
|
|
|
|
def test_no_deletion_error(self) -> None:
|
|
"""Test that calling persisting_state_group_references is fine if
|
|
nothing is pending deletion, but an error occurs."""
|
|
|
|
event, context = self.get_success(
|
|
create_event(
|
|
self.hs,
|
|
room_id=self.room_id,
|
|
type="m.test",
|
|
sender=self.user_id,
|
|
)
|
|
)
|
|
|
|
ctx_mgr = self.state_deletion_store.persisting_state_group_references(
|
|
[(event, context)]
|
|
)
|
|
|
|
self.get_success(ctx_mgr.__aenter__())
|
|
self.get_success(ctx_mgr.__aexit__(Exception, Exception("test"), None))
|
|
|
|
def test_retry_send_after_failed_clean_up(self) -> None:
|
|
"""Test that we can retry sending an event after the clean up of the
|
|
`state_groups_persisting` rows failed, as it does when the database goes
|
|
away while we're persisting."""
|
|
|
|
fail_clean_up = True
|
|
orig_finish_persisting_txn = self.state_deletion_store._finish_persisting_txn
|
|
|
|
def _finish_persisting_txn(
|
|
txn: LoggingTransaction, state_groups: Collection[int], error: bool
|
|
) -> None:
|
|
if fail_clean_up:
|
|
raise Exception("Database has gone away")
|
|
|
|
orig_finish_persisting_txn(txn, state_groups, error)
|
|
|
|
with patch.object(
|
|
self.state_deletion_store,
|
|
"_finish_persisting_txn",
|
|
new=_finish_persisting_txn,
|
|
):
|
|
self.helper.send(self.room_id, body="first", tok=self.tok, expect_code=500)
|
|
|
|
# The rows are still in place, as the transaction that would have
|
|
# removed them was rolled back.
|
|
self.assertNotEqual(self.get_persisting_marker_rows(), [])
|
|
|
|
# The database has come back, so the retry should now go through.
|
|
fail_clean_up = False
|
|
self.helper.send(self.room_id, body="second", tok=self.tok)
|
|
|
|
self.assertEqual(self.get_persisting_marker_rows(), [])
|
|
|
|
def test_existing_pending_deletion_is_cleared(self) -> None:
|
|
"""Test that the pending deletion flag gets cleared when the state group
|
|
gets persisted."""
|
|
|
|
event, context = self.get_success(
|
|
create_event(
|
|
self.hs,
|
|
room_id=self.room_id,
|
|
type="m.test",
|
|
state_key="",
|
|
sender=self.user_id,
|
|
)
|
|
)
|
|
assert context.state_group is not None
|
|
|
|
# Mark a state group that we're referencing as pending deletion.
|
|
self.get_success(
|
|
self.state_deletion_store.mark_state_groups_as_pending_deletion(
|
|
[context.state_group]
|
|
)
|
|
)
|
|
|
|
ctx_mgr = self.state_deletion_store.persisting_state_group_references(
|
|
[(event, context)]
|
|
)
|
|
|
|
self.get_success(ctx_mgr.__aenter__())
|
|
self.get_success(ctx_mgr.__aexit__(None, None, None))
|
|
|
|
# The pending deletion flag should be cleared
|
|
pending_deletion = self.get_success(
|
|
self.state_deletion_store.db_pool.simple_select_one_onecol(
|
|
table="state_groups_pending_deletion",
|
|
keyvalues={"state_group": context.state_group},
|
|
retcol="1",
|
|
allow_none=True,
|
|
desc="test_existing_pending_deletion_is_cleared",
|
|
)
|
|
)
|
|
self.assertIsNone(pending_deletion)
|
|
|
|
def test_pending_deletion_is_cleared_during_persist(self) -> None:
|
|
"""Test that the pending deletion flag is cleared when a state group
|
|
gets marked for deletion during persistence"""
|
|
|
|
event, context = self.get_success(
|
|
create_event(
|
|
self.hs,
|
|
room_id=self.room_id,
|
|
type="m.test",
|
|
state_key="",
|
|
sender=self.user_id,
|
|
)
|
|
)
|
|
assert context.state_group is not None
|
|
|
|
ctx_mgr = self.state_deletion_store.persisting_state_group_references(
|
|
[(event, context)]
|
|
)
|
|
self.get_success(ctx_mgr.__aenter__())
|
|
|
|
# Mark the state group that we're referencing as pending deletion,
|
|
# *after* we have started persisting.
|
|
self.get_success(
|
|
self.state_deletion_store.mark_state_groups_as_pending_deletion(
|
|
[context.state_group]
|
|
)
|
|
)
|
|
|
|
self.get_success(ctx_mgr.__aexit__(None, None, None))
|
|
|
|
# The pending deletion flag should be cleared
|
|
pending_deletion = self.get_success(
|
|
self.state_deletion_store.db_pool.simple_select_one_onecol(
|
|
table="state_groups_pending_deletion",
|
|
keyvalues={"state_group": context.state_group},
|
|
retcol="1",
|
|
allow_none=True,
|
|
desc="test_existing_pending_deletion_is_cleared",
|
|
)
|
|
)
|
|
self.assertIsNone(pending_deletion)
|
|
|
|
def test_deletion_check(self) -> None:
|
|
"""Test that the `get_state_groups_that_can_be_purged_txn` check is
|
|
correct during different points of the lifecycle of persisting an
|
|
event."""
|
|
event, context = self.get_success(
|
|
create_event(
|
|
self.hs,
|
|
room_id=self.room_id,
|
|
type="m.test",
|
|
state_key="",
|
|
sender=self.user_id,
|
|
)
|
|
)
|
|
assert context.state_group is not None
|
|
|
|
self.get_success(
|
|
self.state_deletion_store.mark_state_groups_as_pending_deletion(
|
|
[context.state_group]
|
|
)
|
|
)
|
|
|
|
# We shouldn't be able to delete the state group as not enough time as passed
|
|
can_be_deleted = self.check_if_can_be_deleted(context.state_group)
|
|
self.assertFalse(can_be_deleted)
|
|
|
|
# After enough time we can delete the state group
|
|
self.reactor.advance(
|
|
1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
|
|
)
|
|
can_be_deleted = self.check_if_can_be_deleted(context.state_group)
|
|
self.assertTrue(can_be_deleted)
|
|
|
|
ctx_mgr = self.state_deletion_store.persisting_state_group_references(
|
|
[(event, context)]
|
|
)
|
|
self.get_success(ctx_mgr.__aenter__())
|
|
|
|
# But once we start persisting we can't delete the state group
|
|
can_be_deleted = self.check_if_can_be_deleted(context.state_group)
|
|
self.assertFalse(can_be_deleted)
|
|
|
|
self.get_success(ctx_mgr.__aexit__(None, None, None))
|
|
|
|
# The pending deletion flag should remain cleared after persistence has
|
|
# finished.
|
|
can_be_deleted = self.check_if_can_be_deleted(context.state_group)
|
|
self.assertFalse(can_be_deleted)
|
|
|
|
def test_deletion_error_during_persistence(self) -> None:
|
|
"""Test that state groups remain marked as pending deletion if persisting
|
|
the event fails."""
|
|
|
|
event, context = self.get_success(
|
|
create_event(
|
|
self.hs,
|
|
room_id=self.room_id,
|
|
type="m.test",
|
|
state_key="",
|
|
sender=self.user_id,
|
|
)
|
|
)
|
|
assert context.state_group is not None
|
|
|
|
# Mark a state group that we're referencing as pending deletion.
|
|
self.get_success(
|
|
self.state_deletion_store.mark_state_groups_as_pending_deletion(
|
|
[context.state_group]
|
|
)
|
|
)
|
|
|
|
ctx_mgr = self.state_deletion_store.persisting_state_group_references(
|
|
[(event, context)]
|
|
)
|
|
|
|
self.get_success(ctx_mgr.__aenter__())
|
|
self.get_success(ctx_mgr.__aexit__(Exception, Exception("test"), None))
|
|
|
|
# We should be able to delete the state group after a certain amount of
|
|
# time
|
|
self.reactor.advance(
|
|
1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
|
|
)
|
|
can_be_deleted = self.check_if_can_be_deleted(context.state_group)
|
|
self.assertTrue(can_be_deleted)
|
|
|
|
def test_race_between_check_and_insert(self) -> None:
|
|
"""Check that we correctly handle the race where we go to delete a
|
|
state group, check that it is unreferenced, and then it becomes
|
|
referenced just before we delete it."""
|
|
|
|
event, context = self.get_success(
|
|
create_event(
|
|
self.hs,
|
|
room_id=self.room_id,
|
|
type="m.test",
|
|
state_key="",
|
|
sender=self.user_id,
|
|
)
|
|
)
|
|
assert context.state_group is not None
|
|
|
|
# Mark a state group that we're referencing as pending deletion.
|
|
self.get_success(
|
|
self.state_deletion_store.mark_state_groups_as_pending_deletion(
|
|
[context.state_group]
|
|
)
|
|
)
|
|
|
|
# Advance time enough so we can delete the state group
|
|
self.reactor.advance(
|
|
1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
|
|
)
|
|
|
|
# Check that we'd be able to delete this state group.
|
|
state_group_to_sequence_number = self.get_success(
|
|
self.state_deletion_store.get_pending_deletions([context.state_group])
|
|
)
|
|
|
|
can_be_deleted = self.get_success(
|
|
self.state_deletion_store.db_pool.runInteraction(
|
|
"test_existing_pending_deletion_is_cleared",
|
|
self.state_deletion_store.get_state_groups_ready_for_potential_deletion_txn,
|
|
state_group_to_sequence_number,
|
|
)
|
|
)
|
|
self.assertCountEqual(can_be_deleted, [context.state_group])
|
|
|
|
# ... in the real world we'd check that the state group isn't referenced here ...
|
|
|
|
# Now we persist the event to reference the state group, *after* we
|
|
# check that the state group wasn't referenced
|
|
ctx_mgr = self.state_deletion_store.persisting_state_group_references(
|
|
[(event, context)]
|
|
)
|
|
|
|
self.get_success(ctx_mgr.__aenter__())
|
|
self.get_success(ctx_mgr.__aexit__(Exception, Exception("test"), None))
|
|
|
|
# We simulate a pause (required to hit the race)
|
|
self.reactor.advance(
|
|
1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
|
|
)
|
|
|
|
# We should no longer be able to delete the state group, without having
|
|
# to recheck if its referenced.
|
|
can_be_deleted = self.get_success(
|
|
self.state_deletion_store.db_pool.runInteraction(
|
|
"test_existing_pending_deletion_is_cleared",
|
|
self.state_deletion_store.get_state_groups_ready_for_potential_deletion_txn,
|
|
state_group_to_sequence_number,
|
|
)
|
|
)
|
|
self.assertCountEqual(can_be_deleted, [])
|
|
|
|
def test_remove_ancestors_from_can_delete(self) -> None:
|
|
"""Test that if a state group is not ready to be deleted, we also don't
|
|
delete anything that is referenced by it"""
|
|
|
|
event, context = self.get_success(
|
|
create_event(
|
|
self.hs,
|
|
room_id=self.room_id,
|
|
type="m.test",
|
|
state_key="",
|
|
sender=self.user_id,
|
|
)
|
|
)
|
|
assert context.state_group is not None
|
|
|
|
# Create a new state group that references the one from the event
|
|
new_state_group = self.get_success(
|
|
self.state_store.store_state_group(
|
|
event.event_id,
|
|
event.room_id,
|
|
prev_group=context.state_group,
|
|
delta_ids={},
|
|
current_state_ids=None,
|
|
)
|
|
)
|
|
|
|
# Mark them both as pending deletion
|
|
self.get_success(
|
|
self.state_deletion_store.mark_state_groups_as_pending_deletion(
|
|
[context.state_group, new_state_group]
|
|
)
|
|
)
|
|
|
|
# Advance time enough so we can delete the state group so they're both
|
|
# ready for deletion.
|
|
self.reactor.advance(
|
|
1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
|
|
)
|
|
|
|
# We can now delete both state groups
|
|
self.assertTrue(self.check_if_can_be_deleted(context.state_group))
|
|
self.assertTrue(self.check_if_can_be_deleted(new_state_group))
|
|
|
|
# Use the new_state_group to bump its deletion time
|
|
self.get_success(
|
|
self.state_store.store_state_group(
|
|
event.event_id,
|
|
event.room_id,
|
|
prev_group=new_state_group,
|
|
delta_ids={},
|
|
current_state_ids=None,
|
|
)
|
|
)
|
|
|
|
# We should now not be able to delete either of the state groups.
|
|
state_group_to_sequence_number = self.get_success(
|
|
self.state_deletion_store.get_pending_deletions(
|
|
[context.state_group, new_state_group]
|
|
)
|
|
)
|
|
|
|
# We shouldn't be able to delete the state group as not enough time has passed
|
|
can_be_deleted = self.get_success(
|
|
self.state_deletion_store.db_pool.runInteraction(
|
|
"test_existing_pending_deletion_is_cleared",
|
|
self.state_deletion_store.get_state_groups_ready_for_potential_deletion_txn,
|
|
state_group_to_sequence_number,
|
|
)
|
|
)
|
|
self.assertCountEqual(can_be_deleted, [])
|
|
|
|
def test_newly_referenced_state_group_gets_removed_from_pending(self) -> None:
|
|
"""Check that if a state group marked for deletion becomes referenced
|
|
(without being removed from pending deletion table), it gets removed
|
|
from pending deletion table."""
|
|
|
|
event, context = self.get_success(
|
|
create_event(
|
|
self.hs,
|
|
room_id=self.room_id,
|
|
type="m.test",
|
|
state_key="",
|
|
sender=self.user_id,
|
|
)
|
|
)
|
|
assert context.state_group is not None
|
|
|
|
# Mark a state group that we're referencing as pending deletion.
|
|
self.get_success(
|
|
self.state_deletion_store.mark_state_groups_as_pending_deletion(
|
|
[context.state_group]
|
|
)
|
|
)
|
|
|
|
# Advance time enough so we can delete the state group so they're both
|
|
# ready for deletion.
|
|
self.reactor.advance(
|
|
1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
|
|
)
|
|
|
|
# Manually insert into the table to mimic the state group getting used.
|
|
self.get_success(
|
|
self.store.db_pool.simple_insert(
|
|
table="event_to_state_groups",
|
|
values={"state_group": context.state_group, "event_id": event.event_id},
|
|
desc="test_newly_referenced_state_group_gets_removed_from_pending",
|
|
)
|
|
)
|
|
|
|
# Manually run the background task to delete pending state groups.
|
|
self.get_success(self.purge_events._delete_state_groups_loop())
|
|
|
|
# The pending deletion flag should be cleared...
|
|
pending_deletion = self.get_success(
|
|
self.state_deletion_store.db_pool.simple_select_one_onecol(
|
|
table="state_groups_pending_deletion",
|
|
keyvalues={"state_group": context.state_group},
|
|
retcol="1",
|
|
allow_none=True,
|
|
desc="test_newly_referenced_state_group_gets_removed_from_pending",
|
|
)
|
|
)
|
|
self.assertIsNone(pending_deletion)
|
|
|
|
# .. but the state should not have been deleted.
|
|
state = self.get_success(
|
|
self.state_store._get_state_for_groups([context.state_group])
|
|
)
|
|
self.assertGreater(len(state[context.state_group]), 0)
|