mirror of
https://github.com/element-hq/synapse.git
synced 2026-08-14 07:10:48 +00:00
Olivier 'reivilibreandGitHub
2d06667661
Not aware of an open ticket for this. I came across it when I accidentally broke the feature even more (as part of another piece of work), then discovered there weren't tests for this. So this is overall a low-priority drive-by fix. Requires a fix to SyTest https://github.com/matrix-org/sytest/pull/1426 (as it depended on the bug). <ol> <li> Add a test for purging rooms with `delete_local_events=False` \ Parameterised by room version, this test currently succeeds on v2 but fails on v12. This is because the condition checking for local events relies on the old event ID format, which has not been used since v2. </li> <li> Fix delete_local_events=False for room versions above v2 \ The event ID format changing means that we have to rely on `sender` to know the origin of an event </li> </ol> --------- Signed-off-by: Olivier 'reivilibre <oliverw@matrix.org>
555 lines
21 KiB
Python
555 lines
21 KiB
Python
#
|
|
# This file is licensed under the Affero General Public License (AGPL) version 3.
|
|
#
|
|
# Copyright (C) 2023 New Vector, Ltd
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# See the GNU Affero General Public License for more details:
|
|
# <https://www.gnu.org/licenses/agpl-3.0.html>.
|
|
#
|
|
# Originally licensed under the Apache License, Version 2.0:
|
|
# <http://www.apache.org/licenses/LICENSE-2.0>.
|
|
#
|
|
# [This file includes modifications made by New Vector Limited]
|
|
#
|
|
#
|
|
|
|
from parameterized import parameterized
|
|
|
|
from twisted.internet.testing import MemoryReactor
|
|
|
|
from synapse.api.constants import EventTypes
|
|
from synapse.api.errors import NotFoundError, SynapseError
|
|
from synapse.api.room_versions import (
|
|
EventFormatVersions,
|
|
RoomVersion,
|
|
RoomVersions,
|
|
)
|
|
from synapse.events import EventBase
|
|
from synapse.rest.client import room
|
|
from synapse.server import HomeServer
|
|
from synapse.types.state import StateFilter
|
|
from synapse.types.storage import _BackgroundUpdates
|
|
from synapse.util.clock import Clock
|
|
|
|
from tests.test_utils.event_builders import make_test_pdu_event
|
|
from tests.unittest import HomeserverTestCase
|
|
|
|
|
|
class PurgeTests(HomeserverTestCase):
|
|
user_id = "@red:server"
|
|
servlets = [room.register_servlets]
|
|
|
|
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
|
|
hs = self.setup_test_homeserver("server")
|
|
return hs
|
|
|
|
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
|
|
self.room_id = self.helper.create_room_as(self.user_id)
|
|
|
|
self.store = hs.get_datastores().main
|
|
self.state_store = hs.get_datastores().state
|
|
self.state_deletion_store = hs.get_datastores().state_deletion
|
|
self._storage_controllers = self.hs.get_storage_controllers()
|
|
|
|
def test_purge_history(self) -> None:
|
|
"""
|
|
Purging a room history will delete everything before the topological point.
|
|
"""
|
|
# Send four messages to the room
|
|
first = self.helper.send(self.room_id, body="test1")
|
|
second = self.helper.send(self.room_id, body="test2")
|
|
third = self.helper.send(self.room_id, body="test3")
|
|
last = self.helper.send(self.room_id, body="test4")
|
|
|
|
# Get the topological token
|
|
token = self.get_success(
|
|
self.store.get_topological_token_for_event(last["event_id"])
|
|
)
|
|
token_str = self.get_success(token.to_string(self.hs.get_datastores().main))
|
|
|
|
# Purge everything before this topological token
|
|
self.get_success(
|
|
self._storage_controllers.purge_events.purge_history(
|
|
self.room_id, token_str, True
|
|
)
|
|
)
|
|
|
|
# 1-3 should fail and last will succeed, meaning that 1-3 are deleted
|
|
# and last is not.
|
|
self.get_failure(self.store.get_event(first["event_id"]), NotFoundError)
|
|
self.get_failure(self.store.get_event(second["event_id"]), NotFoundError)
|
|
self.get_failure(self.store.get_event(third["event_id"]), NotFoundError)
|
|
self.get_success(self.store.get_event(last["event_id"]))
|
|
|
|
def test_purge_history_wont_delete_extrems(self) -> None:
|
|
"""
|
|
Purging a room history will delete everything before the topological point.
|
|
"""
|
|
# Send four messages to the room
|
|
first = self.helper.send(self.room_id, body="test1")
|
|
second = self.helper.send(self.room_id, body="test2")
|
|
third = self.helper.send(self.room_id, body="test3")
|
|
last = self.helper.send(self.room_id, body="test4")
|
|
|
|
# Set the topological token higher than it should be
|
|
token = self.get_success(
|
|
self.store.get_topological_token_for_event(last["event_id"])
|
|
)
|
|
assert token.topological is not None
|
|
event = f"t{token.topological + 1}-{token.stream + 1}"
|
|
|
|
# Purge everything before this topological token
|
|
f = self.get_failure(
|
|
self._storage_controllers.purge_events.purge_history(
|
|
self.room_id, event, True
|
|
),
|
|
SynapseError,
|
|
)
|
|
self.assertIn("greater than forward", f.value.args[0])
|
|
|
|
# Try and get the events
|
|
self.get_success(self.store.get_event(first["event_id"]))
|
|
self.get_success(self.store.get_event(second["event_id"]))
|
|
self.get_success(self.store.get_event(third["event_id"]))
|
|
self.get_success(self.store.get_event(last["event_id"]))
|
|
|
|
def test_purge_room(self) -> None:
|
|
"""
|
|
Purging a room will delete everything about it.
|
|
"""
|
|
# Send four messages to the room
|
|
first = self.helper.send(self.room_id, body="test1")
|
|
|
|
# Get the current room state.
|
|
create_event = self.get_success(
|
|
self._storage_controllers.state.get_current_state_event(
|
|
self.room_id, "m.room.create", ""
|
|
)
|
|
)
|
|
assert create_event is not None
|
|
|
|
# Purge everything before this topological token
|
|
self.get_success(
|
|
self._storage_controllers.purge_events.purge_room(self.room_id)
|
|
)
|
|
|
|
# The events aren't found.
|
|
self.store._invalidate_local_get_event_cache(create_event.event_id)
|
|
self.get_failure(self.store.get_event(create_event.event_id), NotFoundError)
|
|
self.get_failure(self.store.get_event(first["event_id"]), NotFoundError)
|
|
|
|
def test_purge_history_deletes_state_groups(self) -> None:
|
|
"""Test that unreferenced state groups get cleaned up after purge"""
|
|
|
|
# Send four state changes to the room.
|
|
first = self.helper.send_state(
|
|
self.room_id, event_type="m.foo", body={"test": 1}
|
|
)
|
|
second = self.helper.send_state(
|
|
self.room_id, event_type="m.foo", body={"test": 2}
|
|
)
|
|
third = self.helper.send_state(
|
|
self.room_id, event_type="m.foo", body={"test": 3}
|
|
)
|
|
last = self.helper.send_state(
|
|
self.room_id, event_type="m.foo", body={"test": 4}
|
|
)
|
|
|
|
# Get references to the state groups
|
|
event_to_groups = self.get_success(
|
|
self.store._get_state_group_for_events(
|
|
[
|
|
first["event_id"],
|
|
second["event_id"],
|
|
third["event_id"],
|
|
last["event_id"],
|
|
]
|
|
)
|
|
)
|
|
|
|
# Get the topological token
|
|
token = self.get_success(
|
|
self.store.get_topological_token_for_event(last["event_id"])
|
|
)
|
|
token_str = self.get_success(token.to_string(self.hs.get_datastores().main))
|
|
|
|
# Purge everything before this topological token
|
|
self.get_success(
|
|
self._storage_controllers.purge_events.purge_history(
|
|
self.room_id, token_str, True
|
|
)
|
|
)
|
|
|
|
# Advance so that the background jobs to delete the state groups runs
|
|
self.reactor.advance(
|
|
1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
|
|
)
|
|
|
|
# We expect all the state groups associated with events above, except
|
|
# the last one, should return no state.
|
|
state_groups = self.get_success(
|
|
self.state_store._get_state_groups_from_groups(
|
|
list(event_to_groups.values()), StateFilter.all()
|
|
)
|
|
)
|
|
first_state = state_groups[event_to_groups[first["event_id"]]]
|
|
second_state = state_groups[event_to_groups[second["event_id"]]]
|
|
third_state = state_groups[event_to_groups[third["event_id"]]]
|
|
last_state = state_groups[event_to_groups[last["event_id"]]]
|
|
|
|
self.assertEqual(first_state, {})
|
|
self.assertEqual(second_state, {})
|
|
self.assertEqual(third_state, {})
|
|
self.assertNotEqual(last_state, {})
|
|
|
|
def test_purge_unreferenced_state_group(self) -> None:
|
|
"""Test that purging a room also gets rid of unreferenced state groups
|
|
it encounters during the purge.
|
|
|
|
This is important, as otherwise these unreferenced state groups get
|
|
"de-deltaed" during the purge process, consuming lots of disk space.
|
|
"""
|
|
|
|
self.helper.send(self.room_id, body="test1")
|
|
state1 = self.helper.send_state(
|
|
self.room_id, "org.matrix.test", body={"number": 2}
|
|
)
|
|
state2 = self.helper.send_state(
|
|
self.room_id, "org.matrix.test", body={"number": 3}
|
|
)
|
|
self.helper.send(self.room_id, body="test4")
|
|
last = self.helper.send(self.room_id, body="test5")
|
|
|
|
# Create an unreferenced state group that has a prev group of one of the
|
|
# to-be-purged events.
|
|
prev_group = self.get_success(
|
|
self.store._get_state_group_for_event(state1["event_id"])
|
|
)
|
|
unreferenced_state_group = self.get_success(
|
|
self.state_store.store_state_group(
|
|
event_id=last["event_id"],
|
|
room_id=self.room_id,
|
|
prev_group=prev_group,
|
|
delta_ids={("org.matrix.test", ""): state2["event_id"]},
|
|
current_state_ids=None,
|
|
)
|
|
)
|
|
|
|
# Get the topological token
|
|
token = self.get_success(
|
|
self.store.get_topological_token_for_event(last["event_id"])
|
|
)
|
|
token_str = self.get_success(token.to_string(self.hs.get_datastores().main))
|
|
|
|
# Purge everything before this topological token
|
|
self.get_success(
|
|
self._storage_controllers.purge_events.purge_history(
|
|
self.room_id, token_str, True
|
|
)
|
|
)
|
|
|
|
# Advance so that the background jobs to delete the state groups runs
|
|
self.reactor.advance(
|
|
1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
|
|
)
|
|
|
|
# We expect that the unreferenced state group has been deleted from all tables.
|
|
row = self.get_success(
|
|
self.state_store.db_pool.simple_select_one_onecol(
|
|
table="state_groups",
|
|
keyvalues={"id": unreferenced_state_group},
|
|
retcol="id",
|
|
allow_none=True,
|
|
desc="test_purge_unreferenced_state_group",
|
|
)
|
|
)
|
|
self.assertIsNone(row)
|
|
|
|
row = self.get_success(
|
|
self.state_store.db_pool.simple_select_one_onecol(
|
|
table="state_groups_state",
|
|
keyvalues={"state_group": unreferenced_state_group},
|
|
retcol="state_group",
|
|
allow_none=True,
|
|
desc="test_purge_unreferenced_state_group",
|
|
)
|
|
)
|
|
self.assertIsNone(row)
|
|
|
|
row = self.get_success(
|
|
self.state_store.db_pool.simple_select_one_onecol(
|
|
table="state_group_edges",
|
|
keyvalues={"state_group": unreferenced_state_group},
|
|
retcol="state_group",
|
|
allow_none=True,
|
|
desc="test_purge_unreferenced_state_group",
|
|
)
|
|
)
|
|
self.assertIsNone(row)
|
|
|
|
row = self.get_success(
|
|
self.state_store.db_pool.simple_select_one_onecol(
|
|
table="state_groups_pending_deletion",
|
|
keyvalues={"state_group": unreferenced_state_group},
|
|
retcol="state_group",
|
|
allow_none=True,
|
|
desc="test_purge_unreferenced_state_group",
|
|
)
|
|
)
|
|
self.assertIsNone(row)
|
|
|
|
# We expect there to now only be one state group for the room, which is
|
|
# the state group of the last event (as the only outlier).
|
|
state_groups = self.get_success(
|
|
self.state_store.db_pool.simple_select_onecol(
|
|
table="state_groups",
|
|
keyvalues={"room_id": self.room_id},
|
|
retcol="id",
|
|
desc="test_purge_unreferenced_state_group",
|
|
)
|
|
)
|
|
self.assertEqual(len(state_groups), 1)
|
|
|
|
def test_clear_unreferenced_state_groups(self) -> None:
|
|
"""Test that any unreferenced state groups are automatically cleaned up."""
|
|
|
|
self.helper.send(self.room_id, body="test1")
|
|
state1 = self.helper.send_state(
|
|
self.room_id, "org.matrix.test", body={"number": 2}
|
|
)
|
|
# Create enough state events to require multiple batches of
|
|
# mark_unreferenced_state_groups_for_deletion_bg_update to be run.
|
|
for i in range(200):
|
|
self.helper.send_state(self.room_id, "org.matrix.test", body={"number": i})
|
|
self.helper.send(self.room_id, body="test4")
|
|
last = self.helper.send(self.room_id, body="test5")
|
|
|
|
# Create an unreferenced state group that has no prev group.
|
|
unreferenced_free_state_group = self.get_success(
|
|
self.state_store.store_state_group(
|
|
event_id=last["event_id"],
|
|
room_id=self.room_id,
|
|
prev_group=None,
|
|
delta_ids={("org.matrix.test", ""): state1["event_id"]},
|
|
current_state_ids={("org.matrix.test", ""): ""},
|
|
)
|
|
)
|
|
|
|
# Create some unreferenced state groups that have a prev group of one of the
|
|
# existing state groups.
|
|
prev_group = self.get_success(
|
|
self.store._get_state_group_for_event(state1["event_id"])
|
|
)
|
|
unreferenced_end_state_group = self.get_success(
|
|
self.state_store.store_state_group(
|
|
event_id=last["event_id"],
|
|
room_id=self.room_id,
|
|
prev_group=prev_group,
|
|
delta_ids={("org.matrix.test", ""): state1["event_id"]},
|
|
current_state_ids=None,
|
|
)
|
|
)
|
|
another_unreferenced_end_state_group = self.get_success(
|
|
self.state_store.store_state_group(
|
|
event_id=last["event_id"],
|
|
room_id=self.room_id,
|
|
prev_group=unreferenced_end_state_group,
|
|
delta_ids={("org.matrix.test", ""): state1["event_id"]},
|
|
current_state_ids=None,
|
|
)
|
|
)
|
|
|
|
# Add some other unreferenced state groups which lead to a referenced state
|
|
# group.
|
|
# These state groups should not get deleted.
|
|
chain_state_group = self.get_success(
|
|
self.state_store.store_state_group(
|
|
event_id=last["event_id"],
|
|
room_id=self.room_id,
|
|
prev_group=None,
|
|
delta_ids={("org.matrix.test", ""): ""},
|
|
current_state_ids={("org.matrix.test", ""): ""},
|
|
)
|
|
)
|
|
chain_state_group_2 = self.get_success(
|
|
self.state_store.store_state_group(
|
|
event_id=last["event_id"],
|
|
room_id=self.room_id,
|
|
prev_group=chain_state_group,
|
|
delta_ids={("org.matrix.test", ""): ""},
|
|
current_state_ids=None,
|
|
)
|
|
)
|
|
referenced_chain_state_group = self.get_success(
|
|
self.state_store.store_state_group(
|
|
event_id=last["event_id"],
|
|
room_id=self.room_id,
|
|
prev_group=chain_state_group_2,
|
|
delta_ids={("org.matrix.test", ""): ""},
|
|
current_state_ids=None,
|
|
)
|
|
)
|
|
self.get_success(
|
|
self.store.db_pool.simple_insert(
|
|
"event_to_state_groups",
|
|
{
|
|
"event_id": "$new_event",
|
|
"state_group": referenced_chain_state_group,
|
|
},
|
|
)
|
|
)
|
|
|
|
# Insert and run the background update.
|
|
self.get_success(
|
|
self.store.db_pool.simple_insert(
|
|
"background_updates",
|
|
{
|
|
"update_name": _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE,
|
|
"progress_json": "{}",
|
|
},
|
|
)
|
|
)
|
|
self.store.db_pool.updates._all_done = False
|
|
self.wait_for_background_updates()
|
|
|
|
# Advance so that the background job to delete the state groups runs
|
|
self.reactor.advance(
|
|
1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
|
|
)
|
|
|
|
# We expect that the unreferenced free state group has been deleted.
|
|
row = self.get_success(
|
|
self.state_store.db_pool.simple_select_one_onecol(
|
|
table="state_groups",
|
|
keyvalues={"id": unreferenced_free_state_group},
|
|
retcol="id",
|
|
allow_none=True,
|
|
desc="test_purge_unreferenced_state_group",
|
|
)
|
|
)
|
|
self.assertIsNone(row)
|
|
|
|
# We expect that both unreferenced end state groups have been deleted.
|
|
row = self.get_success(
|
|
self.state_store.db_pool.simple_select_one_onecol(
|
|
table="state_groups",
|
|
keyvalues={"id": unreferenced_end_state_group},
|
|
retcol="id",
|
|
allow_none=True,
|
|
desc="test_purge_unreferenced_state_group",
|
|
)
|
|
)
|
|
self.assertIsNone(row)
|
|
row = self.get_success(
|
|
self.state_store.db_pool.simple_select_one_onecol(
|
|
table="state_groups",
|
|
keyvalues={"id": another_unreferenced_end_state_group},
|
|
retcol="id",
|
|
allow_none=True,
|
|
desc="test_purge_unreferenced_state_group",
|
|
)
|
|
)
|
|
self.assertIsNone(row)
|
|
|
|
# We expect there to now only be one state group for the room, which is
|
|
# the state group of the last event (as the only outlier).
|
|
state_groups = self.get_success(
|
|
self.state_store.db_pool.simple_select_onecol(
|
|
table="state_groups",
|
|
keyvalues={"room_id": self.room_id},
|
|
retcol="id",
|
|
desc="test_purge_unreferenced_state_group",
|
|
)
|
|
)
|
|
self.assertEqual(len(state_groups), 210)
|
|
|
|
|
|
class PurgeLocalEventsTests(HomeserverTestCase):
|
|
"""Tests that purging history with ``delete_local_events=False`` preserves
|
|
locally-originated events while still deleting remote ones.
|
|
"""
|
|
|
|
user_id = "@red:server"
|
|
servlets = [room.register_servlets]
|
|
|
|
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
|
|
hs = self.setup_test_homeserver("server")
|
|
return hs
|
|
|
|
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
|
|
self.store = hs.get_datastores().main
|
|
self.state = hs.get_state_handler()
|
|
self._storage_controllers = hs.get_storage_controllers()
|
|
|
|
@parameterized.expand([(RoomVersions.V2,), (RoomVersions.V12,)])
|
|
def test_purge_history_keeps_local_events(self, room_version: RoomVersion) -> None:
|
|
"""
|
|
Purging with `delete_local_events=False` should keep local events.
|
|
"""
|
|
room_id = self.helper.create_room_as(
|
|
self.user_id, room_version=room_version.identifier
|
|
)
|
|
|
|
def inject_remote_event(*, depth: int, prev_event_id: str) -> EventBase:
|
|
pdu = {
|
|
"type": EventTypes.Message,
|
|
"sender": "@bob:other.example.com",
|
|
"room_id": room_id,
|
|
"content": {"msgtype": "m.text", "body": "this is a message"},
|
|
"prev_events": [prev_event_id],
|
|
"auth_events": [],
|
|
"depth": depth,
|
|
"origin_server_ts": 42,
|
|
}
|
|
if room_version.event_format == EventFormatVersions.ROOM_V1_V2:
|
|
# For v1/v2 rooms we need to assign our own arbitrary event ID at creation time
|
|
# For later versions, the event ID is computed (it's a hash)
|
|
pdu["event_id"] = "$remote_abc:other.example.com"
|
|
# The format of prev_events was also different
|
|
pdu["prev_events"] = [[prev_event_id, {}]]
|
|
|
|
remote_event = make_test_pdu_event(pdu, room_version)
|
|
context = self.get_success(self.state.compute_event_context(remote_event))
|
|
persistence = self._storage_controllers.persistence
|
|
assert persistence is not None
|
|
self.get_success(persistence.persist_event(remote_event, context))
|
|
return remote_event
|
|
|
|
# A local event before the purge threshold
|
|
local_event_id = self.helper.send(room_id, body="local event")["event_id"]
|
|
local_event = self.get_success(self.store.get_event(local_event_id))
|
|
|
|
# A remote event before the purge threshold
|
|
remote_event = inject_remote_event(
|
|
prev_event_id=local_event_id,
|
|
depth=local_event.depth + 1,
|
|
)
|
|
|
|
# A final local event, after the cutoff, to serve as the new forward
|
|
# extremity (so the purge doesn't refuse to delete the extremities).
|
|
last_event_id = self.helper.send(room_id, body="new fwd extrem")["event_id"]
|
|
|
|
token = self.get_success(
|
|
self.store.get_topological_token_for_event(last_event_id)
|
|
)
|
|
token_str = self.get_success(token.to_string(self.store))
|
|
|
|
# Purge everything before the last event, keeping local events.
|
|
self.get_success(
|
|
self._storage_controllers.purge_events.purge_history(
|
|
room_id, token_str, delete_local_events=False
|
|
)
|
|
)
|
|
|
|
# The remote event below the cutoff should have been deleted.
|
|
self.get_failure(self.store.get_event(remote_event.event_id), NotFoundError)
|
|
|
|
# Both local event must be preserved
|
|
self.get_success(self.store.get_event(local_event_id))
|
|
self.get_success(self.store.get_event(last_event_id))
|